1/**
2 * This file has no copyright assigned and is placed in the Public Domain.
3 * This file is part of the mingw-w64 runtime package.
4 * No warranty is given; refer to the file DISCLAIMER.PD within this package.
5 */
6
7/* There are 3 separate ways this file is intended to be used:
8
9 1) Included from intrin.h. In this case, all intrinsics in this file get declarations and
10 implementations. No special #defines are needed for this case.
11
12 2) Included from the library versions of these functions (ie mingw-w64-crt\intrincs\*.c). All
13 intrinsics in this file must also be included in the library. In this case, only the
14 specific functions requested will get defined, and they will not be defined as inline. If
15 you have followed the instructions (below) for adding functions to this file, then all you
16 need to have in the .c file is the following:
17
18 #define __INTRINSIC_ONLYSPECIAL
19 #define __INTRINSIC_SPECIAL___stosb // Causes code generation in intrin-impl.h
20
21 #include <intrin.h>
22
23 3) Included from various platform sdk headers. Some platform sdk headers (such as winnt.h)
24 define a subset of intrinsics. To avoid potential conflicts, this file is designed to
25 allow for specific subsets of functions to be defined. This is done by defining the
26 appropriate variable before including this file:
27
28 #define __INTRINSIC_GROUP_WINNT
29 #include <psdk_inc/intrin-impl.h>
30
31 In all cases, it is acceptable to include this file multiple times in any order (ie include
32 winnt.h to get its subset, then include intrin.h to get everything, or vice versa).
33
34 See also the comments at the top of intrin.h.
35*/
36
37/* To add an implementation for a new intrinsic to this file, you should comment out the current prototype in intrin.h.
38 If the function you are adding is not in intrin.h, you should not be adding it to this file. This file is only
39 for MSVC intrinsics.
40
41 Make sure you put your definition in the right section (x86 vs x64), and use this outline when adding definitions
42 to this file:
43
44#if __INTRINSIC_PROLOG(__int2c)
45
46<prototype goes here>
47
48__INTRINSICS_USEINLINE
49<code goes here>
50
51#define __INTRINSIC_DEFINED___int2c
52#endif
53*/
54
55/* Note that there is no file-wide #if to prevent intrin-impl.h from being
56 included multiple times. This is because this file might be included multiple
57 times to define various subsets of the functions it contains. */
58
59/* However we do check for __MINGW_INTRIN_INLINE. In theory this means we
60 can work with other compilers. */
61
62#ifdef __MINGW_INTRIN_INLINE
63
64/* Clang has support for MSVC builtins, GCC doesn't */
65#pragma push_macro("__has_builtin")
66#ifndef __has_builtin
67 #define __has_builtin(x) 0
68#endif
69
70/*
71 * Macro __INTRINSIC_PROLOG uses non-portable Conditional inclusion
72 * (ISO WG14 N2176 (C17) 6.10.1/4). Avoid gcc 7+ -Wexpansion-to-defined
73 * warning enabled by -W or -Wextra option.
74 * In Clang, this warning is enabled by -pedantic.
75 */
76#if defined(__GNUC__) && (__GNUC__ >= 7 || defined(__clang__))
77#pragma GCC diagnostic push
78#pragma GCC diagnostic ignored "-Wexpansion-to-defined"
79#endif
80
81/* These macros are used by the routines below. While this file may be included
82 multiple times, these macros only need to be defined once. */
83#ifndef _INTRIN_MAC_
84#define _INTRIN_MAC_
85
86/* GCC v6 added support for outputting flags. This allows better code to be
87 produced for a number of intrinsics. */
88#ifndef __GCC_ASM_FLAG_OUTPUTS__
89#define __FLAGCONSTRAINT "=qm"
90#define __FLAGSET "\n\tsetc %[old]"
91#define __FLAGCLOBBER1 , "cc"
92#define __FLAGCLOBBER2 "cc"
93#else
94#define __FLAGCONSTRAINT "=@ccc"
95#define __FLAGSET
96#define __FLAGCLOBBER1
97#define __FLAGCLOBBER2
98#endif
99
100/* This macro is used by __stosb, __stosw, __stosd, __stosq */
101
102/* Parameters: (FunctionName, DataType, Operator)
103 FunctionName: Any valid function name
104 DataType: BYTE, WORD, DWORD or DWORD64
105 InstructionSize: b|b, w|w, l|d, q|q */
106
107/* While we don't need the output values for Dest or Count, we
108 must still inform the compiler the asm changes them. */
109#define __buildstos(x, y, z) void x(y *Dest, y Data, size_t Count) \
110{ \
111 __asm__ __volatile__ ("rep stos{" z "}" \
112 : "+D" (Dest), "+c" (Count) \
113 : [Data] "a" (Data) \
114 : "memory"); \
115}
116
117/* This macro is used by InterlockedAnd, InterlockedOr, InterlockedXor, InterlockedAnd64, InterlockedOr64, InterlockedXor64 */
118
119/* Parameters: (FunctionName, DataType, Operator)
120 FunctionName: Any valid function name
121 DataType: __LONG32 or __int64
122 Operator: One of xor, or, and */
123#define __buildlogicali(x, y, o) y x(volatile y *Destination, y Value) \
124{ \
125 return __sync_fetch_and_ ## o(Destination, Value); \
126}
127
128/* This macro is used by InterlockedBitTestAndSet, InterlockedBitTestAndReset, InterlockedBitTestAndComplement,
129 InterlockedBitTestAndSet64, InterlockedBitTestAndReset64, InterlockedBitTestAndComplement64
130 _interlockedbittestandset, _interlockedbittestandreset, _interlockedbittestandcomplement
131 _interlockedbittestandset64, _interlockedbittestandreset64, _interlockedbittestandcomplement64 */
132
133/* Parameters: (FunctionName, DataType, AsmCode, OffsetConstraint)
134 FunctionName: Any valid function name
135 DataType: __LONG32 or __int64
136 OffsetConstraint: either "I" for 32bit data types or "J" for 64. */
137#if (defined(__x86_64__) && !defined(__arm64ec__)) || (defined(_AMD64_) && !defined(_ARM64EC_)) || defined(__i386__) || defined(_X86_)
138#define __buildbittesti(x, y, z, a) unsigned char x(y volatile *Base, y Offset) \
139{ \
140 unsigned char old; \
141 __asm__ __volatile__ (z \
142 : [old] __FLAGCONSTRAINT (old), [Base] "+m" (*Base) \
143 : [Offset] a "r" (Offset) \
144 : "memory" __FLAGCLOBBER1); \
145 return old; \
146}
147#elif defined(__arm__) || defined(_ARM_)
148#define __buildbittesti(x, y, z, a) unsigned char x(y volatile *Base, y Offset) \
149{ \
150 unsigned int old, tmp1, tmp2; \
151 unsigned int bit = 1 << Offset; \
152 __asm__ __volatile__ ("dmb sy\n\t" \
153 "1: ldrex %[old], %[Base]\n\t" \
154 "mov %[tmp1], %[old]\n\t" \
155 z " %[tmp1], %[tmp1], %[bit]\n\t" \
156 "strex %[tmp2], %[tmp1], %[Base]\n\t" \
157 "cmp %[tmp2], #0\n\t" \
158 "bne 1b\n\t" \
159 "dmb sy" \
160 : [old] "=&r" (old), [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [Base] "+m" (*Base) \
161 : [bit] a "r" (bit) \
162 : "memory", "cc"); \
163 return (old >> Offset) & 1; \
164}
165#elif defined(__aarch64__) || defined(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_)
166#define __buildbittesti(x, y, z, a) unsigned char x(y volatile *Base, y Offset) \
167{ \
168 unsigned int old, tmp1, tmp2; \
169 unsigned int bit = 1 << Offset; \
170 __asm__ __volatile__ ("dmb sy\n\t" \
171 "1: ldxr %w[old], %[Base]\n\t" \
172 "mov %w[tmp1], %w[old]\n\t" \
173 z " %w[tmp1], %w[tmp1], %w[bit]\n\t" \
174 "stxr %w[tmp2], %w[tmp1], %[Base]\n\t" \
175 "cmp %w[tmp2], #0\n\t" \
176 "b.ne 1b\n\t" \
177 "dmb sy" \
178 : [old] "=&r" (old), [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [Base] "+m" (*Base) \
179 : [bit] a "r" (bit) \
180 : "memory", "cc"); \
181 return (old >> Offset) & 1; \
182}
183#define __buildbittesti64(x, y, z, a) unsigned char x(y volatile *Base, y Offset) \
184{ \
185 unsigned __int64 old, tmp1; \
186 unsigned int tmp2; \
187 unsigned __int64 bit = 1ULL << Offset; \
188 __asm__ __volatile__ ("dmb sy\n\t" \
189 "1: ldxr %[old], %[Base]\n\t" \
190 "mov %[tmp1], %[old]\n\t" \
191 z " %[tmp1], %[tmp1], %[bit]\n\t" \
192 "stxr %w[tmp2], %[tmp1], %[Base]\n\t" \
193 "cmp %w[tmp2], #0\n\t" \
194 "b.ne 1b\n\t" \
195 "dmb sy" \
196 : [old] "=&r" (old), [tmp1] "=&r" (tmp1), [tmp2] "=&r" (tmp2), [Base] "+m" (*Base) \
197 : [bit] a "r" (bit) \
198 : "memory", "cc"); \
199 return (old >> Offset) & 1; \
200}
201#endif /* (defined(__x86_64__) && !defined(__arm64ec__)) || (defined(_AMD64_) && !defined(_ARM64EC_)) || defined(__i386__) || defined(_X86_) */
202
203/* This macro is used by YieldProcessor when compiling x86 w/o SSE2.
204It generates the same opcodes as _mm_pause. */
205#define __buildpause() __asm__ __volatile__("rep nop")
206
207/* This macro is used by DbgRaiseAssertionFailure and __int2c
208
209Parameters: (IntNum)
210IntNum: Interrupt number in hex */
211#define __buildint(a) __asm__ __volatile__("int {$}" #a :)
212
213/* This macro is used by MemoryBarrier when compiling x86 w/o SSE2.
214Note that on i386, xchg performs an implicit lock. */
215#define __buildmemorybarrier() \
216{ \
217unsigned char Barrier; \
218__asm__ __volatile__("xchg{b %%| }al, %0" :"=m" (Barrier) : /* no inputs */ : "eax", "memory"); \
219}
220
221/* This macro is used by __readfsbyte, __readfsword, __readfsdword
222 __readgsbyte, __readgsword, __readgsdword, __readgsqword
223
224Parameters: (FunctionName, DataType, Segment)
225 FunctionName: Any valid function name
226 DataType: char, short, __LONG32 or __int64
227 Segment: fs or gs
228 Type: b, w, l, q
229 */
230
231#define __buildreadseg(x, y, z, a) y x(unsigned __LONG32 Offset) { \
232 y ret; \
233 __asm__ ("mov{" a " %%" z ":%a[offset], %[ret] | %[ret], " z ":[%[offset]] }" \
234 : [ret] "=r" (ret) \
235 : [offset] "ir" (Offset) \
236 : "memory"); \
237 return ret; \
238}
239
240/* This macro is used by __writefsbyte, __writefsword, __writefsdword
241 __writegsbyte, __writegsword, __writegsdword, __writegsqword
242
243Parameters: (FunctionName, DataType, Segment)
244 FunctionName: Any valid function name
245 DataType: char, short, __LONG32 or __int64
246 Segment: fs or gs
247 Type: b, w, l, q
248 */
249
250#define __buildwriteseg(x, y, z, a) void x(unsigned __LONG32 Offset, y Data) { \
251 __asm__ volatile ("mov{" a " %[Data], %%" z ":%a[offset] | " z ":[%[offset]], %[Data] }" \
252 : \
253 : [offset] "ir" (Offset), [Data] "r" (Data) \
254 : "memory"); \
255}
256
257/* This macro is used by _BitScanForward, _BitScanForward64, _BitScanReverse _BitScanReverse64
258
259Parameters: (FunctionName, DataType, Segment)
260 FunctionName: Any valid function name
261 DataType: unsigned __LONG32 or unsigned __int64
262 Statement: BSF or BSR */
263
264/* GCC v6 added support for outputting flags. This allows better code to be
265 produced for a number of intrinsics. */
266#ifndef __GCC_ASM_FLAG_OUTPUTS__
267#define __buildbitscan(x, y, z) unsigned char x(unsigned __LONG32 *Index, y Mask) \
268{ \
269 y n; \
270 __asm__ (z \
271 : [Index] "=r" (n) \
272 : [Mask] "r" (Mask) \
273 : "cc"); \
274 *Index = n; \
275 return Mask!=0; \
276}
277#else
278#define __buildbitscan(x, y, z) unsigned char x(unsigned __LONG32 *Index, y Mask) \
279{ \
280 y n; \
281 unsigned char old; \
282 __asm__ (z \
283 : "=@ccnz" (old), [Index] "=r" (n) \
284 : [Mask] "r" (Mask)); \
285 *Index = n; \
286 return old; \
287}
288#endif
289
290/* This macro is used by _bittest & _bittest64
291
292Parameters: (FunctionName, DataType, OffsetConstraint)
293 FunctionName: Any valid function name
294 DataType: __LONG32 or __int64
295 Type: l, q
296 OffsetConstraint: either "I" for 32bit data types or "J" for 64.
297
298 */
299#define __buildbittest(x, y, z, a) unsigned char x(const y *Base, y Offset) \
300{ \
301 unsigned char old; \
302 __asm__ ("bt{" z " %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET \
303 : [old] __FLAGCONSTRAINT (old) \
304 : [Offset] a "r" (Offset), [Base] "rm" (*Base) \
305 : __FLAGCLOBBER2); \
306 return old; \
307}
308
309/* This macro is used by _bittestandset, _bittestandreset, _bittestandcomplement,
310 _bittestandset64, _bittestandreset64, _bittestandcomplement64
311
312Parameters: (FunctionName, DataType, Statement, OffsetConstraint)
313 FunctionName: Any valid function name
314 DataType: __LONG32 or __int64
315 Statement: asm statement (bts, btr, btc)
316 OffsetConstraint: either "I" for 32bit data types or "J" for 64.
317 Type: l, q
318 */
319#define __buildbittestand(x, y, z, a, b) unsigned char x(y *Base, y Offset) \
320{ \
321 unsigned char old; \
322 __asm__ (z "{" b " %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET \
323 : [old] __FLAGCONSTRAINT (old), [Base] "+rm" (*Base) \
324 : [Offset] a "r" (Offset) \
325 : __FLAGCLOBBER2); \
326 return old; \
327}
328
329/* This macro is used by __inbyte, __inword, __indword
330
331Parameters: (FunctionName, DataType)
332 FunctionName: Any valid function name
333 DataType: unsigned char, unsigned short, unsigned __LONG32
334 Type: b, w, l
335 */
336#define __build_inport(x, y, z) y x(unsigned short Port) { \
337 y value; \
338 __asm__ __volatile__ ("in{" z " %w[port],%[value]| %[value],%w[port]}" \
339 : [value] "=a" (value) \
340 : [port] "Nd" (Port)); \
341 return value; \
342 }
343
344/* This macro is used by __outbyte, __outword, __outdword
345
346Parameters: (FunctionName, DataType)
347 FunctionName: Any valid function name
348 DataType: unsigned char, unsigned short, unsigned __LONG32
349 Type: b, w, l
350 */
351#define __build_outport(x, y, z) void x(unsigned short Port, y Data) { \
352 __asm__ __volatile__ ("out{" z " %[data],%w[port]| %w[port],%[data]}" \
353 : \
354 : [data] "a" (Data), [port] "Nd" (Port)); \
355 }
356
357/* This macro is used by __inbytestring, __inwordstring, __indwordstring
358
359Parameters: (FunctionName, DataType, InstructionSizeAtt, InstructionSizeIntel)
360 FunctionName: Any valid function name
361 DataType: unsigned char, unsigned short, unsigned __LONG32
362 InstructionSizeAtt: b, w, l
363 InstructionSizeIntel: b, w, d (not b,w,l)
364 */
365#define __build_inportstring(x, y, z, a) void x(unsigned short Port, y *Buffer, unsigned __LONG32 Count) { \
366 __asm__ __volatile__ ("cld ; rep ins{" z "|" a "}" \
367 : "=D" (Buffer), "=c" (Count) \
368 : "d"(Port), "0"(Buffer), "1" (Count) \
369 : "memory"); \
370 }
371
372/* This macro is used by __outbytestring, __outwordstring, __outdwordstring
373
374Parameters: (FunctionName, DataType, InstructionSizeAtt, InstructionSizeIntel)
375 FunctionName: Any valid function name
376 DataType: unsigned char, unsigned short, unsigned __LONG32
377 InstructionSizeAtt: b, w, l
378 InstructionSizeIntel: b, w, d (not b,w,l)
379
380 */
381#define __build_outportstring(x, y, z, a) void x(unsigned short Port, y *Buffer, unsigned __LONG32 Count) { \
382 __asm__ __volatile__ ("cld ; rep outs{" z "|" a "}" \
383 : "=S" (Buffer), "=c" (Count) \
384 : "d"(Port), "0"(Buffer), "1" (Count) \
385 : "memory"); \
386 }
387
388/* This macro is used by __readcr0, __readcr2, __readcr3, __readcr4, __readcr8
389
390Parameters: (FunctionName, DataType, RegisterNumber)
391 FunctionName: Any valid function name
392 DataType: unsigned __LONG32, unsigned __int64
393 RegisterNumber: 0, 2, 3, 4, 8
394
395 */
396#define __build_readcr(x, y, z) y x(void) { \
397 y value; \
398 __asm__ __volatile__ ("mov {%%cr" z ", %[value] | %[value], %%cr" z "}" \
399 : [value] "=q" (value)); \
400 return value; \
401 }
402
403/* This macro is used by __writecr0, __writecr2, __writecr3, __writecr4, __writecr8
404
405Parameters: (FunctionName, DataType, RegisterNumber)
406 FunctionName: Any valid function name
407 DataType: unsigned __LONG32, unsigned __int64
408 RegisterNumber: 0, 2, 3, 4, 8
409
410 */
411#define __build_writecr(x, y, z) void x(y Data) { \
412 __asm__ __volatile__ ("mov {%[Data], %%cr" z "|%%cr" z ", %[Data]}" \
413 : \
414 : [Data] "q" (Data) \
415 : "memory"); \
416 }
417
418/* This macro is used by __movsb, __movsd, __movsq, __movsw
419
420Parameters: (FunctionName, DataType, RegisterNumber)
421 FunctionName: Any valid function name
422 DataType: unsigned char, unsigned short, unsigned __LONG32, unsigned __int64
423 InstructionSize: b, w, d, q
424
425 */
426#define __buildmov(x, y, z, a) void x(y *Destination, y const *Source, size_t Count) \
427{ \
428 __asm__ __volatile__ ( \
429 "rep movs{" z "|" a "}" \
430 : "=D" (Destination), "=S" (Source), "=c" (Count) \
431 : "0" (Destination), "1" (Source), "2" (Count) \
432 : "memory"); \
433}
434
435#endif /* _INTRIN_MAC_ */
436
437/* The Barrier functions can never be in the library. Since gcc only
438supports ReadWriteBarrier, map all 3 to do the same. */
439#ifndef _ReadWriteBarrier
440
441#define _ReadWriteBarrier() __asm__ __volatile__ ("" ::: "memory")
442#define _ReadBarrier _ReadWriteBarrier
443#define _WriteBarrier _ReadWriteBarrier
444
445#endif
446
447/* The logic for this macro is:
448 if the function is not yet defined AND
449 (
450 (if we are not just defining special OR
451 (we are defining special AND this is one of the ones we are defining)
452 )
453 )
454*/
455#define __INTRINSIC_PROLOG(name) (!defined(__INTRINSIC_DEFINED_ ## name)) && ((!defined (__INTRINSIC_ONLYSPECIAL)) || (defined (__INTRINSIC_ONLYSPECIAL) && defined(__INTRINSIC_SPECIAL_ ## name)))
456
457#ifdef __INTRINSIC_ONLYSPECIAL
458#define __INTRINSICS_USEINLINE
459#else
460#define __INTRINSICS_USEINLINE __MINGW_INTRIN_INLINE
461#endif
462
463/* Normally __INTRINSIC_ONLYSPECIAL is used to indicate that we are
464 being included in the library version of the intrinsic (case 2). However,
465 that really only affects the definition of __INTRINSICS_USEINLINE.
466 So here we are letting it serve an additional purpose of only defining
467 the intrinsics for a certain file (case 3). For example, to create the
468 intrinsics for the functions in winnt.h, define __INTRINSIC_GROUP_WINNT.
469
470 Note that this file can be included multiple times, and as a result
471 there can be overlap (definitions that appear in more than one
472 file). This is handled by __INTRINSIC_DEFINED_*
473
474 If no groups are defined (such as what happens when including intrin.h),
475 all intrinsics are defined. */
476
477/* If __INTRINSIC_ONLYSPECIAL is defined at this point, we are processing case 2. In
478 that case, don't go looking for groups */
479#ifndef __INTRINSIC_ONLYSPECIAL
480
481#ifdef __INTRINSIC_GROUP_WINNT
482#undef __INTRINSIC_GROUP_WINNT /* Remove this for efficiency if intrin-impl.h is included again */
483
484/* Note that this gets undefined at the end of this file */
485#define __INTRINSIC_ONLYSPECIAL
486
487#define __INTRINSIC_SPECIAL___faststorefence
488#define __INTRINSIC_SPECIAL___int2c
489#define __INTRINSIC_SPECIAL___stosb
490#define __INTRINSIC_SPECIAL___stosd
491#define __INTRINSIC_SPECIAL___stosq
492#define __INTRINSIC_SPECIAL___stosw
493#define __INTRINSIC_SPECIAL__InterlockedAnd
494#define __INTRINSIC_SPECIAL__InterlockedAnd64
495#define __INTRINSIC_SPECIAL__interlockedbittestandcomplement
496#define __INTRINSIC_SPECIAL__interlockedbittestandcomplement64
497#define __INTRINSIC_SPECIAL__interlockedbittestandreset
498#define __INTRINSIC_SPECIAL__interlockedbittestandreset64
499#define __INTRINSIC_SPECIAL__interlockedbittestandset
500#define __INTRINSIC_SPECIAL__interlockedbittestandset64
501#define __INTRINSIC_SPECIAL__InterlockedOr
502#define __INTRINSIC_SPECIAL__InterlockedOr64
503#define __INTRINSIC_SPECIAL__InterlockedXor
504#define __INTRINSIC_SPECIAL__InterlockedXor64
505#define __INTRINSIC_SPECIAL_InterlockedBitTestAndComplement
506#define __INTRINSIC_SPECIAL_InterlockedBitTestAndComplement64
507#define __INTRINSIC_SPECIAL_InterlockedBitTestAndReset
508#define __INTRINSIC_SPECIAL_InterlockedBitTestAndReset64
509#define __INTRINSIC_SPECIAL_InterlockedBitTestAndSet
510#define __INTRINSIC_SPECIAL_InterlockedBitTestAndSet64
511#define __INTRINSIC_SPECIAL__InterlockedIncrement16
512#define __INTRINSIC_SPECIAL__InterlockedDecrement16
513#define __INTRINSIC_SPECIAL__InterlockedCompareExchange16
514#define __INTRINSIC_SPECIAL__InterlockedIncrement
515#define __INTRINSIC_SPECIAL__InterlockedDecrement
516#define __INTRINSIC_SPECIAL__InterlockedAdd
517#define __INTRINSIC_SPECIAL__InterlockedExchange
518#define __INTRINSIC_SPECIAL__InterlockedExchangeAdd
519#define __INTRINSIC_SPECIAL__InterlockedCompareExchange
520#define __INTRINSIC_SPECIAL__InterlockedIncrement64
521#define __INTRINSIC_SPECIAL__InterlockedDecrement64
522#define __INTRINSIC_SPECIAL__InterlockedAdd64
523#define __INTRINSIC_SPECIAL__InterlockedExchangeAdd64
524#define __INTRINSIC_SPECIAL__InterlockedExchange64
525#define __INTRINSIC_SPECIAL__InterlockedCompareExchange64
526#define __INTRINSIC_SPECIAL__InterlockedExchangePointer
527#define __INTRINSIC_SPECIAL__InterlockedCompareExchangePointer
528#define __INTRINSIC_SPECIAL___readgsbyte
529#define __INTRINSIC_SPECIAL___readgsword
530#define __INTRINSIC_SPECIAL___readgsdword
531#define __INTRINSIC_SPECIAL___readgsqword
532#define __INTRINSIC_SPECIAL___writegsbyte
533#define __INTRINSIC_SPECIAL___writegsword
534#define __INTRINSIC_SPECIAL___writegsdword
535#define __INTRINSIC_SPECIAL___writegsqword
536#define __INTRINSIC_SPECIAL___readfsbyte
537#define __INTRINSIC_SPECIAL___readfsword
538#define __INTRINSIC_SPECIAL___readfsdword
539#define __INTRINSIC_SPECIAL___writefsbyte
540#define __INTRINSIC_SPECIAL___writefsword
541#define __INTRINSIC_SPECIAL___writefsdword
542#define __INTRINSIC_SPECIAL__BitScanForward
543#define __INTRINSIC_SPECIAL__BitScanForward64
544#define __INTRINSIC_SPECIAL__BitScanReverse
545#define __INTRINSIC_SPECIAL__BitScanReverse64
546#define __INTRINSIC_SPECIAL__bittest
547#define __INTRINSIC_SPECIAL__bittestandset
548#define __INTRINSIC_SPECIAL__bittestandreset
549#define __INTRINSIC_SPECIAL__bittestandcomplement
550#define __INTRINSIC_SPECIAL__bittest64
551#define __INTRINSIC_SPECIAL__bittestandset64
552#define __INTRINSIC_SPECIAL__bittestandreset64
553#define __INTRINSIC_SPECIAL__bittestandcomplement64
554#define __INTRINSIC_SPECIAL___movsb
555#define __INTRINSIC_SPECIAL___movsw
556#define __INTRINSIC_SPECIAL___movsd
557#define __INTRINSIC_SPECIAL___movsq
558
559#endif /* __INTRINSIC_GROUP_WINNT */
560
561#ifdef __INTRINSIC_GROUP_WINBASE
562#undef __INTRINSIC_GROUP_WINBASE /* Remove this for efficiency if intrin-impl.h is included again */
563
564/* Note that this gets undefined at the end of this file */
565#define __INTRINSIC_ONLYSPECIAL
566
567#define __INTRINSIC_SPECIAL__InterlockedIncrement
568#define __INTRINSIC_SPECIAL__InterlockedDecrement
569#define __INTRINSIC_SPECIAL__InterlockedAdd
570#define __INTRINSIC_SPECIAL__InterlockedExchange
571#define __INTRINSIC_SPECIAL__InterlockedExchangeAdd
572#define __INTRINSIC_SPECIAL__InterlockedCompareExchange
573#define __INTRINSIC_SPECIAL__InterlockedCompareExchangePointer
574#define __INTRINSIC_SPECIAL__InterlockedExchangePointer
575#define __INTRINSIC_SPECIAL__InterlockedAnd64
576#define __INTRINSIC_SPECIAL__InterlockedOr64
577#define __INTRINSIC_SPECIAL__InterlockedXor64
578#define __INTRINSIC_SPECIAL__InterlockedIncrement64
579#define __INTRINSIC_SPECIAL__InterlockedDecrement64
580#define __INTRINSIC_SPECIAL__InterlockedAdd64
581#define __INTRINSIC_SPECIAL__InterlockedExchange64
582#define __INTRINSIC_SPECIAL__InterlockedExchangeAdd64
583#define __INTRINSIC_SPECIAL__InterlockedCompareExchange64
584
585#endif /* __INTRINSIC_GROUP_WINBASE */
586
587/* To add an additional group, put the #ifdef and definitions here. */
588
589#endif /* __INTRINSIC_ONLYSPECIAL */
590
591#ifdef __cplusplus
592extern "C" {
593#endif
594
595/* Before 4.9.2, ia32intrin.h had broken versions of these. */
596#undef _lrotl
597#undef _lrotr
598
599#if __INTRINSIC_PROLOG(_lrotl)
600unsigned long _lrotl(unsigned long __X, int __C);
601#if !__has_builtin(_lrotl)
602__INTRINSICS_USEINLINE
603unsigned long _lrotl(unsigned long __X, int __C)
604{
605 return (__X << __C) | (__X >> ((sizeof(long) * 8) - __C));
606}
607#endif
608#define __INTRINSIC_DEFINED__lrotl
609#endif /* __INTRINSIC_PROLOG */
610
611#if __INTRINSIC_PROLOG(_lrotr)
612unsigned long _lrotr(unsigned long __X, int __C);
613#if !__has_builtin(_lrotr)
614__INTRINSICS_USEINLINE
615unsigned long _lrotr(unsigned long __X, int __C)
616{
617 return (__X >> __C) | (__X << ((sizeof(long) * 8) - __C));
618}
619#endif
620#define __INTRINSIC_DEFINED__lrotr
621#endif /* __INTRINSIC_PROLOG */
622
623#if __INTRINSIC_PROLOG(_rotl8)
624unsigned char _rotl8(unsigned char __X, unsigned char __C);
625#if !__has_builtin(_rotl8)
626__INTRINSICS_USEINLINE
627unsigned char _rotl8(unsigned char __X, unsigned char __C)
628{
629 return (__X << __C) | (__X >> (8 - __C));
630}
631#endif
632#define __INTRINSIC_DEFINED__rotl8
633#endif /* __INTRINSIC_PROLOG */
634
635#if __INTRINSIC_PROLOG(_rotr8)
636unsigned char _rotr8(unsigned char __X, unsigned char __C);
637#if !__has_builtin(_rotr8)
638__INTRINSICS_USEINLINE
639unsigned char _rotr8(unsigned char __X, unsigned char __C)
640{
641 return (__X >> __C) | (__X << (8 - __C));
642}
643#endif
644#define __INTRINSIC_DEFINED__rotr8
645#endif /* __INTRINSIC_PROLOG */
646
647#if __INTRINSIC_PROLOG(_rotl16)
648unsigned short _rotl16(unsigned short __X, unsigned char __C);
649#if !__has_builtin(_rotl16)
650__INTRINSICS_USEINLINE
651unsigned short _rotl16(unsigned short __X, unsigned char __C)
652{
653 return (__X << __C) | (__X >> (16 - __C));
654}
655#endif
656#define __INTRINSIC_DEFINED__rotl16
657#endif /* __INTRINSIC_PROLOG */
658
659#if __INTRINSIC_PROLOG(_rotr16)
660unsigned short _rotr16(unsigned short __X, unsigned char __C);
661#if !__has_builtin(_rotr16)
662__INTRINSICS_USEINLINE
663unsigned short _rotr16(unsigned short __X, unsigned char __C)
664{
665 return (__X >> __C) | (__X << (16 - __C));
666}
667#endif
668#define __INTRINSIC_DEFINED__rotr16
669#endif /* __INTRINSIC_PROLOG */
670
671#if (defined(__x86_64__) && !defined(__arm64ec__)) || (defined(_AMD64_) && !defined(_ARM64EC_))
672#if __INTRINSIC_PROLOG(__faststorefence)
673void __faststorefence(void);
674#if !__has_builtin(__faststorefence)
675__INTRINSICS_USEINLINE
676void __faststorefence(void) {
677 /* Turns out this is actually faster than MS's "trick" on newer cpus. Note
678 that this builtin performs an implicit ReadWriteBarrier. */
679 __builtin_ia32_sfence();
680}
681#endif
682#define __INTRINSIC_DEFINED___faststorefence
683#endif /* __INTRINSIC_PROLOG */
684
685#if __INTRINSIC_PROLOG(__stosq)
686__MINGW_EXTENSION void __stosq(unsigned __int64 *, unsigned __int64, size_t);
687#if !__has_builtin(__stosq)
688__INTRINSICS_USEINLINE
689__buildstos(__stosq, unsigned __int64, "q|q")
690#endif
691#define __INTRINSIC_DEFINED___stosq
692#endif /* __INTRINSIC_PROLOG */
693
694#if __INTRINSIC_PROLOG(_interlockedbittestandset64)
695__MINGW_EXTENSION unsigned char _interlockedbittestandset64(__int64 volatile *a, __int64 b);
696#if !__has_builtin(_interlockedbittestandset64)
697__INTRINSICS_USEINLINE
698__buildbittesti(_interlockedbittestandset64, __int64, "lock bts{q %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "J")
699#endif
700#define __INTRINSIC_DEFINED__interlockedbittestandset64
701#endif /* __INTRINSIC_PROLOG */
702
703#if __INTRINSIC_PROLOG(_interlockedbittestandreset64)
704__MINGW_EXTENSION unsigned char _interlockedbittestandreset64(__int64 volatile *a, __int64 b);
705#if !__has_builtin(_interlockedbittestandreset64)
706__INTRINSICS_USEINLINE
707__buildbittesti(_interlockedbittestandreset64, __int64, "lock btr{q %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "J")
708#endif
709#define __INTRINSIC_DEFINED__interlockedbittestandreset64
710#endif /* __INTRINSIC_PROLOG */
711
712#if __INTRINSIC_PROLOG(_interlockedbittestandcomplement64)
713__MINGW_EXTENSION unsigned char _interlockedbittestandcomplement64(__int64 volatile *a, __int64 b);
714#if !__has_builtin(_interlockedbittestandcomplement64)
715__INTRINSICS_USEINLINE
716__buildbittesti(_interlockedbittestandcomplement64, __int64, "lock btc{q %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "J")
717#endif
718#define __INTRINSIC_DEFINED__interlockedbittestandcomplement64
719#endif /* __INTRINSIC_PROLOG */
720
721#if __INTRINSIC_PROLOG(InterlockedBitTestAndSet64)
722__MINGW_EXTENSION unsigned char InterlockedBitTestAndSet64(volatile __int64 *a, __int64 b);
723#if !__has_builtin(InterlockedBitTestAndSet64)
724__INTRINSICS_USEINLINE
725__buildbittesti(InterlockedBitTestAndSet64, __int64, "lock bts{q %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "J")
726#endif
727#define __INTRINSIC_DEFINED_InterlockedBitTestAndSet64
728#endif /* __INTRINSIC_PROLOG */
729
730#if __INTRINSIC_PROLOG(InterlockedBitTestAndReset64)
731__MINGW_EXTENSION unsigned char InterlockedBitTestAndReset64(volatile __int64 *a, __int64 b);
732#if !__has_builtin(InterlockedBitTestAndReset64)
733__INTRINSICS_USEINLINE
734__buildbittesti(InterlockedBitTestAndReset64, __int64, "lock btr{q %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "J")
735#endif
736#define __INTRINSIC_DEFINED_InterlockedBitTestAndReset64
737#endif /* __INTRINSIC_PROLOG */
738
739#if __INTRINSIC_PROLOG(InterlockedBitTestAndComplement64)
740__MINGW_EXTENSION unsigned char InterlockedBitTestAndComplement64(volatile __int64 *a, __int64 b);
741#if !__has_builtin(InterlockedBitTestAndComplement64)
742__INTRINSICS_USEINLINE
743__buildbittesti(InterlockedBitTestAndComplement64, __int64, "lock btc{q %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "J")
744#endif
745#define __INTRINSIC_DEFINED_InterlockedBitTestAndComplement64
746#endif /* __INTRINSIC_PROLOG */
747
748#if __INTRINSIC_PROLOG(_InterlockedAnd64)
749__MINGW_EXTENSION __int64 _InterlockedAnd64(__int64 volatile *, __int64);
750#if !__has_builtin(_InterlockedAnd64)
751__INTRINSICS_USEINLINE
752__buildlogicali(_InterlockedAnd64, __int64, and)
753#endif
754#define __INTRINSIC_DEFINED__InterlockedAnd64
755#endif /* __INTRINSIC_PROLOG */
756
757#if __INTRINSIC_PROLOG(_InterlockedOr64)
758__MINGW_EXTENSION __int64 _InterlockedOr64(__int64 volatile *, __int64);
759#if !__has_builtin(_InterlockedOr64)
760__INTRINSICS_USEINLINE
761__buildlogicali(_InterlockedOr64, __int64, or)
762#endif
763#define __INTRINSIC_DEFINED__InterlockedOr64
764#endif /* __INTRINSIC_PROLOG */
765
766#if __INTRINSIC_PROLOG(_InterlockedXor64)
767__MINGW_EXTENSION __int64 _InterlockedXor64(__int64 volatile *, __int64);
768#if !__has_builtin(_InterlockedXor64)
769__INTRINSICS_USEINLINE
770__buildlogicali(_InterlockedXor64, __int64, xor)
771#endif
772#define __INTRINSIC_DEFINED__InterlockedXor64
773#endif /* __INTRINSIC_PROLOG */
774
775#if __INTRINSIC_PROLOG(_InterlockedIncrement64)
776__MINGW_EXTENSION __int64 _InterlockedIncrement64(__int64 volatile *Addend);
777#if !__has_builtin(_InterlockedIncrement64)
778__MINGW_EXTENSION __INTRINSICS_USEINLINE
779__int64 _InterlockedIncrement64(__int64 volatile *Addend) {
780 return __sync_add_and_fetch(Addend, 1);
781}
782#endif
783#define __INTRINSIC_DEFINED__InterlockedIncrement64
784#endif /* __INTRINSIC_PROLOG */
785
786#if __INTRINSIC_PROLOG(_InterlockedDecrement64)
787__MINGW_EXTENSION __int64 _InterlockedDecrement64(__int64 volatile *Addend);
788#if !__has_builtin(_InterlockedDecrement64)
789__MINGW_EXTENSION __INTRINSICS_USEINLINE
790__int64 _InterlockedDecrement64(__int64 volatile *Addend) {
791 return __sync_sub_and_fetch(Addend, 1);
792}
793#endif
794#define __INTRINSIC_DEFINED__InterlockedDecrement64
795#endif /* __INTRINSIC_PROLOG */
796
797#if __INTRINSIC_PROLOG(_InterlockedExchange64)
798__MINGW_EXTENSION __int64 _InterlockedExchange64(__int64 volatile *Target, __int64 Value);
799#if !__has_builtin(_InterlockedExchange64)
800__MINGW_EXTENSION __INTRINSICS_USEINLINE
801__int64 _InterlockedExchange64(__int64 volatile *Target, __int64 Value) {
802 return __sync_lock_test_and_set(Target, Value);
803}
804#endif
805#define __INTRINSIC_DEFINED__InterlockedExchange64
806#endif /* __INTRINSIC_PROLOG */
807
808#if __INTRINSIC_PROLOG(_InterlockedExchangeAdd64)
809__MINGW_EXTENSION __int64 _InterlockedExchangeAdd64(__int64 volatile *Addend, __int64 Value);
810#if !__has_builtin(_InterlockedExchangeAdd64)
811__MINGW_EXTENSION __INTRINSICS_USEINLINE
812__int64 _InterlockedExchangeAdd64(__int64 volatile *Addend, __int64 Value) {
813 return __sync_fetch_and_add(Addend, Value);
814}
815#endif
816#define __INTRINSIC_DEFINED__InterlockedExchangeAdd64
817#endif /* __INTRINSIC_PROLOG */
818
819#if __INTRINSIC_PROLOG(__readgsbyte)
820unsigned char __readgsbyte(unsigned __LONG32 Offset);
821#if !__has_builtin(__readgsbyte)
822__INTRINSICS_USEINLINE
823__buildreadseg(__readgsbyte, unsigned char, "gs", "b")
824#endif
825#define __INTRINSIC_DEFINED___readgsbyte
826#endif /* __INTRINSIC_PROLOG */
827
828#if __INTRINSIC_PROLOG(__readgsword)
829unsigned short __readgsword(unsigned __LONG32 Offset);
830#if !__has_builtin(__readgsword)
831__INTRINSICS_USEINLINE
832__buildreadseg(__readgsword, unsigned short, "gs", "w")
833#endif
834#define __INTRINSIC_DEFINED___readgsword
835#endif /* __INTRINSIC_PROLOG */
836
837#if __INTRINSIC_PROLOG(__readgsdword)
838unsigned __LONG32 __readgsdword(unsigned __LONG32 Offset);
839#if !__has_builtin(__readgsdword)
840__INTRINSICS_USEINLINE
841__buildreadseg(__readgsdword, unsigned __LONG32, "gs", "l")
842#endif
843#define __INTRINSIC_DEFINED___readgsdword
844#endif /* __INTRINSIC_PROLOG */
845
846#if __INTRINSIC_PROLOG(__readgsqword)
847__MINGW_EXTENSION unsigned __int64 __readgsqword(unsigned __LONG32 Offset);
848#if !__has_builtin(__readgsqword)
849__MINGW_EXTENSION __INTRINSICS_USEINLINE
850__buildreadseg(__readgsqword, unsigned __int64, "gs", "q")
851#endif
852#define __INTRINSIC_DEFINED___readgsqword
853#endif /* __INTRINSIC_PROLOG */
854
855#if __INTRINSIC_PROLOG(__writegsbyte)
856void __writegsbyte(unsigned __LONG32 Offset,unsigned char Data);
857#if !__has_builtin(__writegsbyte)
858__INTRINSICS_USEINLINE
859__buildwriteseg(__writegsbyte, unsigned char, "gs", "b")
860#endif
861#define __INTRINSIC_DEFINED___writegsbyte
862#endif /* __INTRINSIC_PROLOG */
863
864#if __INTRINSIC_PROLOG(__writegsword)
865void __writegsword(unsigned __LONG32 Offset,unsigned short Data);
866#if !__has_builtin(__writegsword)
867__INTRINSICS_USEINLINE
868__buildwriteseg(__writegsword, unsigned short, "gs", "w")
869#endif
870#define __INTRINSIC_DEFINED___writegsword
871#endif /* __INTRINSIC_PROLOG */
872
873#if __INTRINSIC_PROLOG(__writegsdword)
874void __writegsdword(unsigned __LONG32 Offset,unsigned __LONG32 Data);
875#if !__has_builtin(__writegsdword)
876__INTRINSICS_USEINLINE
877__buildwriteseg(__writegsdword, unsigned __LONG32, "gs", "l")
878#endif
879#define __INTRINSIC_DEFINED___writegsdword
880#endif /* __INTRINSIC_PROLOG */
881
882#if __INTRINSIC_PROLOG(__writegsqword)
883__MINGW_EXTENSION void __writegsqword(unsigned __LONG32 Offset,unsigned __int64 Data);
884#if !__has_builtin(__writegsqword)
885__MINGW_EXTENSION __INTRINSICS_USEINLINE
886__buildwriteseg(__writegsqword, unsigned __int64, "gs", "q")
887#endif
888#define __INTRINSIC_DEFINED___writegsqword
889#endif /* __INTRINSIC_PROLOG */
890
891#if __INTRINSIC_PROLOG(_BitScanForward64)
892__MINGW_EXTENSION unsigned char _BitScanForward64(unsigned __LONG32 *Index, unsigned __int64 Mask);
893#if !__has_builtin(_BitScanForward64)
894__MINGW_EXTENSION __INTRINSICS_USEINLINE
895__buildbitscan(_BitScanForward64, unsigned __int64, "bsf{q %[Mask],%[Index] | %[Index],%[Mask]}")
896#endif
897#define __INTRINSIC_DEFINED__BitScanForward64
898#endif /* __INTRINSIC_PROLOG */
899
900#if __INTRINSIC_PROLOG(_BitScanReverse64)
901__MINGW_EXTENSION unsigned char _BitScanReverse64(unsigned __LONG32 *Index, unsigned __int64 Mask);
902#if !__has_builtin(_BitScanReverse64)
903__MINGW_EXTENSION __INTRINSICS_USEINLINE
904__buildbitscan(_BitScanReverse64, unsigned __int64, "bsr{q %[Mask],%[Index] | %[Index],%[Mask]}")
905#endif
906#define __INTRINSIC_DEFINED__BitScanReverse64
907#endif /* __INTRINSIC_PROLOG */
908
909#if __INTRINSIC_PROLOG(_bittest64)
910__MINGW_EXTENSION unsigned char _bittest64(__int64 const *a, __int64 b);
911#if !__has_builtin(_bittest64)
912__MINGW_EXTENSION __INTRINSICS_USEINLINE
913__buildbittest(_bittest64, __int64, "q", "J")
914#endif
915#define __INTRINSIC_DEFINED__bittest64
916#endif /* __INTRINSIC_PROLOG */
917
918#if __INTRINSIC_PROLOG(_bittestandset64)
919__MINGW_EXTENSION unsigned char _bittestandset64(__int64 *a, __int64 b);
920#if !__has_builtin(_bittestandset64)
921__MINGW_EXTENSION __INTRINSICS_USEINLINE
922__buildbittestand(_bittestandset64, __int64, "bts", "J", "q")
923#endif
924#define __INTRINSIC_DEFINED__bittestandset64
925#endif /* __INTRINSIC_PROLOG */
926
927#if __INTRINSIC_PROLOG(_bittestandreset64)
928__MINGW_EXTENSION unsigned char _bittestandreset64(__int64 *a, __int64 b);
929#if !__has_builtin(_bittestandreset64)
930__MINGW_EXTENSION __INTRINSICS_USEINLINE
931__buildbittestand(_bittestandreset64, __int64, "btr", "J", "q")
932#endif
933#define __INTRINSIC_DEFINED__bittestandreset64
934#endif /* __INTRINSIC_PROLOG */
935
936#if __INTRINSIC_PROLOG(_bittestandcomplement64)
937__MINGW_EXTENSION unsigned char _bittestandcomplement64(__int64 *a, __int64 b);
938#if !__has_builtin(_bittestandcomplement64)
939__MINGW_EXTENSION __INTRINSICS_USEINLINE
940__buildbittestand(_bittestandcomplement64, __int64, "btc", "J", "q")
941#endif
942#define __INTRINSIC_DEFINED__bittestandcomplement64
943#endif /* __INTRINSIC_PROLOG */
944
945#if __INTRINSIC_PROLOG(__readcr0)
946__MINGW_EXTENSION unsigned __int64 __readcr0(void);
947#if !__has_builtin(__readcr0)
948__INTRINSICS_USEINLINE
949__build_readcr(__readcr0, unsigned __int64, "0")
950#endif
951#define __INTRINSIC_DEFINED___readcr0
952#endif /* __INTRINSIC_PROLOG */
953
954#if __INTRINSIC_PROLOG(__readcr2)
955__MINGW_EXTENSION unsigned __int64 __readcr2(void);
956#if !__has_builtin(__readcr2)
957__INTRINSICS_USEINLINE
958__build_readcr(__readcr2, unsigned __int64, "2")
959#endif
960#define __INTRINSIC_DEFINED___readcr2
961#endif /* __INTRINSIC_PROLOG */
962
963#if __INTRINSIC_PROLOG(__readcr3)
964__MINGW_EXTENSION unsigned __int64 __readcr3(void);
965#if !__has_builtin(__readcr3)
966__INTRINSICS_USEINLINE
967__build_readcr(__readcr3, unsigned __int64, "3")
968#endif
969#define __INTRINSIC_DEFINED___readcr3
970#endif /* __INTRINSIC_PROLOG */
971
972#if __INTRINSIC_PROLOG(__readcr4)
973__MINGW_EXTENSION unsigned __int64 __readcr4(void);
974#if !__has_builtin(__readcr4)
975__INTRINSICS_USEINLINE
976__build_readcr(__readcr4, unsigned __int64, "4")
977#endif
978#define __INTRINSIC_DEFINED___readcr4
979#endif /* __INTRINSIC_PROLOG */
980
981#if __INTRINSIC_PROLOG(__readcr8)
982__MINGW_EXTENSION unsigned __int64 __readcr8(void);
983#if !__has_builtin(__readcr8)
984__INTRINSICS_USEINLINE
985__build_readcr(__readcr8, unsigned __int64, "8")
986#endif
987#define __INTRINSIC_DEFINED___readcr8
988#endif /* __INTRINSIC_PROLOG */
989
990#if __INTRINSIC_PROLOG(__writecr0)
991__MINGW_EXTENSION void __writecr0(unsigned __int64);
992#if !__has_builtin(__writecr0)
993__INTRINSICS_USEINLINE
994__build_writecr(__writecr0, unsigned __int64, "0")
995#endif
996#define __INTRINSIC_DEFINED___writecr0
997#endif /* __INTRINSIC_PROLOG */
998
999#if __INTRINSIC_PROLOG(__writecr3)
1000__MINGW_EXTENSION void __writecr3(unsigned __int64);
1001#if !__has_builtin(__writecr3)
1002__INTRINSICS_USEINLINE
1003__build_writecr(__writecr3, unsigned __int64, "3")
1004#endif
1005#define __INTRINSIC_DEFINED___writecr3
1006#endif /* __INTRINSIC_PROLOG */
1007
1008#if __INTRINSIC_PROLOG(__writecr4)
1009__MINGW_EXTENSION void __writecr4(unsigned __int64);
1010#if !__has_builtin(__writecr4)
1011__INTRINSICS_USEINLINE
1012__build_writecr(__writecr4, unsigned __int64, "4")
1013#endif
1014#define __INTRINSIC_DEFINED___writecr4
1015#endif /* __INTRINSIC_PROLOG */
1016
1017#if __INTRINSIC_PROLOG(__writecr8)
1018__MINGW_EXTENSION void __writecr8(unsigned __int64);
1019#if !__has_builtin(__writecr8)
1020__INTRINSICS_USEINLINE
1021__build_writecr(__writecr8, unsigned __int64, "8")
1022#endif
1023#define __INTRINSIC_DEFINED___writecr8
1024#endif /* __INTRINSIC_PROLOG */
1025
1026#if __INTRINSIC_PROLOG(__movsq)
1027__MINGW_EXTENSION void __movsq(unsigned __int64 *Dest, unsigned __int64 const *Source, size_t Count);
1028#if !__has_builtin(__movsq)
1029__MINGW_EXTENSION __INTRINSICS_USEINLINE
1030__buildmov(__movsq, unsigned __int64, "q", "q")
1031#endif
1032#define __INTRINSIC_DEFINED___movsq
1033#endif /* __INTRINSIC_PROLOG */
1034
1035#if __INTRINSIC_PROLOG(_umul128)
1036unsigned __int64 _umul128(unsigned __int64, unsigned __int64, unsigned __int64 *);
1037#if !__has_builtin(_umul128)
1038__INTRINSICS_USEINLINE
1039unsigned __int64 _umul128(unsigned __int64 a, unsigned __int64 b, unsigned __int64 *hi)
1040{
1041 __MINGW_EXTENSION union { unsigned __int128 v; unsigned __int64 sv[2]; } var;
1042 var.v = a;
1043 var.v *= b;
1044 if (hi) *hi = var.sv[1];
1045 return var.sv[0];
1046}
1047#endif
1048#define __INTRINSIC_DEFINED__umul128
1049#endif /* __INTRINSIC_PROLOG */
1050
1051#if __INTRINSIC_PROLOG(_mul128)
1052__int64 _mul128(__int64, __int64, __int64 *);
1053#if !__has_builtin(_mul128)
1054__INTRINSICS_USEINLINE
1055__int64 _mul128(__int64 a, __int64 b, __int64 *hi)
1056{
1057 __MINGW_EXTENSION union { __int128 v; __int64 sv[2]; } var;
1058 var.v = a;
1059 var.v *= b;
1060 if (hi) *hi = var.sv[1];
1061 return var.sv[0];
1062}
1063#endif
1064#define __INTRINSIC_DEFINED__mul128
1065#endif /* __INTRINSIC_PROLOG */
1066
1067#if __INTRINSIC_PROLOG(__shiftleft128)
1068unsigned __int64 __shiftleft128(unsigned __int64 LowPart, unsigned __int64 HighPart, unsigned char Shift);
1069#if !__has_builtin(__shiftleft128)
1070__INTRINSICS_USEINLINE
1071unsigned __int64 __shiftleft128 (unsigned __int64 LowPart, unsigned __int64 HighPart, unsigned char Shift)
1072{
1073 unsigned __int64 ret;
1074
1075 __asm__ ("shld {%[Shift],%[LowPart],%[HighPart]|%[HighPart], %[LowPart], %[Shift]}"
1076 : [ret] "=r" (ret)
1077 : [LowPart] "r" (LowPart), [HighPart] "0" (HighPart), [Shift] "Jc" (Shift)
1078 : "cc");
1079
1080 return ret;
1081}
1082#endif
1083#define __INTRINSIC_DEFINED___shiftleft128
1084#endif /* __INTRINSIC_PROLOG */
1085
1086#if __INTRINSIC_PROLOG(__shiftright128)
1087unsigned __int64 __shiftright128 (unsigned __int64 LowPart, unsigned __int64 HighPart, unsigned char Shift);
1088#if !__has_builtin(__shiftright128)
1089__INTRINSICS_USEINLINE
1090unsigned __int64 __shiftright128 (unsigned __int64 LowPart, unsigned __int64 HighPart, unsigned char Shift)
1091{
1092 unsigned __int64 ret;
1093
1094 __asm__ ("shrd {%[Shift],%[HighPart],%[LowPart]|%[LowPart], %[HighPart], %[Shift]}"
1095 : [ret] "=r" (ret)
1096 : [LowPart] "0" (LowPart), [HighPart] "r" (HighPart), [Shift] "Jc" (Shift)
1097 : "cc");
1098
1099 return ret;
1100}
1101#endif
1102#define __INTRINSIC_DEFINED___shiftright128
1103#endif /* __INTRINSIC_PROLOG */
1104
1105#endif /* #(defined(__x86_64__) && !defined(__arm64ec__)) || (defined(_AMD64_) && !defined(_ARM64EC_)) */
1106
1107/* ***************************************************** */
1108
1109#if defined(__arm__) || defined(_ARM_)
1110
1111#if __INTRINSIC_PROLOG(_interlockedbittestandset)
1112unsigned char _interlockedbittestandset(__LONG32 volatile *a, __LONG32 b);
1113#if !__has_builtin(_interlockedbittestandset)
1114__INTRINSICS_USEINLINE
1115__buildbittesti(_interlockedbittestandset, __LONG32, "orr", /* unused param */)
1116#endif
1117#define __INTRINSIC_DEFINED__interlockedbittestandset
1118#endif /* __INTRINSIC_PROLOG */
1119
1120#if __INTRINSIC_PROLOG(_interlockedbittestandreset)
1121unsigned char _interlockedbittestandreset(__LONG32 volatile *a, __LONG32 b);
1122__INTRINSICS_USEINLINE
1123#if !__has_builtin(_interlockedbittestandreset)
1124__buildbittesti(_interlockedbittestandreset, __LONG32, "bic", /* unused param */)
1125#endif
1126#define __INTRINSIC_DEFINED__interlockedbittestandreset
1127#endif /* __INTRINSIC_PROLOG */
1128
1129#if __INTRINSIC_PROLOG(_interlockedbittestandcomplement)
1130unsigned char _interlockedbittestandcomplement(__LONG32 volatile *a, __LONG32 b);
1131#if !__has_builtin(_interlockedbittestandcomplement)
1132__INTRINSICS_USEINLINE
1133__buildbittesti(_interlockedbittestandcomplement, __LONG32, "eor", /* unused param */)
1134#endif
1135#define __INTRINSIC_DEFINED__interlockedbittestandcomplement
1136#endif /* __INTRINSIC_PROLOG */
1137
1138#if __INTRINSIC_PROLOG(InterlockedBitTestAndSet)
1139unsigned char InterlockedBitTestAndSet(volatile __LONG32 *a, __LONG32 b);
1140#if !__has_builtin(InterlockedBitTestAndSet)
1141__INTRINSICS_USEINLINE
1142__buildbittesti(InterlockedBitTestAndSet, __LONG32, "orr", /* unused param */)
1143#endif
1144#define __INTRINSIC_DEFINED_InterlockedBitTestAndSet
1145#endif /* __INTRINSIC_PROLOG */
1146
1147#if __INTRINSIC_PROLOG(InterlockedBitTestAndReset)
1148unsigned char InterlockedBitTestAndReset(volatile __LONG32 *a, __LONG32 b);
1149#if !__has_builtin(InterlockedBitTestAndReset)
1150__INTRINSICS_USEINLINE
1151__buildbittesti(InterlockedBitTestAndReset, __LONG32, "bic", /* unused param */)
1152#endif
1153#define __INTRINSIC_DEFINED_InterlockedBitTestAndReset
1154#endif /* __INTRINSIC_PROLOG */
1155
1156#if __INTRINSIC_PROLOG(InterlockedBitTestAndComplement)
1157unsigned char InterlockedBitTestAndComplement(volatile __LONG32 *a, __LONG32 b);
1158#if !__has_builtin(InterlockedBitTestAndComplement)
1159__INTRINSICS_USEINLINE
1160__buildbittesti(InterlockedBitTestAndComplement, __LONG32, "eor", /* unused param */)
1161#endif
1162#define __INTRINSIC_DEFINED_InterlockedBitTestAndComplement
1163#endif /* __INTRINSIC_PROLOG */
1164
1165#if __INTRINSIC_PROLOG(_BitScanForward)
1166__MINGW_EXTENSION unsigned char _BitScanForward(unsigned __LONG32 *Index, unsigned __LONG32 Mask);
1167#if !__has_builtin(_BitScanForward)
1168__MINGW_EXTENSION __INTRINSICS_USEINLINE
1169unsigned char _BitScanForward(unsigned __LONG32 *Index, unsigned __LONG32 Mask)
1170{
1171 if (Mask == 0)
1172 return 0;
1173 *Index = __builtin_ctz(Mask);
1174 return 1;
1175}
1176#endif
1177#define __INTRINSIC_DEFINED__BitScanForward
1178#endif /* __INTRINSIC_PROLOG */
1179
1180#if __INTRINSIC_PROLOG(_BitScanReverse)
1181__MINGW_EXTENSION unsigned char _BitScanReverse(unsigned __LONG32 *Index, unsigned __LONG32 Mask);
1182#if !__has_builtin(_BitScanReverse)
1183__MINGW_EXTENSION __INTRINSICS_USEINLINE
1184unsigned char _BitScanReverse(unsigned __LONG32 *Index, unsigned __LONG32 Mask)
1185{
1186 if (Mask == 0)
1187 return 0;
1188 *Index = 31 - __builtin_clz(Mask);
1189 return 1;
1190}
1191#endif
1192#define __INTRINSIC_DEFINED__BitScanReverse
1193#endif /* __INTRINSIC_PROLOG */
1194
1195#endif /* defined(__arm__) || defined(_ARM_) */
1196
1197#if defined(__aarch64__) || defined(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_)
1198
1199#if __INTRINSIC_PROLOG(_interlockedbittestandset)
1200unsigned char _interlockedbittestandset(__LONG32 volatile *a, __LONG32 b);
1201#if !__has_builtin(_interlockedbittestandset)
1202__INTRINSICS_USEINLINE
1203__buildbittesti(_interlockedbittestandset, __LONG32, "orr", /* unused param */)
1204#endif
1205#define __INTRINSIC_DEFINED__interlockedbittestandset
1206#endif /* __INTRINSIC_PROLOG */
1207
1208#if __INTRINSIC_PROLOG(_interlockedbittestandreset)
1209unsigned char _interlockedbittestandreset(__LONG32 volatile *a, __LONG32 b);
1210__INTRINSICS_USEINLINE
1211#if !__has_builtin(_interlockedbittestandreset)
1212__buildbittesti(_interlockedbittestandreset, __LONG32, "bic", /* unused param */)
1213#endif
1214#define __INTRINSIC_DEFINED__interlockedbittestandreset
1215#endif /* __INTRINSIC_PROLOG */
1216
1217#if __INTRINSIC_PROLOG(_interlockedbittestandcomplement)
1218unsigned char _interlockedbittestandcomplement(__LONG32 volatile *a, __LONG32 b);
1219#if !__has_builtin(_interlockedbittestandcomplement)
1220__INTRINSICS_USEINLINE
1221__buildbittesti(_interlockedbittestandcomplement, __LONG32, "eor", /* unused param */)
1222#endif
1223#define __INTRINSIC_DEFINED__interlockedbittestandcomplement
1224#endif /* __INTRINSIC_PROLOG */
1225
1226#if __INTRINSIC_PROLOG(InterlockedBitTestAndSet)
1227unsigned char InterlockedBitTestAndSet(volatile __LONG32 *a, __LONG32 b);
1228#if !__has_builtin(InterlockedBitTestAndSet)
1229__INTRINSICS_USEINLINE
1230__buildbittesti(InterlockedBitTestAndSet, __LONG32, "orr", /* unused param */)
1231#endif
1232#define __INTRINSIC_DEFINED_InterlockedBitTestAndSet
1233#endif /* __INTRINSIC_PROLOG */
1234
1235#if __INTRINSIC_PROLOG(InterlockedBitTestAndReset)
1236unsigned char InterlockedBitTestAndReset(volatile __LONG32 *a, __LONG32 b);
1237#if !__has_builtin(InterlockedBitTestAndReset)
1238__INTRINSICS_USEINLINE
1239__buildbittesti(InterlockedBitTestAndReset, __LONG32, "bic", /* unused param */)
1240#endif
1241#define __INTRINSIC_DEFINED_InterlockedBitTestAndReset
1242#endif /* __INTRINSIC_PROLOG */
1243
1244#if __INTRINSIC_PROLOG(InterlockedBitTestAndComplement)
1245unsigned char InterlockedBitTestAndComplement(volatile __LONG32 *a, __LONG32 b);
1246#if !__has_builtin(InterlockedBitTestAndComplement)
1247__INTRINSICS_USEINLINE
1248__buildbittesti(InterlockedBitTestAndComplement, __LONG32, "eor", /* unused param */)
1249#endif
1250#define __INTRINSIC_DEFINED_InterlockedBitTestAndComplement
1251#endif /* __INTRINSIC_PROLOG */
1252
1253#if __INTRINSIC_PROLOG(_interlockedbittestandset64)
1254unsigned char _interlockedbittestandset64(__int64 volatile *a, __int64 b);
1255#if !__has_builtin(_interlockedbittestandset64)
1256__INTRINSICS_USEINLINE
1257__buildbittesti64(_interlockedbittestandset64, __int64, "orr", /* unused param */)
1258#endif
1259#define __INTRINSIC_DEFINED__interlockedbittestandset64
1260#endif /* __INTRINSIC_PROLOG */
1261
1262#if __INTRINSIC_PROLOG(_interlockedbittestandreset64)
1263unsigned char _interlockedbittestandreset64(__int64 volatile *a, __int64 b);
1264__INTRINSICS_USEINLINE
1265#if !__has_builtin(_interlockedbittestandreset64)
1266__buildbittesti64(_interlockedbittestandreset64, __int64, "bic", /* unused param */)
1267#endif
1268#define __INTRINSIC_DEFINED__interlockedbittestandreset64
1269#endif /* __INTRINSIC_PROLOG */
1270
1271#if __INTRINSIC_PROLOG(_interlockedbittestandcomplement64)
1272unsigned char _interlockedbittestandcomplement64(__int64 volatile *a, __int64 b);
1273#if !__has_builtin(_interlockedbittestandcomplement64)
1274__INTRINSICS_USEINLINE
1275__buildbittesti64(_interlockedbittestandcomplement64, __int64, "eor", /* unused param */)
1276#endif
1277#define __INTRINSIC_DEFINED__interlockedbittestandcomplement64
1278#endif /* __INTRINSIC_PROLOG */
1279
1280#if __INTRINSIC_PROLOG(InterlockedBitTestAndSet64)
1281unsigned char InterlockedBitTestAndSet64(volatile __int64 *a, __int64 b);
1282#if !__has_builtin(InterlockedBitTestAndSet64)
1283__INTRINSICS_USEINLINE
1284__buildbittesti64(InterlockedBitTestAndSet64, __int64, "orr", /* unused param */)
1285#endif
1286#define __INTRINSIC_DEFINED_InterlockedBitTestAndSet64
1287#endif /* __INTRINSIC_PROLOG */
1288
1289#if __INTRINSIC_PROLOG(InterlockedBitTestAndReset64)
1290unsigned char InterlockedBitTestAndReset64(volatile __int64 *a, __int64 b);
1291#if !__has_builtin(InterlockedBitTestAndReset64)
1292__INTRINSICS_USEINLINE
1293__buildbittesti64(InterlockedBitTestAndReset64, __int64, "bic", /* unused param */)
1294#endif
1295#define __INTRINSIC_DEFINED_InterlockedBitTestAndReset64
1296#endif /* __INTRINSIC_PROLOG */
1297
1298#if __INTRINSIC_PROLOG(InterlockedBitTestAndComplement64)
1299unsigned char InterlockedBitTestAndComplement64(volatile __int64 *a, __int64 b);
1300#if !__has_builtin(InterlockedBitTestAndComplement64)
1301__INTRINSICS_USEINLINE
1302__buildbittesti64(InterlockedBitTestAndComplement64, __int64, "eor", /* unused param */)
1303#endif
1304#define __INTRINSIC_DEFINED_InterlockedBitTestAndComplement64
1305#endif /* __INTRINSIC_PROLOG */
1306
1307#if __INTRINSIC_PROLOG(_InterlockedAnd64)
1308__MINGW_EXTENSION __int64 _InterlockedAnd64(__int64 volatile *, __int64);
1309#if !__has_builtin(_InterlockedAnd64)
1310__INTRINSICS_USEINLINE
1311__buildlogicali(_InterlockedAnd64, __int64, and)
1312#endif
1313#define __INTRINSIC_DEFINED__InterlockedAnd64
1314#endif /* __INTRINSIC_PROLOG */
1315
1316#if __INTRINSIC_PROLOG(_InterlockedOr64)
1317__MINGW_EXTENSION __int64 _InterlockedOr64(__int64 volatile *, __int64);
1318#if !__has_builtin(_InterlockedOr64)
1319__INTRINSICS_USEINLINE
1320__buildlogicali(_InterlockedOr64, __int64, or)
1321#endif
1322#define __INTRINSIC_DEFINED__InterlockedOr64
1323#endif /* __INTRINSIC_PROLOG */
1324
1325#if __INTRINSIC_PROLOG(_InterlockedXor64)
1326__MINGW_EXTENSION __int64 _InterlockedXor64(__int64 volatile *, __int64);
1327#if !__has_builtin(_InterlockedXor64)
1328__INTRINSICS_USEINLINE
1329__buildlogicali(_InterlockedXor64, __int64, xor)
1330#endif
1331#define __INTRINSIC_DEFINED__InterlockedXor64
1332#endif /* __INTRINSIC_PROLOG */
1333
1334#if __INTRINSIC_PROLOG(_InterlockedIncrement64)
1335__MINGW_EXTENSION __int64 _InterlockedIncrement64(__int64 volatile *Addend);
1336#if !__has_builtin(_InterlockedIncrement64)
1337__MINGW_EXTENSION __INTRINSICS_USEINLINE
1338__int64 _InterlockedIncrement64(__int64 volatile *Addend) {
1339 return __sync_add_and_fetch(Addend, 1);
1340}
1341#endif
1342#define __INTRINSIC_DEFINED__InterlockedIncrement64
1343#endif /* __INTRINSIC_PROLOG */
1344
1345#if __INTRINSIC_PROLOG(_InterlockedDecrement64)
1346__MINGW_EXTENSION __int64 _InterlockedDecrement64(__int64 volatile *Addend);
1347#if !__has_builtin(_InterlockedDecrement64)
1348__MINGW_EXTENSION __INTRINSICS_USEINLINE
1349__int64 _InterlockedDecrement64(__int64 volatile *Addend) {
1350 return __sync_sub_and_fetch(Addend, 1);
1351}
1352#endif
1353#define __INTRINSIC_DEFINED__InterlockedDecrement64
1354#endif /* __INTRINSIC_PROLOG */
1355
1356#if __INTRINSIC_PROLOG(_InterlockedExchange64)
1357__MINGW_EXTENSION __int64 _InterlockedExchange64(__int64 volatile *Target, __int64 Value);
1358#if !__has_builtin(_InterlockedExchange64)
1359__MINGW_EXTENSION __INTRINSICS_USEINLINE
1360__int64 _InterlockedExchange64(__int64 volatile *Target, __int64 Value) {
1361 return __sync_lock_test_and_set(Target, Value);
1362}
1363#endif
1364#define __INTRINSIC_DEFINED__InterlockedExchange64
1365#endif /* __INTRINSIC_PROLOG */
1366
1367#if __INTRINSIC_PROLOG(_InterlockedExchangeAdd64)
1368__MINGW_EXTENSION __int64 _InterlockedExchangeAdd64(__int64 volatile *Addend, __int64 Value);
1369#if !__has_builtin(_InterlockedExchangeAdd64)
1370__MINGW_EXTENSION __INTRINSICS_USEINLINE
1371__int64 _InterlockedExchangeAdd64(__int64 volatile *Addend, __int64 Value) {
1372 return __sync_fetch_and_add(Addend, Value);
1373}
1374#endif
1375#define __INTRINSIC_DEFINED__InterlockedExchangeAdd64
1376#endif /* __INTRINSIC_PROLOG */
1377
1378#if __INTRINSIC_PROLOG(_BitScanForward)
1379__MINGW_EXTENSION unsigned char _BitScanForward(unsigned __LONG32 *Index, unsigned __LONG32 Mask);
1380#if !__has_builtin(_BitScanForward)
1381__MINGW_EXTENSION __INTRINSICS_USEINLINE
1382unsigned char _BitScanForward(unsigned __LONG32 *Index, unsigned __LONG32 Mask)
1383{
1384 if (Mask == 0)
1385 return 0;
1386 *Index = __builtin_ctz(Mask);
1387 return 1;
1388}
1389#endif
1390#define __INTRINSIC_DEFINED__BitScanForward
1391#endif /* __INTRINSIC_PROLOG */
1392
1393#if __INTRINSIC_PROLOG(_BitScanReverse)
1394__MINGW_EXTENSION unsigned char _BitScanReverse(unsigned __LONG32 *Index, unsigned __LONG32 Mask);
1395#if !__has_builtin(_BitScanReverse)
1396__MINGW_EXTENSION __INTRINSICS_USEINLINE
1397unsigned char _BitScanReverse(unsigned __LONG32 *Index, unsigned __LONG32 Mask)
1398{
1399 if (Mask == 0)
1400 return 0;
1401 *Index = 31 - __builtin_clz(Mask);
1402 return 1;
1403}
1404#endif
1405#define __INTRINSIC_DEFINED__BitScanReverse
1406#endif /* __INTRINSIC_PROLOG */
1407
1408#if __INTRINSIC_PROLOG(_BitScanForward64)
1409__MINGW_EXTENSION unsigned char _BitScanForward64(unsigned __LONG32 *Index, unsigned __int64 Mask);
1410#if !__has_builtin(_BitScanForward64)
1411__MINGW_EXTENSION __INTRINSICS_USEINLINE
1412unsigned char _BitScanForward64(unsigned __LONG32 *Index, unsigned __int64 Mask)
1413{
1414 if (Mask == 0)
1415 return 0;
1416 *Index = __builtin_ctzll(Mask);
1417 return 1;
1418}
1419#endif
1420#define __INTRINSIC_DEFINED__BitScanForward64
1421#endif /* __INTRINSIC_PROLOG */
1422
1423#if __INTRINSIC_PROLOG(_BitScanReverse64)
1424__MINGW_EXTENSION unsigned char _BitScanReverse64(unsigned __LONG32 *Index, unsigned __int64 Mask);
1425#if !__has_builtin(_BitScanReverse64)
1426__MINGW_EXTENSION __INTRINSICS_USEINLINE
1427unsigned char _BitScanReverse64(unsigned __LONG32 *Index, unsigned __int64 Mask)
1428{
1429 if (Mask == 0)
1430 return 0;
1431 *Index = 63 - __builtin_clzll(Mask);
1432 return 1;
1433}
1434#endif
1435#define __INTRINSIC_DEFINED__BitScanReverse64
1436#endif /* __INTRINSIC_PROLOG */
1437
1438#endif /* defined(__aarch64__) || define(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_) */
1439
1440#if defined(__arm__) || defined(_ARM_) || defined(__aarch64__) || defined(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_)
1441
1442#if __INTRINSIC_PROLOG(_bittest)
1443unsigned char _bittest(const __LONG32 *__a, __LONG32 __b);
1444#if !__has_builtin(_bittest)
1445__INTRINSICS_USEINLINE
1446unsigned char _bittest(const __LONG32 *__a, __LONG32 __b)
1447{
1448 return (*__a >> __b) & 1;
1449}
1450#endif
1451#define __INTRINSIC_DEFINED__bittest
1452#endif /* __INTRINSIC_PROLOG */
1453
1454#if __INTRINSIC_PROLOG(_bittestandset)
1455unsigned char _bittestandset(__LONG32 *__a, __LONG32 __b);
1456#if !__has_builtin(_bittestandset)
1457__INTRINSICS_USEINLINE
1458unsigned char _bittestandset(__LONG32 *__a, __LONG32 __b)
1459{
1460 unsigned char __v = (*__a >> __b) & 1;
1461 *__a |= 1UL << __b;
1462 return __v;
1463}
1464#endif
1465#define __INTRINSIC_DEFINED__bittestandset
1466#endif /* __INTRINSIC_PROLOG */
1467
1468#if __INTRINSIC_PROLOG(_bittestandreset)
1469unsigned char _bittestandreset(__LONG32 *__a, __LONG32 __b);
1470#if !__has_builtin(_bittestandreset)
1471__INTRINSICS_USEINLINE
1472unsigned char _bittestandreset(__LONG32 *__a, __LONG32 __b)
1473{
1474 unsigned char __v = (*__a >> __b) & 1;
1475 *__a &= ~(1UL << __b);
1476 return __v;
1477}
1478#endif
1479#define __INTRINSIC_DEFINED__bittestandreset
1480#endif /* __INTRINSIC_PROLOG */
1481
1482#if __INTRINSIC_PROLOG(_bittestandcomplement)
1483unsigned char _bittestandcomplement(__LONG32 *a, __LONG32 b);
1484#if !__has_builtin(_bittestandcomplement)
1485__INTRINSICS_USEINLINE
1486unsigned char _bittestandcomplement(__LONG32 *__a, __LONG32 __b)
1487{
1488 unsigned char __v = (*__a >> __b) & 1;
1489 *__a ^= 1UL << __b;
1490 return __v;
1491}
1492#endif
1493#define __INTRINSIC_DEFINED__bittestandcomplement
1494#endif /* __INTRINSIC_PROLOG */
1495
1496#endif /* defined(__arm__) || defined(_ARM_) || defined(__aarch64__) || defined(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_) */
1497
1498#if defined(__aarch64__) || defined(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_)
1499
1500#if __INTRINSIC_PROLOG(_bittest64)
1501unsigned char _bittest64(const __int64 *__a, __int64 __b);
1502#if !__has_builtin(_bittest64)
1503__INTRINSICS_USEINLINE
1504unsigned char _bittest64(const __int64 *__a, __int64 __b)
1505{
1506 return (*__a >> __b) & 1;
1507}
1508#endif
1509#define __INTRINSIC_DEFINED__bittest64
1510#endif /* __INTRINSIC_PROLOG */
1511
1512#if __INTRINSIC_PROLOG(_bittestandset64)
1513unsigned char _bittestandset64(__int64 *__a, __int64 __b);
1514#if !__has_builtin(_bittestandset64)
1515__INTRINSICS_USEINLINE
1516unsigned char _bittestandset64(__int64 *__a, __int64 __b)
1517{
1518 unsigned char __v = (*__a >> __b) & 1;
1519 *__a |= 1ULL << __b;
1520 return __v;
1521}
1522#endif
1523#define __INTRINSIC_DEFINED__bittestandset64
1524#endif /* __INTRINSIC_PROLOG */
1525
1526#if __INTRINSIC_PROLOG(_bittestandreset64)
1527unsigned char _bittestandreset64(__int64 *__a, __int64 __b);
1528#if !__has_builtin(_bittestandreset64)
1529__INTRINSICS_USEINLINE
1530unsigned char _bittestandreset64(__int64 *__a, __int64 __b)
1531{
1532 unsigned char __v = (*__a >> __b) & 1;
1533 *__a &= ~(1ULL << __b);
1534 return __v;
1535}
1536#endif
1537#define __INTRINSIC_DEFINED__bittestandreset64
1538#endif /* __INTRINSIC_PROLOG */
1539
1540#if __INTRINSIC_PROLOG(_bittestandcomplement64)
1541unsigned char _bittestandcomplement64(__int64 *a, __int64 b);
1542#if !__has_builtin(_bittestandcomplement64)
1543__INTRINSICS_USEINLINE
1544unsigned char _bittestandcomplement64(__int64 *__a, __int64 __b)
1545{
1546 unsigned char __v = (*__a >> __b) & 1;
1547 *__a ^= 1ULL << __b;
1548 return __v;
1549}
1550#endif
1551#define __INTRINSIC_DEFINED__bittestandcomplement64
1552#endif /* __INTRINSIC_PROLOG */
1553
1554#endif /* defined(__aarch64__) || define(_ARM64_) || defined(__arm64ec__) || defined(_ARM64EC_) */
1555
1556/* ***************************************************** */
1557
1558#if defined(__x86_64__) || defined(_AMD64_) || defined(__i386__) || defined(_X86_) || defined(__arm__) || defined(_ARM_) || defined(__aarch64__) || defined(_ARM64_)
1559
1560#if __INTRINSIC_PROLOG(__popcnt16)
1561unsigned short __popcnt16(unsigned short);
1562#if !__has_builtin(__popcnt16)
1563__INTRINSICS_USEINLINE
1564unsigned short __popcnt16(unsigned short value)
1565{
1566 return __builtin_popcount(value);
1567}
1568#endif
1569#define __INTRINSIC_DEFINED___popcnt16
1570#endif /* __INTRINSIC_PROLOG */
1571
1572#if __INTRINSIC_PROLOG(__popcnt)
1573unsigned int __popcnt(unsigned int);
1574#if !__has_builtin(__popcnt)
1575__INTRINSICS_USEINLINE
1576unsigned int __popcnt(unsigned int value)
1577{
1578 return __builtin_popcount(value);
1579}
1580#endif
1581#define __INTRINSIC_DEFINED___popcnt
1582#endif /* __INTRINSIC_PROLOG */
1583
1584#if __INTRINSIC_PROLOG(__popcnt64)
1585unsigned __int64 __popcnt64(unsigned __int64);
1586#if !__has_builtin(__popcnt64)
1587__INTRINSICS_USEINLINE
1588unsigned __int64 __popcnt64(unsigned __int64 value)
1589{
1590 return __builtin_popcountll(value);
1591}
1592#endif
1593#define __INTRINSIC_DEFINED___popcnt64
1594#endif /* __INTRINSIC_PROLOG */
1595
1596#if __INTRINSIC_PROLOG(_InterlockedAnd)
1597__LONG32 _InterlockedAnd(__LONG32 volatile *, __LONG32);
1598#if !__has_builtin(_InterlockedAnd)
1599__INTRINSICS_USEINLINE
1600__buildlogicali(_InterlockedAnd, __LONG32, and)
1601#endif
1602#define __INTRINSIC_DEFINED__InterlockedAnd
1603#endif /* __INTRINSIC_PROLOG */
1604
1605#if __INTRINSIC_PROLOG(_InterlockedOr)
1606__LONG32 _InterlockedOr(__LONG32 volatile *, __LONG32);
1607#if !__has_builtin(_InterlockedOr)
1608__INTRINSICS_USEINLINE
1609__buildlogicali(_InterlockedOr, __LONG32, or)
1610#endif
1611#define __INTRINSIC_DEFINED__InterlockedOr
1612#endif /* __INTRINSIC_PROLOG */
1613
1614#if __INTRINSIC_PROLOG(_InterlockedXor)
1615__LONG32 _InterlockedXor(__LONG32 volatile *, __LONG32);
1616#if !__has_builtin(_InterlockedXor)
1617__INTRINSICS_USEINLINE
1618__buildlogicali(_InterlockedXor, __LONG32, xor)
1619#endif
1620#define __INTRINSIC_DEFINED__InterlockedXor
1621#endif /* __INTRINSIC_PROLOG */
1622
1623#if __INTRINSIC_PROLOG(_InterlockedCompareExchange8)
1624char _InterlockedCompareExchange8(char volatile *destination, char exchange, char comperand);
1625#if !__has_builtin(_InterlockedCompareExchange8)
1626__INTRINSICS_USEINLINE
1627char _InterlockedCompareExchange8(char volatile *destination, char exchange, char comperand) {
1628 return __sync_val_compare_and_swap(destination, comperand, exchange);
1629}
1630#endif
1631#define __INTRINSIC_DEFINED__InterlockedCompareExchange8
1632#endif /* __INTRINSIC_PROLOG */
1633
1634#if __INTRINSIC_PROLOG(_InterlockedIncrement16)
1635short _InterlockedIncrement16(short volatile *Addend);
1636#if !__has_builtin(_InterlockedIncrement16)
1637__INTRINSICS_USEINLINE
1638short _InterlockedIncrement16(short volatile *Addend) {
1639 return __sync_add_and_fetch(Addend, 1);
1640}
1641#endif
1642#define __INTRINSIC_DEFINED__InterlockedIncrement16
1643#endif /* __INTRINSIC_PROLOG */
1644
1645#if __INTRINSIC_PROLOG(_InterlockedDecrement16)
1646short _InterlockedDecrement16(short volatile *Addend);
1647#if !__has_builtin(_InterlockedDecrement16)
1648__INTRINSICS_USEINLINE
1649short _InterlockedDecrement16(short volatile *Addend) {
1650 return __sync_sub_and_fetch(Addend, 1);
1651}
1652#endif
1653#define __INTRINSIC_DEFINED__InterlockedDecrement16
1654#endif /* __INTRINSIC_PROLOG */
1655
1656#if __INTRINSIC_PROLOG(_InterlockedCompareExchange16)
1657short _InterlockedCompareExchange16(short volatile *Destination, short ExChange, short Comperand);
1658#if !__has_builtin(_InterlockedCompareExchange16)
1659__INTRINSICS_USEINLINE
1660short _InterlockedCompareExchange16(short volatile *Destination, short ExChange, short Comperand) {
1661 return __sync_val_compare_and_swap(Destination, Comperand, ExChange);
1662}
1663#endif
1664#define __INTRINSIC_DEFINED__InterlockedCompareExchange16
1665#endif /* __INTRINSIC_PROLOG */
1666
1667/* On i386, fall back to the kernel32 Interlocked exports because GCC
1668 cannot inline the __sync_* builtins (no CMPXCHG/XADD). __asm__
1669 labels avoid collisions with the winnt.h Interlocked macros.
1670 GCC prepends _ to __asm__ label strings on i386; Clang does not. */
1671#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1672#ifdef __clang__
1673#define __MINGW_K32_ASM(name) __asm__("_" name)
1674#else
1675#define __MINGW_K32_ASM(name) __asm__(name)
1676#endif
1677__LONG32 __attribute__((__dllimport__, __stdcall__))
1678__mingw_k32_InterlockedExchangeAdd(__LONG32 volatile *, __LONG32)
1679 __MINGW_K32_ASM("InterlockedExchangeAdd@8");
1680__LONG32 __attribute__((__dllimport__, __stdcall__))
1681__mingw_k32_InterlockedCompareExchange(__LONG32 volatile *, __LONG32, __LONG32)
1682 __MINGW_K32_ASM("InterlockedCompareExchange@12");
1683__LONG32 __attribute__((__dllimport__, __stdcall__))
1684__mingw_k32_InterlockedIncrement(__LONG32 volatile *)
1685 __MINGW_K32_ASM("InterlockedIncrement@4");
1686__LONG32 __attribute__((__dllimport__, __stdcall__))
1687__mingw_k32_InterlockedDecrement(__LONG32 volatile *)
1688 __MINGW_K32_ASM("InterlockedDecrement@4");
1689__LONG32 __attribute__((__dllimport__, __stdcall__))
1690__mingw_k32_InterlockedExchange(__LONG32 volatile *, __LONG32)
1691 __MINGW_K32_ASM("InterlockedExchange@8");
1692#undef __MINGW_K32_ASM
1693#endif
1694
1695#if __INTRINSIC_PROLOG(_InterlockedExchangeAdd)
1696__LONG32 _InterlockedExchangeAdd(__LONG32 volatile *Addend, __LONG32 Value);
1697#if !__has_builtin(_InterlockedExchangeAdd)
1698__INTRINSICS_USEINLINE
1699__LONG32 _InterlockedExchangeAdd(__LONG32 volatile *Addend, __LONG32 Value) {
1700#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1701 return __mingw_k32_InterlockedExchangeAdd(Addend, Value);
1702#else
1703 return __sync_fetch_and_add(Addend, Value);
1704#endif
1705}
1706#endif
1707#define __INTRINSIC_DEFINED__InterlockedExchangeAdd
1708#endif /* __INTRINSIC_PROLOG */
1709
1710#if __INTRINSIC_PROLOG(_InterlockedCompareExchange)
1711__LONG32 _InterlockedCompareExchange(__LONG32 volatile *Destination, __LONG32 ExChange, __LONG32 Comperand);
1712#if !__has_builtin(_InterlockedCompareExchange)
1713__INTRINSICS_USEINLINE
1714__LONG32 _InterlockedCompareExchange(__LONG32 volatile *Destination, __LONG32 ExChange, __LONG32 Comperand) {
1715#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1716 return __mingw_k32_InterlockedCompareExchange(Destination, ExChange, Comperand);
1717#else
1718 return __sync_val_compare_and_swap(Destination, Comperand, ExChange);
1719#endif
1720}
1721#endif
1722#define __INTRINSIC_DEFINED__InterlockedCompareExchange
1723#endif /* __INTRINSIC_PROLOG */
1724
1725#if __INTRINSIC_PROLOG(_InterlockedIncrement)
1726__LONG32 _InterlockedIncrement(__LONG32 volatile *Addend);
1727#if !__has_builtin(_InterlockedIncrement)
1728__INTRINSICS_USEINLINE
1729__LONG32 _InterlockedIncrement(__LONG32 volatile *Addend) {
1730#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1731 return __mingw_k32_InterlockedIncrement(Addend);
1732#else
1733 return __sync_add_and_fetch(Addend, 1);
1734#endif
1735}
1736#endif
1737#define __INTRINSIC_DEFINED__InterlockedIncrement
1738#endif /* __INTRINSIC_PROLOG */
1739
1740#if __INTRINSIC_PROLOG(_InterlockedDecrement)
1741__LONG32 _InterlockedDecrement(__LONG32 volatile *Addend);
1742#if !__has_builtin(_InterlockedDecrement)
1743__INTRINSICS_USEINLINE
1744__LONG32 _InterlockedDecrement(__LONG32 volatile *Addend) {
1745#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1746 return __mingw_k32_InterlockedDecrement(Addend);
1747#else
1748 return __sync_sub_and_fetch(Addend, 1);
1749#endif
1750}
1751#endif
1752#define __INTRINSIC_DEFINED__InterlockedDecrement
1753#endif /* __INTRINSIC_PROLOG */
1754
1755#if __INTRINSIC_PROLOG(_InterlockedAdd)
1756__LONG32 _InterlockedAdd(__LONG32 volatile *Addend, __LONG32 Value);
1757#if !__has_builtin(_InterlockedAdd)
1758__INTRINSICS_USEINLINE
1759__LONG32 _InterlockedAdd(__LONG32 volatile *Addend, __LONG32 Value) {
1760 return __sync_add_and_fetch(Addend, Value);
1761}
1762#endif
1763#define __INTRINSIC_DEFINED__InterlockedAdd
1764#endif /* __INTRINSIC_PROLOG */
1765
1766#if __INTRINSIC_PROLOG(_InterlockedAdd64)
1767__MINGW_EXTENSION __int64 _InterlockedAdd64(__int64 volatile *Addend, __int64 Value);
1768#if !__has_builtin(_InterlockedAdd64)
1769__MINGW_EXTENSION __INTRINSICS_USEINLINE
1770__int64 _InterlockedAdd64(__int64 volatile *Addend, __int64 Value) {
1771 return __sync_add_and_fetch(Addend, Value);
1772}
1773#endif
1774#define __INTRINSIC_DEFINED__InterlockedAdd64
1775#endif /* __INTRINSIC_PROLOG */
1776
1777#if __INTRINSIC_PROLOG(_InterlockedExchange)
1778__LONG32 _InterlockedExchange(__LONG32 volatile *Target, __LONG32 Value);
1779#if !__has_builtin(_InterlockedExchange)
1780__INTRINSICS_USEINLINE
1781__LONG32 _InterlockedExchange(__LONG32 volatile *Target, __LONG32 Value) {
1782#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1783 return __mingw_k32_InterlockedExchange(Target, Value);
1784#else
1785 return __sync_lock_test_and_set(Target, Value);
1786#endif
1787}
1788#endif
1789#define __INTRINSIC_DEFINED__InterlockedExchange
1790#endif /* __INTRINSIC_PROLOG */
1791
1792#if __INTRINSIC_PROLOG(_InterlockedCompareExchange64)
1793__MINGW_EXTENSION __int64 _InterlockedCompareExchange64(__int64 volatile *Destination, __int64 ExChange, __int64 Comperand);
1794#if !__has_builtin(_InterlockedCompareExchange64)
1795__MINGW_EXTENSION __INTRINSICS_USEINLINE
1796__int64 _InterlockedCompareExchange64(__int64 volatile *Destination, __int64 ExChange, __int64 Comperand) {
1797 return __sync_val_compare_and_swap(Destination, Comperand, ExChange);
1798}
1799#endif
1800#define __INTRINSIC_DEFINED__InterlockedCompareExchange64
1801#endif /* __INTRINSIC_PROLOG */
1802
1803#if __INTRINSIC_PROLOG(_InterlockedCompareExchangePointer)
1804void *_InterlockedCompareExchangePointer(void * volatile *Destination, void *ExChange, void *Comperand);
1805#if !__has_builtin(_InterlockedCompareExchangePointer)
1806__INTRINSICS_USEINLINE
1807void *_InterlockedCompareExchangePointer(void *volatile *Destination, void *ExChange, void *Comperand) {
1808#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1809 return (void *)(long)__mingw_k32_InterlockedCompareExchange(
1810 (__LONG32 volatile *)Destination,
1811 (__LONG32)(long)ExChange,
1812 (__LONG32)(long)Comperand);
1813#else
1814 return __sync_val_compare_and_swap(Destination, Comperand, ExChange);
1815#endif
1816}
1817#endif
1818#define __INTRINSIC_DEFINED__InterlockedCompareExchangePointer
1819#endif /* __INTRINSIC_PROLOG */
1820
1821#if __INTRINSIC_PROLOG(_InterlockedExchangePointer)
1822void *_InterlockedExchangePointer(void *volatile *Target,void *Value);
1823#if !__has_builtin(_InterlockedExchangePointer)
1824__INTRINSICS_USEINLINE
1825void *_InterlockedExchangePointer(void *volatile *Target,void *Value) {
1826#if defined(__i386__) && !defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)
1827 return (void *)(long)__mingw_k32_InterlockedExchange(
1828 (__LONG32 volatile *)Target,
1829 (__LONG32)(long)Value);
1830#else
1831 return __sync_lock_test_and_set(Target, Value);
1832#endif
1833}
1834#endif
1835#define __INTRINSIC_DEFINED__InterlockedExchangePointer
1836#endif /* __INTRINSIC_PROLOG */
1837
1838#endif /* defined(__x86_64__) || defined(_AMD64_) || defined(__i386__) || defined(_X86_) || defined(__arm__) || defined(_ARM_) || defined(__aarch64__) || defined(_ARM64_) */
1839
1840#if (defined(__x86_64__) && !defined(__arm64ec__)) || (defined(_AMD64_) && !defined(_ARM64EC_)) || defined(__i386__) || defined(_X86_)
1841
1842#if __INTRINSIC_PROLOG(__int2c)
1843void __int2c(void);
1844#if !__has_builtin(__int2c)
1845__INTRINSICS_USEINLINE
1846void __int2c(void) {
1847 __buildint(0x2c);
1848}
1849#endif
1850#define __INTRINSIC_DEFINED___int2c
1851#endif /* __INTRINSIC_PROLOG */
1852
1853#if __INTRINSIC_PROLOG(__stosb)
1854void __stosb(unsigned char *, unsigned char, size_t);
1855#if !__has_builtin(__stosb)
1856__INTRINSICS_USEINLINE
1857__buildstos(__stosb, unsigned char, "b|b")
1858#endif
1859#define __INTRINSIC_DEFINED___stosb
1860#endif /* __INTRINSIC_PROLOG */
1861
1862#if __INTRINSIC_PROLOG(__stosw)
1863void __stosw(unsigned short *, unsigned short, size_t);
1864#if !__has_builtin(__stosw)
1865__INTRINSICS_USEINLINE
1866__buildstos(__stosw, unsigned short, "w|w")
1867#endif
1868#define __INTRINSIC_DEFINED___stosw
1869#endif /* __INTRINSIC_PROLOG */
1870
1871#if __INTRINSIC_PROLOG(__stosd)
1872void __stosd(unsigned __LONG32 *, unsigned __LONG32, size_t);
1873#if !__has_builtin(__stosd)
1874__INTRINSICS_USEINLINE
1875__buildstos(__stosd, unsigned __LONG32, "l|d")
1876#endif
1877#define __INTRINSIC_DEFINED___stosd
1878#endif /* __INTRINSIC_PROLOG */
1879
1880#if __INTRINSIC_PROLOG(_interlockedbittestandset)
1881unsigned char _interlockedbittestandset(__LONG32 volatile *a, __LONG32 b);
1882#if !__has_builtin(_interlockedbittestandset)
1883__INTRINSICS_USEINLINE
1884__buildbittesti(_interlockedbittestandset, __LONG32, "lock bts{l %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "I")
1885#endif
1886#define __INTRINSIC_DEFINED__interlockedbittestandset
1887#endif /* __INTRINSIC_PROLOG */
1888
1889#if __INTRINSIC_PROLOG(_interlockedbittestandreset)
1890unsigned char _interlockedbittestandreset(__LONG32 volatile *a, __LONG32 b);
1891#if !__has_builtin(_interlockedbittestandreset)
1892__INTRINSICS_USEINLINE
1893__buildbittesti(_interlockedbittestandreset, __LONG32, "lock btr{l %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "I")
1894#endif
1895#define __INTRINSIC_DEFINED__interlockedbittestandreset
1896#endif /* __INTRINSIC_PROLOG */
1897
1898#if __INTRINSIC_PROLOG(_interlockedbittestandcomplement)
1899unsigned char _interlockedbittestandcomplement(__LONG32 volatile *a, __LONG32 b);
1900#if !__has_builtin(_interlockedbittestandcomplement)
1901__INTRINSICS_USEINLINE
1902__buildbittesti(_interlockedbittestandcomplement, __LONG32, "lock btc{l %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "I")
1903#endif
1904#define __INTRINSIC_DEFINED__interlockedbittestandcomplement
1905#endif /* __INTRINSIC_PROLOG */
1906
1907#if __INTRINSIC_PROLOG(InterlockedBitTestAndSet)
1908unsigned char InterlockedBitTestAndSet(volatile __LONG32 *a, __LONG32 b);
1909#if !__has_builtin(InterlockedBitTestAndSet)
1910__INTRINSICS_USEINLINE
1911__buildbittesti(InterlockedBitTestAndSet, __LONG32, "lock bts{l %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "I")
1912#endif
1913#define __INTRINSIC_DEFINED_InterlockedBitTestAndSet
1914#endif /* __INTRINSIC_PROLOG */
1915
1916#if __INTRINSIC_PROLOG(InterlockedBitTestAndReset)
1917unsigned char InterlockedBitTestAndReset(volatile __LONG32 *a, __LONG32 b);
1918#if !__has_builtin(InterlockedBitTestAndReset)
1919__INTRINSICS_USEINLINE
1920__buildbittesti(InterlockedBitTestAndReset, __LONG32, "lock btr{l %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "I")
1921#endif
1922#define __INTRINSIC_DEFINED_InterlockedBitTestAndReset
1923#endif /* __INTRINSIC_PROLOG */
1924
1925#if __INTRINSIC_PROLOG(InterlockedBitTestAndComplement)
1926unsigned char InterlockedBitTestAndComplement(volatile __LONG32 *a, __LONG32 b);
1927#if !__has_builtin(InterlockedBitTestAndComplement)
1928__INTRINSICS_USEINLINE
1929__buildbittesti(InterlockedBitTestAndComplement, __LONG32, "lock btc{l %[Offset],%[Base] | %[Base],%[Offset]}" __FLAGSET, "I")
1930#endif
1931#define __INTRINSIC_DEFINED_InterlockedBitTestAndComplement
1932#endif /* __INTRINSIC_PROLOG */
1933
1934#if __INTRINSIC_PROLOG(_BitScanForward)
1935unsigned char _BitScanForward(unsigned __LONG32 *Index, unsigned __LONG32 Mask);
1936#if !__has_builtin(_BitScanForward)
1937__INTRINSICS_USEINLINE
1938__buildbitscan(_BitScanForward, unsigned __LONG32, "bsf{l %[Mask],%[Index] | %[Index],%[Mask]}")
1939#endif
1940#define __INTRINSIC_DEFINED__BitScanForward
1941#endif /* __INTRINSIC_PROLOG */
1942
1943#if __INTRINSIC_PROLOG(_BitScanReverse)
1944unsigned char _BitScanReverse(unsigned __LONG32 *Index, unsigned __LONG32 Mask);
1945#if !__has_builtin(_BitScanReverse)
1946__INTRINSICS_USEINLINE
1947__buildbitscan(_BitScanReverse, unsigned __LONG32, "bsr{l %[Mask],%[Index] | %[Index],%[Mask]}")
1948#endif
1949#define __INTRINSIC_DEFINED__BitScanReverse
1950#endif /* __INTRINSIC_PROLOG */
1951
1952#if __INTRINSIC_PROLOG(_bittest)
1953unsigned char _bittest(__LONG32 const *a, __LONG32 b);
1954#if !__has_builtin(_bittest)
1955__INTRINSICS_USEINLINE
1956__buildbittest(_bittest, __LONG32, "l", "I")
1957#endif
1958#define __INTRINSIC_DEFINED__bittest
1959#endif /* __INTRINSIC_PROLOG */
1960
1961#if __INTRINSIC_PROLOG(_bittestandset)
1962unsigned char _bittestandset(__LONG32 *a, __LONG32 b);
1963#if !__has_builtin(_bittestandset)
1964__INTRINSICS_USEINLINE
1965__buildbittestand(_bittestandset, __LONG32, "bts", "I", "l")
1966#endif
1967#define __INTRINSIC_DEFINED__bittestandset
1968#endif /* __INTRINSIC_PROLOG */
1969
1970#if __INTRINSIC_PROLOG(_bittestandreset)
1971unsigned char _bittestandreset(__LONG32 *a, __LONG32 b);
1972#if !__has_builtin(_bittestandreset)
1973__INTRINSICS_USEINLINE
1974__buildbittestand(_bittestandreset, __LONG32, "btr", "I", "l")
1975#endif
1976#define __INTRINSIC_DEFINED__bittestandreset
1977#endif /* __INTRINSIC_PROLOG */
1978
1979#if __INTRINSIC_PROLOG(_bittestandcomplement)
1980unsigned char _bittestandcomplement(__LONG32 *a, __LONG32 b);
1981#if !__has_builtin(_bittestandcomplement)
1982__INTRINSICS_USEINLINE
1983__buildbittestand(_bittestandcomplement, __LONG32, "btc", "I", "l")
1984#endif
1985#define __INTRINSIC_DEFINED__bittestandcomplement
1986#endif /* __INTRINSIC_PROLOG */
1987
1988#if __INTRINSIC_PROLOG(__inbyte)
1989unsigned char __inbyte(unsigned short Port);
1990#if !__has_builtin(__inbyte)
1991__INTRINSICS_USEINLINE
1992__build_inport(__inbyte, unsigned char, "b")
1993#endif
1994#define __INTRINSIC_DEFINED___inbyte
1995#endif /* __INTRINSIC_PROLOG */
1996
1997#if __INTRINSIC_PROLOG(__inword)
1998unsigned short __inword(unsigned short Port);
1999#if !__has_builtin(__inword)
2000__INTRINSICS_USEINLINE
2001__build_inport(__inword, unsigned short, "w")
2002#endif
2003#define __INTRINSIC_DEFINED___inword
2004#endif /* __INTRINSIC_PROLOG */
2005
2006#if __INTRINSIC_PROLOG(__indword)
2007unsigned __LONG32 __indword(unsigned short Port);
2008#if !__has_builtin(__indword)
2009__INTRINSICS_USEINLINE
2010__build_inport(__indword, unsigned __LONG32, "l")
2011#endif
2012#define __INTRINSIC_DEFINED___indword
2013#endif /* __INTRINSIC_PROLOG */
2014
2015#if __INTRINSIC_PROLOG(__outbyte)
2016void __outbyte(unsigned short Port, unsigned char Data);
2017#if !__has_builtin(__outbyte)
2018__INTRINSICS_USEINLINE
2019__build_outport(__outbyte, unsigned char, "b")
2020#endif
2021#define __INTRINSIC_DEFINED___outbyte
2022#endif /* __INTRINSIC_PROLOG */
2023
2024#if __INTRINSIC_PROLOG(__outword)
2025void __outword(unsigned short Port, unsigned short Data);
2026#if !__has_builtin(__outword)
2027__INTRINSICS_USEINLINE
2028__build_outport(__outword, unsigned short, "w")
2029#endif
2030#define __INTRINSIC_DEFINED___outword
2031#endif /* __INTRINSIC_PROLOG */
2032
2033#if __INTRINSIC_PROLOG(__outdword)
2034void __outdword(unsigned short Port, unsigned __LONG32 Data);
2035#if !__has_builtin(__outdword)
2036__INTRINSICS_USEINLINE
2037__build_outport(__outdword, unsigned __LONG32, "l")
2038#endif
2039#define __INTRINSIC_DEFINED___outdword
2040#endif /* __INTRINSIC_PROLOG */
2041
2042#if __INTRINSIC_PROLOG(__inbytestring)
2043void __inbytestring(unsigned short Port, unsigned char *Buffer, unsigned __LONG32 Count);
2044#if !__has_builtin(__inbytestring)
2045__INTRINSICS_USEINLINE
2046__build_inportstring(__inbytestring, unsigned char, "b", "b")
2047#endif
2048#define __INTRINSIC_DEFINED___inbytestring
2049#endif /* __INTRINSIC_PROLOG */
2050
2051#if __INTRINSIC_PROLOG(__inwordstring)
2052void __inwordstring(unsigned short Port, unsigned short *Buffer, unsigned __LONG32 Count);
2053#if !__has_builtin(__inwordstring)
2054__INTRINSICS_USEINLINE
2055__build_inportstring(__inwordstring, unsigned short, "w", "w")
2056#endif
2057#define __INTRINSIC_DEFINED___inwordstring
2058#endif /* __INTRINSIC_PROLOG */
2059
2060#if __INTRINSIC_PROLOG(__indwordstring)
2061void __indwordstring(unsigned short Port, unsigned __LONG32 *Buffer, unsigned __LONG32 Count);
2062#if !__has_builtin(__indwordstring)
2063__INTRINSICS_USEINLINE
2064__build_inportstring(__indwordstring, unsigned __LONG32, "l", "d")
2065#endif
2066#define __INTRINSIC_DEFINED___indwordstring
2067#endif /* __INTRINSIC_PROLOG */
2068
2069#if __INTRINSIC_PROLOG(__outbytestring)
2070void __outbytestring(unsigned short Port, unsigned char *Buffer, unsigned __LONG32 Count);
2071#if !__has_builtin(__outbytestring)
2072__INTRINSICS_USEINLINE
2073__build_outportstring(__outbytestring, unsigned char, "b", "b")
2074#endif
2075#define __INTRINSIC_DEFINED___outbytestring
2076#endif /* __INTRINSIC_PROLOG */
2077
2078#if __INTRINSIC_PROLOG(__outwordstring)
2079void __outwordstring(unsigned short Port, unsigned short *Buffer, unsigned __LONG32 Count);
2080#if !__has_builtin(__outwordstring)
2081__INTRINSICS_USEINLINE
2082__build_outportstring(__outwordstring, unsigned short, "w", "w")
2083#endif
2084#define __INTRINSIC_DEFINED___outwordstring
2085#endif /* __INTRINSIC_PROLOG */
2086
2087#if __INTRINSIC_PROLOG(__outdwordstring)
2088void __outdwordstring(unsigned short Port, unsigned __LONG32 *Buffer, unsigned __LONG32 Count);
2089#if !__has_builtin(__outdwordstring)
2090__INTRINSICS_USEINLINE
2091__build_outportstring(__outdwordstring, unsigned __LONG32, "l", "d")
2092#endif
2093#define __INTRINSIC_DEFINED___outdwordstring
2094#endif /* __INTRINSIC_PROLOG */
2095
2096#if __INTRINSIC_PROLOG(__cpuid)
2097void __cpuid(int CPUInfo[4], int InfoType);
2098#if !__has_builtin(__cpuid)
2099__INTRINSICS_USEINLINE
2100void __cpuid(int CPUInfo[4], int InfoType) {
2101 __asm__ __volatile__ (
2102 "cpuid"
2103 : "=a" (CPUInfo [0]), "=b" (CPUInfo [1]), "=c" (CPUInfo [2]), "=d" (CPUInfo [3])
2104 : "a" (InfoType));
2105}
2106#endif
2107#define __INTRINSIC_DEFINED___cpuid
2108#endif /* __INTRINSIC_PROLOG */
2109
2110#if (!defined(__GNUC__) || __GNUC__ < 11) && (!defined(__clang__) || __clang_major__ < 19)
2111#if __INTRINSIC_PROLOG(__cpuidex)
2112void __cpuidex(int CPUInfo[4], int, int);
2113#if !__has_builtin(__cpuidex)
2114__INTRINSICS_USEINLINE
2115void __cpuidex(int CPUInfo[4], int function_id, int subfunction_id) {
2116 __asm__ __volatile__ (
2117 "cpuid"
2118 : "=a" (CPUInfo [0]), "=b" (CPUInfo [1]), "=c" (CPUInfo [2]), "=d" (CPUInfo [3])
2119 : "a" (function_id), "c" (subfunction_id));
2120}
2121#endif
2122#define __INTRINSIC_DEFINED___cpuidex
2123#endif /* __INTRINSIC_PROLOG */
2124#endif /* __GNUC__ < 11 */
2125
2126#if __INTRINSIC_PROLOG(__readmsr)
2127__MINGW_EXTENSION unsigned __int64 __readmsr(unsigned __LONG32);
2128#if !__has_builtin(__readmsr)
2129__INTRINSICS_USEINLINE
2130unsigned __int64 __readmsr(unsigned __LONG32 msr)
2131{
2132#if defined(__x86_64__) || defined(_AMD64_)
2133 unsigned __int64 val1, val2;
2134#else
2135 unsigned __LONG32 val1, val2;
2136#endif /* defined(__x86_64__) || defined(_AMD64_) */
2137
2138 __asm__ __volatile__(
2139 "rdmsr"
2140 : "=a" (val1), "=d" (val2)
2141 : "c" (msr));
2142
2143 return ((unsigned __int64) val1) | (((unsigned __int64)val2) << 32);
2144}
2145#endif
2146#define __INTRINSIC_DEFINED___readmsr
2147#endif /* __INTRINSIC_PROLOG */
2148
2149#if __INTRINSIC_PROLOG(__writemsr)
2150__MINGW_EXTENSION void __writemsr(unsigned __LONG32, unsigned __int64);
2151#if !__has_builtin(__writemsr)
2152__INTRINSICS_USEINLINE
2153void __writemsr(unsigned __LONG32 msr, unsigned __int64 Value)
2154{
2155 unsigned __LONG32 val1 = Value, val2 = Value >> 32;
2156 __asm__ __volatile__ (
2157 "wrmsr"
2158 :
2159 : "c" (msr), "a" (val1), "d" (val2));
2160}
2161#endif
2162#define __INTRINSIC_DEFINED___writemsr
2163#endif /* __INTRINSIC_PROLOG */
2164
2165#if __INTRINSIC_PROLOG(__movsb)
2166void __movsb(unsigned char *Destination, unsigned char const *Source, size_t Count);
2167#if !__has_builtin(__movsb)
2168__INTRINSICS_USEINLINE
2169__buildmov(__movsb, unsigned char, "b", "b")
2170#endif
2171#define __INTRINSIC_DEFINED___movsb
2172#endif /* __INTRINSIC_PROLOG */
2173
2174#if __INTRINSIC_PROLOG(__movsw)
2175void __movsw(unsigned short *Dest, unsigned short const *Source, size_t Count);
2176#if !__has_builtin(__movsw)
2177__INTRINSICS_USEINLINE
2178__buildmov(__movsw, unsigned short, "w", "w")
2179#endif
2180#define __INTRINSIC_DEFINED___movsw
2181#endif /* __INTRINSIC_PROLOG */
2182
2183#if __INTRINSIC_PROLOG(__movsd)
2184void __movsd(unsigned __LONG32 *Dest, unsigned __LONG32 const *Source, size_t Count);
2185#if !__has_builtin(__movsd)
2186__INTRINSICS_USEINLINE
2187__buildmov(__movsd, unsigned __LONG32, "l", "d")
2188#endif
2189#define __INTRINSIC_DEFINED___movsd
2190#endif /* __INTRINSIC_PROLOG */
2191
2192/* GCC 8 has already defined _xgetbv, Clang 9 has _xgetbv defined as a macro
2193 * redirecting to the __builtin_ia32_xgetbv builtin. */
2194#if (!defined(__GNUC__) || __GNUC__ < 8) && !defined(_xgetbv)
2195/* NOTE: This should be in immintrin.h */
2196#if __INTRINSIC_PROLOG(_xgetbv)
2197unsigned __int64 _xgetbv(unsigned int);
2198#if !__has_builtin(_xgetbv)
2199__INTRINSICS_USEINLINE
2200unsigned __int64 _xgetbv(unsigned int index)
2201{
2202#if defined(__x86_64__) || defined(_AMD64_)
2203 unsigned __int64 val1, val2;
2204#else
2205 unsigned __LONG32 val1, val2;
2206#endif /* defined(__x86_64__) || defined(_AMD64_) */
2207
2208 __asm__ __volatile__(
2209 "xgetbv"
2210 : "=a" (val1), "=d" (val2)
2211 : "c" (index));
2212
2213 return (((unsigned __int64)val2) << 32) | val1;
2214}
2215#endif
2216#define __INTRINSIC_DEFINED__xgetbv
2217#endif /* __INTRINSIC_PROLOG */
2218#endif /* __GNUC__ < 8 */
2219
2220#endif /* (defined(__x86_64__) && !defined(__arm64ec__)) || (defined(_AMD64_) && !defined(_ARM64EC_)) || defined(__i386__) || defined(_X86_) */
2221
2222/* ***************************************************** */
2223
2224#if defined(__i386__) || defined(_X86_)
2225
2226#if __INTRINSIC_PROLOG(__readfsbyte)
2227unsigned char __readfsbyte(unsigned __LONG32 Offset);
2228#if !__has_builtin(__readfsbyte)
2229__INTRINSICS_USEINLINE
2230__buildreadseg(__readfsbyte, unsigned char, "fs", "b")
2231#endif
2232#define __INTRINSIC_DEFINED___readfsbyte
2233#endif /* __INTRINSIC_PROLOG */
2234
2235#if __INTRINSIC_PROLOG(__readfsword)
2236unsigned short __readfsword(unsigned __LONG32 Offset);
2237#if !__has_builtin(__readfsword)
2238__INTRINSICS_USEINLINE
2239__buildreadseg(__readfsword, unsigned short, "fs", "w")
2240#endif
2241#define __INTRINSIC_DEFINED___readfsword
2242#endif /* __INTRINSIC_PROLOG */
2243
2244#if __INTRINSIC_PROLOG(__readfsdword)
2245unsigned __LONG32 __readfsdword(unsigned __LONG32 Offset);
2246#if !__has_builtin(__readfsdword)
2247__INTRINSICS_USEINLINE
2248__buildreadseg(__readfsdword, unsigned __LONG32, "fs", "l")
2249#endif
2250#define __INTRINSIC_DEFINED___readfsdword
2251#endif /* __INTRINSIC_PROLOG */
2252
2253#if __INTRINSIC_PROLOG(__writefsbyte)
2254void __writefsbyte(unsigned __LONG32 Offset,unsigned char Data);
2255#if !__has_builtin(__writefsbyte)
2256__INTRINSICS_USEINLINE
2257__buildwriteseg(__writefsbyte, unsigned char, "fs", "b")
2258#endif
2259#define __INTRINSIC_DEFINED___writefsbyte
2260#endif /* __INTRINSIC_PROLOG */
2261
2262#if __INTRINSIC_PROLOG(__writefsword)
2263void __writefsword(unsigned __LONG32 Offset,unsigned short Data);
2264#if !__has_builtin(__writefsword)
2265__INTRINSICS_USEINLINE
2266__buildwriteseg(__writefsword, unsigned short, "fs", "w")
2267#endif
2268#define __INTRINSIC_DEFINED___writefsword
2269#endif /* __INTRINSIC_PROLOG */
2270
2271#if __INTRINSIC_PROLOG(__writefsdword)
2272void __writefsdword(unsigned __LONG32 Offset,unsigned __LONG32 Data);
2273#if !__has_builtin(__writefsdword)
2274__INTRINSICS_USEINLINE
2275__buildwriteseg(__writefsdword, unsigned __LONG32, "fs", "l")
2276#endif
2277#define __INTRINSIC_DEFINED___writefsdword
2278#endif /* __INTRINSIC_PROLOG */
2279
2280#if __INTRINSIC_PROLOG(__readcr0)
2281unsigned __LONG32 __readcr0(void);
2282#if !__has_builtin(__readcr0)
2283__INTRINSICS_USEINLINE
2284__build_readcr(__readcr0, unsigned __LONG32, "0")
2285#endif
2286#define __INTRINSIC_DEFINED___readcr0
2287#endif /* __INTRINSIC_PROLOG */
2288
2289#if __INTRINSIC_PROLOG(__readcr2)
2290unsigned __LONG32 __readcr2(void);
2291#if !__has_builtin(__readcr2)
2292__INTRINSICS_USEINLINE
2293__build_readcr(__readcr2, unsigned __LONG32, "2")
2294#endif
2295#define __INTRINSIC_DEFINED___readcr2
2296#endif /* __INTRINSIC_PROLOG */
2297
2298#if __INTRINSIC_PROLOG(__readcr3)
2299unsigned __LONG32 __readcr3(void);
2300#if !__has_builtin(__readcr3)
2301__INTRINSICS_USEINLINE
2302__build_readcr(__readcr3, unsigned __LONG32, "3")
2303#endif
2304#define __INTRINSIC_DEFINED___readcr3
2305#endif /* __INTRINSIC_PROLOG */
2306
2307#if __INTRINSIC_PROLOG(__readcr4)
2308unsigned __LONG32 __readcr4(void);
2309#if !__has_builtin(__readcr4)
2310__INTRINSICS_USEINLINE
2311__build_readcr(__readcr4, unsigned __LONG32, "4")
2312#endif
2313#define __INTRINSIC_DEFINED___readcr4
2314#endif /* __INTRINSIC_PROLOG */
2315
2316#if __INTRINSIC_PROLOG(__readcr8)
2317unsigned __LONG32 __readcr8(void);
2318#if !__has_builtin(__readcr8)
2319__INTRINSICS_USEINLINE
2320__build_readcr(__readcr8, unsigned __LONG32, "8")
2321#endif
2322#define __INTRINSIC_DEFINED___readcr8
2323#endif /* __INTRINSIC_PROLOG */
2324
2325#if __INTRINSIC_PROLOG(__writecr0)
2326void __writecr0(unsigned __LONG32);
2327#if !__has_builtin(__writecr0)
2328__INTRINSICS_USEINLINE
2329__build_writecr(__writecr0, unsigned __LONG32, "0")
2330#endif
2331#define __INTRINSIC_DEFINED___writecr0
2332#endif /* __INTRINSIC_PROLOG */
2333
2334#if __INTRINSIC_PROLOG(__writecr3)
2335void __writecr3(unsigned __LONG32);
2336#if !__has_builtin(__writecr3)
2337__INTRINSICS_USEINLINE
2338__build_writecr(__writecr3, unsigned __LONG32, "3")
2339#endif
2340#define __INTRINSIC_DEFINED___writecr3
2341#endif /* __INTRINSIC_PROLOG */
2342
2343#if __INTRINSIC_PROLOG(__writecr4)
2344void __writecr4(unsigned __LONG32);
2345#if !__has_builtin(__writecr4)
2346__INTRINSICS_USEINLINE
2347__build_writecr(__writecr4, unsigned __LONG32, "4")
2348#endif
2349#define __INTRINSIC_DEFINED___writecr4
2350#endif /* __INTRINSIC_PROLOG */
2351
2352#if __INTRINSIC_PROLOG(__writecr8)
2353void __writecr8(unsigned __LONG32);
2354#if !__has_builtin(__writecr8)
2355__INTRINSICS_USEINLINE
2356__build_writecr(__writecr8, unsigned __LONG32, "8")
2357#endif
2358#define __INTRINSIC_DEFINED___writecr8
2359#endif /* __INTRINSIC_PROLOG */
2360
2361#endif /* defined(__i386__) || defined(_X86_) */
2362
2363#ifdef __cplusplus
2364}
2365#endif
2366
2367#undef __INTRINSIC_ONLYSPECIAL
2368#undef __INTRINSIC_PROLOG
2369#undef __INTRINSIC_EPILOG
2370#undef __INTRINSICS_USEINLINE
2371#undef __FLAGCONSTRAINT
2372#undef __FLAGSET
2373#undef __FLAGCLOBBER1
2374#undef __FLAGCLOBBER2
2375
2376#if defined(__GNUC__) && (__GNUC__ >= 7 || defined(__clang__))
2377#pragma GCC diagnostic pop
2378#endif
2379
2380#pragma pop_macro("__has_builtin")
2381
2382#endif /* __MINGW_INTRIN_INLINE */