1const builtin = @import("builtin");
2const cpu = builtin.cpu;
3const arch = cpu.arch;
4
5const std = @import("std");
6
7const compiler_rt = @import("../compiler_rt.zig");
8const symbol = compiler_rt.symbol;
9
10// This parameter is true iff the target architecture supports the bare minimum
11// to implement the atomic load/store intrinsics.
12// Some architectures support atomic load/stores but no CAS, but we ignore this
13// detail to keep the export logic clean and because we need some kind of CAS to
14// implement the spinlocks.
15const supports_atomic_ops = switch (arch) {
16 .msp430, .avr, .bpfel, .bpfeb => false,
17 .arm, .armeb, .thumb, .thumbeb =>
18 // The ARM v6m ISA has no ldrex/strex and so it's impossible to do CAS
19 // operations (unless we're targeting Linux, the kernel provides a way to
20 // perform CAS operations).
21 // XXX: The Linux code path is not implemented yet.
22 !builtin.cpu.has(.arm, .has_v6m),
23 else => true,
24};
25
26// The size (in bytes) of the biggest object that the architecture can
27// load/store atomically.
28// Objects bigger than this threshold require the use of a lock.
29const largest_atomic_size = switch (arch) {
30 // On SPARC systems that lacks CAS and/or swap instructions, the only
31 // available atomic operation is a test-and-set (`ldstub`), so we force
32 // every atomic memory access to go through the lock.
33 .sparc => if (builtin.cpu.has(.sparc, .hasleoncasa)) @sizeOf(usize) else 0,
34
35 // XXX: On x86/x86_64 we could check the presence of cmpxchg8b/cmpxchg16b
36 // and set this parameter accordingly.
37 else => @sizeOf(usize),
38};
39
40// The size (in bytes) of the smallest atomic object that the architecture can
41// perform fetch/exchange atomically. Note, this does not encompass load and store.
42// Objects smaller than this threshold are implemented in terms of compare-exchange
43// of a larger value.
44const smallest_atomic_fetch_exch_size = switch (arch) {
45 // On AMDGCN, there are no instructions for atomic operations other than load and store
46 // (as of LLVM 15), and so these need to be implemented in terms of atomic CAS.
47 .amdgcn => @sizeOf(u32),
48 else => @sizeOf(u8),
49};
50
51const cache_line_size = 64;
52
53const SpinlockTable = struct {
54 // Allocate ~4096 bytes of memory for the spinlock table
55 const max_spinlocks = 64;
56
57 const Spinlock = struct {
58 // SPARC ldstub instruction will write a 255 into the memory location.
59 // We'll use that as a sign that the lock is currently held.
60 // See also: Section B.7 in SPARCv8 spec & A.29 in SPARCv9 spec.
61 const sparc_lock: type = enum(u8) { Unlocked = 0, Locked = 255 };
62 const other_lock: type = enum(usize) { Unlocked = 0, Locked };
63
64 // Prevent false sharing by providing enough padding between two
65 // consecutive spinlock elements
66 v: if (arch.isSPARC()) sparc_lock else other_lock align(cache_line_size) = .Unlocked,
67
68 fn acquire(self: *@This()) void {
69 while (true) {
70 const flag = if (comptime arch.isSPARC()) flag: {
71 break :flag asm volatile ("ldstub [%[addr]], %[flag]"
72 : [flag] "=r" (-> @TypeOf(self.v)),
73 : [addr] "r" (&self.v),
74 : .{ .memory = true });
75 } else flag: {
76 break :flag @atomicRmw(@TypeOf(self.v), &self.v, .Xchg, .Locked, .acquire);
77 };
78
79 switch (flag) {
80 .Unlocked => break,
81 .Locked => {},
82 }
83 }
84 }
85 fn release(self: *@This()) void {
86 if (comptime arch.isSPARC()) {
87 _ = asm volatile ("clrb [%[addr]]"
88 :
89 : [addr] "r" (&self.v),
90 : .{ .memory = true });
91 } else {
92 @atomicStore(@TypeOf(self.v), &self.v, .Unlocked, .release);
93 }
94 }
95 };
96
97 list: [max_spinlocks]Spinlock = @splat(.{}),
98
99 // The spinlock table behaves as a really simple hash table, mapping
100 // addresses to spinlocks. The mapping is not unique but that's only a
101 // performance problem as the lock will be contended by more than a pair of
102 // threads.
103 fn get(self: *@This(), address: usize) *Spinlock {
104 var sl = &self.list[(address >> 3) % max_spinlocks];
105 sl.acquire();
106 return sl;
107 }
108};
109
110var spinlocks: SpinlockTable = SpinlockTable{};
111
112// The following builtins do not respect the specified memory model and instead
113// uses seq_cst, the strongest one, for simplicity sake.
114
115// Generic version of GCC atomic builtin functions.
116// Those work on any object no matter the pointer alignment nor its size.
117
118fn __atomic_load(size: u32, src: [*]u8, dest: [*]u8, model: i32) callconv(.c) void {
119 _ = model;
120 var sl = spinlocks.get(@intFromPtr(src));
121 defer sl.release();
122 @memcpy(dest[0..size], src);
123}
124
125fn __atomic_store(size: u32, dest: [*]u8, src: [*]u8, model: i32) callconv(.c) void {
126 _ = model;
127 var sl = spinlocks.get(@intFromPtr(dest));
128 defer sl.release();
129 @memcpy(dest[0..size], src);
130}
131
132fn __atomic_exchange(size: u32, ptr: [*]u8, val: [*]u8, old: [*]u8, model: i32) callconv(.c) void {
133 _ = model;
134 var sl = spinlocks.get(@intFromPtr(ptr));
135 defer sl.release();
136 @memcpy(old[0..size], ptr);
137 @memcpy(ptr[0..size], val);
138}
139
140fn __atomic_compare_exchange(
141 size: u32,
142 ptr: [*]u8,
143 expected: [*]u8,
144 desired: [*]u8,
145 success: i32,
146 failure: i32,
147) callconv(.c) i32 {
148 _ = success;
149 _ = failure;
150 var sl = spinlocks.get(@intFromPtr(ptr));
151 defer sl.release();
152 for (ptr[0..size], 0..) |b, i| {
153 if (expected[i] != b) break;
154 } else {
155 // The two objects, ptr and expected, are equal
156 @memcpy(ptr[0..size], desired);
157 return 1;
158 }
159 @memcpy(expected[0..size], ptr);
160 return 0;
161}
162
163// Specialized versions of the GCC atomic builtin functions.
164// LLVM emits those iff the object size is known and the pointers are correctly
165// aligned.
166inline fn atomic_load_N(comptime T: type, src: *T, model: i32) T {
167 _ = model;
168 if (@sizeOf(T) > largest_atomic_size) {
169 var sl = spinlocks.get(@intFromPtr(src));
170 defer sl.release();
171 return src.*;
172 } else {
173 return @atomicLoad(T, src, .seq_cst);
174 }
175}
176
177fn __atomic_load_1(src: *u8, model: i32) callconv(.c) u8 {
178 return atomic_load_N(u8, src, model);
179}
180
181fn __atomic_load_2(src: *u16, model: i32) callconv(.c) u16 {
182 return atomic_load_N(u16, src, model);
183}
184
185fn __atomic_load_4(src: *u32, model: i32) callconv(.c) u32 {
186 return atomic_load_N(u32, src, model);
187}
188
189fn __atomic_load_8(src: *u64, model: i32) callconv(.c) u64 {
190 return atomic_load_N(u64, src, model);
191}
192
193fn __atomic_load_16(src: *u128, model: i32) callconv(.c) u128 {
194 return atomic_load_N(u128, src, model);
195}
196
197inline fn atomic_store_N(comptime T: type, dst: *T, value: T, model: i32) void {
198 _ = model;
199 if (@sizeOf(T) > largest_atomic_size) {
200 var sl = spinlocks.get(@intFromPtr(dst));
201 defer sl.release();
202 dst.* = value;
203 } else {
204 @atomicStore(T, dst, value, .seq_cst);
205 }
206}
207
208fn __atomic_store_1(dst: *u8, value: u8, model: i32) callconv(.c) void {
209 return atomic_store_N(u8, dst, value, model);
210}
211
212fn __atomic_store_2(dst: *u16, value: u16, model: i32) callconv(.c) void {
213 return atomic_store_N(u16, dst, value, model);
214}
215
216fn __atomic_store_4(dst: *u32, value: u32, model: i32) callconv(.c) void {
217 return atomic_store_N(u32, dst, value, model);
218}
219
220fn __atomic_store_8(dst: *u64, value: u64, model: i32) callconv(.c) void {
221 return atomic_store_N(u64, dst, value, model);
222}
223
224fn __atomic_store_16(dst: *u128, value: u128, model: i32) callconv(.c) void {
225 return atomic_store_N(u128, dst, value, model);
226}
227
228fn wideUpdate(comptime T: type, ptr: *T, val: T, update: anytype) T {
229 const WideAtomic = @Int(.unsigned, smallest_atomic_fetch_exch_size * 8);
230
231 const addr = @intFromPtr(ptr);
232 const wide_addr = addr & ~(@as(T, smallest_atomic_fetch_exch_size) - 1);
233 const wide_ptr: *align(smallest_atomic_fetch_exch_size) WideAtomic = @alignCast(@as(*WideAtomic, @ptrFromInt(wide_addr)));
234
235 const inner_offset = addr & (@as(T, smallest_atomic_fetch_exch_size) - 1);
236 const inner_shift = @as(std.math.Log2Int(T), @intCast(inner_offset * 8));
237
238 const mask = @as(WideAtomic, std.math.maxInt(T)) << inner_shift;
239
240 var wide_old = @atomicLoad(WideAtomic, wide_ptr, .seq_cst);
241 while (true) {
242 const old = @as(T, @truncate((wide_old & mask) >> inner_shift));
243 const new = update(val, old);
244 const wide_new = wide_old & ~mask | (@as(WideAtomic, new) << inner_shift);
245 if (@cmpxchgWeak(WideAtomic, wide_ptr, wide_old, wide_new, .seq_cst, .seq_cst)) |new_wide_old| {
246 wide_old = new_wide_old;
247 } else {
248 return old;
249 }
250 }
251}
252
253inline fn atomic_exchange_N(comptime T: type, ptr: *T, val: T, model: i32) T {
254 _ = model;
255 if (@sizeOf(T) > largest_atomic_size) {
256 var sl = spinlocks.get(@intFromPtr(ptr));
257 defer sl.release();
258 const value = ptr.*;
259 ptr.* = val;
260 return value;
261 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
262 // Machine does not support this type, but it does support a larger type.
263 const Updater = struct {
264 fn update(new: T, old: T) T {
265 _ = old;
266 return new;
267 }
268 };
269 return wideUpdate(T, ptr, val, Updater.update);
270 } else {
271 return @atomicRmw(T, ptr, .Xchg, val, .seq_cst);
272 }
273}
274
275fn __atomic_exchange_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
276 return atomic_exchange_N(u8, ptr, val, model);
277}
278
279fn __atomic_exchange_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
280 return atomic_exchange_N(u16, ptr, val, model);
281}
282
283fn __atomic_exchange_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
284 return atomic_exchange_N(u32, ptr, val, model);
285}
286
287fn __atomic_exchange_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
288 return atomic_exchange_N(u64, ptr, val, model);
289}
290
291fn __atomic_exchange_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
292 return atomic_exchange_N(u128, ptr, val, model);
293}
294
295inline fn atomic_compare_exchange_N(
296 comptime T: type,
297 ptr: *T,
298 expected: *T,
299 desired: T,
300 success: i32,
301 failure: i32,
302) i32 {
303 _ = success;
304 _ = failure;
305 if (@sizeOf(T) > largest_atomic_size) {
306 var sl = spinlocks.get(@intFromPtr(ptr));
307 defer sl.release();
308 const value = ptr.*;
309 if (value == expected.*) {
310 ptr.* = desired;
311 return 1;
312 }
313 expected.* = value;
314 return 0;
315 } else {
316 if (@cmpxchgStrong(T, ptr, expected.*, desired, .seq_cst, .seq_cst)) |old_value| {
317 expected.* = old_value;
318 return 0;
319 }
320 return 1;
321 }
322}
323
324fn __atomic_compare_exchange_1(ptr: *u8, expected: *u8, desired: u8, success: i32, failure: i32) callconv(.c) i32 {
325 return atomic_compare_exchange_N(u8, ptr, expected, desired, success, failure);
326}
327
328fn __atomic_compare_exchange_2(ptr: *u16, expected: *u16, desired: u16, success: i32, failure: i32) callconv(.c) i32 {
329 return atomic_compare_exchange_N(u16, ptr, expected, desired, success, failure);
330}
331
332fn __atomic_compare_exchange_4(ptr: *u32, expected: *u32, desired: u32, success: i32, failure: i32) callconv(.c) i32 {
333 return atomic_compare_exchange_N(u32, ptr, expected, desired, success, failure);
334}
335
336fn __atomic_compare_exchange_8(ptr: *u64, expected: *u64, desired: u64, success: i32, failure: i32) callconv(.c) i32 {
337 return atomic_compare_exchange_N(u64, ptr, expected, desired, success, failure);
338}
339
340fn __atomic_compare_exchange_16(ptr: *u128, expected: *u128, desired: u128, success: i32, failure: i32) callconv(.c) i32 {
341 return atomic_compare_exchange_N(u128, ptr, expected, desired, success, failure);
342}
343
344inline fn fetch_op_N(comptime T: type, comptime op: std.builtin.AtomicRmwOp, ptr: *T, val: T, model: i32) T {
345 _ = model;
346 const Updater = struct {
347 fn update(new: T, old: T) T {
348 return switch (op) {
349 .Add => old +% new,
350 .Sub => old -% new,
351 .And => old & new,
352 .Nand => ~(old & new),
353 .Or => old | new,
354 .Xor => old ^ new,
355 .Max => @max(old, new),
356 .Min => @min(old, new),
357 else => @compileError("unsupported atomic op"),
358 };
359 }
360 };
361
362 if (@sizeOf(T) > largest_atomic_size) {
363 var sl = spinlocks.get(@intFromPtr(ptr));
364 defer sl.release();
365
366 const value = ptr.*;
367 ptr.* = Updater.update(val, value);
368 return value;
369 } else if (@sizeOf(T) < smallest_atomic_fetch_exch_size) {
370 // Machine does not support this type, but it does support a larger type.
371 return wideUpdate(T, ptr, val, Updater.update);
372 }
373
374 return @atomicRmw(T, ptr, op, val, .seq_cst);
375}
376
377fn __atomic_fetch_add_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
378 return fetch_op_N(u8, .Add, ptr, val, model);
379}
380
381fn __atomic_fetch_add_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
382 return fetch_op_N(u16, .Add, ptr, val, model);
383}
384
385fn __atomic_fetch_add_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
386 return fetch_op_N(u32, .Add, ptr, val, model);
387}
388
389fn __atomic_fetch_add_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
390 return fetch_op_N(u64, .Add, ptr, val, model);
391}
392
393fn __atomic_fetch_add_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
394 return fetch_op_N(u128, .Add, ptr, val, model);
395}
396
397fn __atomic_fetch_sub_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
398 return fetch_op_N(u8, .Sub, ptr, val, model);
399}
400
401fn __atomic_fetch_sub_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
402 return fetch_op_N(u16, .Sub, ptr, val, model);
403}
404
405fn __atomic_fetch_sub_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
406 return fetch_op_N(u32, .Sub, ptr, val, model);
407}
408
409fn __atomic_fetch_sub_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
410 return fetch_op_N(u64, .Sub, ptr, val, model);
411}
412
413fn __atomic_fetch_sub_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
414 return fetch_op_N(u128, .Sub, ptr, val, model);
415}
416
417fn __atomic_fetch_and_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
418 return fetch_op_N(u8, .And, ptr, val, model);
419}
420
421fn __atomic_fetch_and_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
422 return fetch_op_N(u16, .And, ptr, val, model);
423}
424
425fn __atomic_fetch_and_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
426 return fetch_op_N(u32, .And, ptr, val, model);
427}
428
429fn __atomic_fetch_and_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
430 return fetch_op_N(u64, .And, ptr, val, model);
431}
432
433fn __atomic_fetch_and_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
434 return fetch_op_N(u128, .And, ptr, val, model);
435}
436
437fn __atomic_fetch_or_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
438 return fetch_op_N(u8, .Or, ptr, val, model);
439}
440
441fn __atomic_fetch_or_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
442 return fetch_op_N(u16, .Or, ptr, val, model);
443}
444
445fn __atomic_fetch_or_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
446 return fetch_op_N(u32, .Or, ptr, val, model);
447}
448
449fn __atomic_fetch_or_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
450 return fetch_op_N(u64, .Or, ptr, val, model);
451}
452
453fn __atomic_fetch_or_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
454 return fetch_op_N(u128, .Or, ptr, val, model);
455}
456
457fn __atomic_fetch_xor_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
458 return fetch_op_N(u8, .Xor, ptr, val, model);
459}
460
461fn __atomic_fetch_xor_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
462 return fetch_op_N(u16, .Xor, ptr, val, model);
463}
464
465fn __atomic_fetch_xor_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
466 return fetch_op_N(u32, .Xor, ptr, val, model);
467}
468
469fn __atomic_fetch_xor_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
470 return fetch_op_N(u64, .Xor, ptr, val, model);
471}
472
473fn __atomic_fetch_xor_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
474 return fetch_op_N(u128, .Xor, ptr, val, model);
475}
476
477fn __atomic_fetch_nand_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
478 return fetch_op_N(u8, .Nand, ptr, val, model);
479}
480
481fn __atomic_fetch_nand_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
482 return fetch_op_N(u16, .Nand, ptr, val, model);
483}
484
485fn __atomic_fetch_nand_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
486 return fetch_op_N(u32, .Nand, ptr, val, model);
487}
488
489fn __atomic_fetch_nand_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
490 return fetch_op_N(u64, .Nand, ptr, val, model);
491}
492
493fn __atomic_fetch_nand_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
494 return fetch_op_N(u128, .Nand, ptr, val, model);
495}
496
497fn __atomic_fetch_umax_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
498 return fetch_op_N(u8, .Max, ptr, val, model);
499}
500
501fn __atomic_fetch_umax_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
502 return fetch_op_N(u16, .Max, ptr, val, model);
503}
504
505fn __atomic_fetch_umax_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
506 return fetch_op_N(u32, .Max, ptr, val, model);
507}
508
509fn __atomic_fetch_umax_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
510 return fetch_op_N(u64, .Max, ptr, val, model);
511}
512
513fn __atomic_fetch_umax_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
514 return fetch_op_N(u128, .Max, ptr, val, model);
515}
516
517fn __atomic_fetch_umin_1(ptr: *u8, val: u8, model: i32) callconv(.c) u8 {
518 return fetch_op_N(u8, .Min, ptr, val, model);
519}
520
521fn __atomic_fetch_umin_2(ptr: *u16, val: u16, model: i32) callconv(.c) u16 {
522 return fetch_op_N(u16, .Min, ptr, val, model);
523}
524
525fn __atomic_fetch_umin_4(ptr: *u32, val: u32, model: i32) callconv(.c) u32 {
526 return fetch_op_N(u32, .Min, ptr, val, model);
527}
528
529fn __atomic_fetch_umin_8(ptr: *u64, val: u64, model: i32) callconv(.c) u64 {
530 return fetch_op_N(u64, .Min, ptr, val, model);
531}
532
533fn __atomic_fetch_umin_16(ptr: *u128, val: u128, model: i32) callconv(.c) u128 {
534 return fetch_op_N(u128, .Min, ptr, val, model);
535}
536
537comptime {
538 if (supports_atomic_ops and builtin.object_format != .c) {
539 symbol(&__atomic_load, "__atomic_load");
540 symbol(&__atomic_store, "__atomic_store");
541 symbol(&__atomic_exchange, "__atomic_exchange");
542 symbol(&__atomic_compare_exchange, "__atomic_compare_exchange");
543
544 symbol(&__atomic_fetch_add_1, "__atomic_fetch_add_1");
545 symbol(&__atomic_fetch_add_2, "__atomic_fetch_add_2");
546 symbol(&__atomic_fetch_add_4, "__atomic_fetch_add_4");
547 symbol(&__atomic_fetch_add_8, "__atomic_fetch_add_8");
548 symbol(&__atomic_fetch_add_16, "__atomic_fetch_add_16");
549
550 symbol(&__atomic_fetch_sub_1, "__atomic_fetch_sub_1");
551 symbol(&__atomic_fetch_sub_2, "__atomic_fetch_sub_2");
552 symbol(&__atomic_fetch_sub_4, "__atomic_fetch_sub_4");
553 symbol(&__atomic_fetch_sub_8, "__atomic_fetch_sub_8");
554 symbol(&__atomic_fetch_sub_16, "__atomic_fetch_sub_16");
555
556 symbol(&__atomic_fetch_and_1, "__atomic_fetch_and_1");
557 symbol(&__atomic_fetch_and_2, "__atomic_fetch_and_2");
558 symbol(&__atomic_fetch_and_4, "__atomic_fetch_and_4");
559 symbol(&__atomic_fetch_and_8, "__atomic_fetch_and_8");
560 symbol(&__atomic_fetch_and_16, "__atomic_fetch_and_16");
561
562 symbol(&__atomic_fetch_or_1, "__atomic_fetch_or_1");
563 symbol(&__atomic_fetch_or_2, "__atomic_fetch_or_2");
564 symbol(&__atomic_fetch_or_4, "__atomic_fetch_or_4");
565 symbol(&__atomic_fetch_or_8, "__atomic_fetch_or_8");
566 symbol(&__atomic_fetch_or_16, "__atomic_fetch_or_16");
567
568 symbol(&__atomic_fetch_xor_1, "__atomic_fetch_xor_1");
569 symbol(&__atomic_fetch_xor_2, "__atomic_fetch_xor_2");
570 symbol(&__atomic_fetch_xor_4, "__atomic_fetch_xor_4");
571 symbol(&__atomic_fetch_xor_8, "__atomic_fetch_xor_8");
572 symbol(&__atomic_fetch_xor_16, "__atomic_fetch_xor_16");
573
574 symbol(&__atomic_fetch_nand_1, "__atomic_fetch_nand_1");
575 symbol(&__atomic_fetch_nand_2, "__atomic_fetch_nand_2");
576 symbol(&__atomic_fetch_nand_4, "__atomic_fetch_nand_4");
577 symbol(&__atomic_fetch_nand_8, "__atomic_fetch_nand_8");
578 symbol(&__atomic_fetch_nand_16, "__atomic_fetch_nand_16");
579
580 symbol(&__atomic_fetch_umax_1, "__atomic_fetch_umax_1");
581 symbol(&__atomic_fetch_umax_2, "__atomic_fetch_umax_2");
582 symbol(&__atomic_fetch_umax_4, "__atomic_fetch_umax_4");
583 symbol(&__atomic_fetch_umax_8, "__atomic_fetch_umax_8");
584 symbol(&__atomic_fetch_umax_16, "__atomic_fetch_umax_16");
585
586 symbol(&__atomic_fetch_umin_1, "__atomic_fetch_umin_1");
587 symbol(&__atomic_fetch_umin_2, "__atomic_fetch_umin_2");
588 symbol(&__atomic_fetch_umin_4, "__atomic_fetch_umin_4");
589 symbol(&__atomic_fetch_umin_8, "__atomic_fetch_umin_8");
590 symbol(&__atomic_fetch_umin_16, "__atomic_fetch_umin_16");
591
592 symbol(&__atomic_load_1, "__atomic_load_1");
593 symbol(&__atomic_load_2, "__atomic_load_2");
594 symbol(&__atomic_load_4, "__atomic_load_4");
595 symbol(&__atomic_load_8, "__atomic_load_8");
596 symbol(&__atomic_load_16, "__atomic_load_16");
597
598 symbol(&__atomic_store_1, "__atomic_store_1");
599 symbol(&__atomic_store_2, "__atomic_store_2");
600 symbol(&__atomic_store_4, "__atomic_store_4");
601 symbol(&__atomic_store_8, "__atomic_store_8");
602 symbol(&__atomic_store_16, "__atomic_store_16");
603
604 symbol(&__atomic_exchange_1, "__atomic_exchange_1");
605 symbol(&__atomic_exchange_2, "__atomic_exchange_2");
606 symbol(&__atomic_exchange_4, "__atomic_exchange_4");
607 symbol(&__atomic_exchange_8, "__atomic_exchange_8");
608 symbol(&__atomic_exchange_16, "__atomic_exchange_16");
609
610 symbol(&__atomic_compare_exchange_1, "__atomic_compare_exchange_1");
611 symbol(&__atomic_compare_exchange_2, "__atomic_compare_exchange_2");
612 symbol(&__atomic_compare_exchange_4, "__atomic_compare_exchange_4");
613 symbol(&__atomic_compare_exchange_8, "__atomic_compare_exchange_8");
614 symbol(&__atomic_compare_exchange_16, "__atomic_compare_exchange_16");
615 }
616}