authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-21 19:17:50-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-01-09 14:36:40-07:00
log2bbaf95ebea577652bcef474c418bf9a31abf4ce
treedc9d66f99e1bcfeb099cf8bb6416cb1c33ac8d9b
parenta3232c6764d9adc2d3b93e9ed76208d60b509f7a

Merge pull request #11828 from devins2518/arm-atomics

compiler_rt: aarch64 outline atomics

3 files changed, 2572 insertions(+), 0 deletions(-)

lib/compiler_rt.zig+7
...@@ -1,8 +1,15 @@...@@ -1,8 +1,15 @@
1const builtin = @import("builtin");
2
1pub const panic = @import("compiler_rt/common.zig").panic;3pub const panic = @import("compiler_rt/common.zig").panic;
24
3comptime {5comptime {
4 _ = @import("compiler_rt/atomics.zig");6 _ = @import("compiler_rt/atomics.zig");
57
8 // macOS has these functions inside libSystem.
9 if (builtin.cpu.arch.isAARCH64() and !builtin.os.tag.isDarwin()) {
10 _ = @import("compiler_rt/aarch64_outline_atomics.zig");
11 }
12
6 _ = @import("compiler_rt/addf3.zig");13 _ = @import("compiler_rt/addf3.zig");
7 _ = @import("compiler_rt/addhf3.zig");14 _ = @import("compiler_rt/addhf3.zig");
8 _ = @import("compiler_rt/addsf3.zig");15 _ = @import("compiler_rt/addsf3.zig");
lib/compiler_rt/aarch64_outline_atomics.zig created+2227
...@@ -0,0 +1,2227 @@
1//! This file is generated by tools/gen_outline_atomics.zig.
2const builtin = @import("builtin");
3const std = @import("std");
4const linkage = @import("./common.zig").linkage;
5const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .lse);
6
7/// This default is overridden at runtime after inspecting CPU properties.
8/// It is intentionally not exported in order to make the machine code that
9/// uses it a statically predicted direct branch rather than using the PLT,
10/// which ARM is concerned would have too much overhead.
11var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);
12
13fn __aarch64_cas1_relax() align(16) callconv(.Naked) void {
14 @setRuntimeSafety(false);
15 asm volatile (
16 \\ cbz w16, 8f
17 \\ .inst 0x08a07c41 + 0x00000000 + 0x000000
18 \\ ret
19 \\8:
20 \\ uxtb w16, w0
21 \\0:
22 \\ ldxrb w0, [x2]
23 \\ cmp w0, w16
24 \\ bne 1f
25 \\ stxrb w17, w1, [x2]
26 \\ cbnz w17, 0b
27 \\1:
28 \\ ret
29 :
30 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
31 : "w15", "w16", "w17", "memory"
32 );
33 unreachable;
34}
35fn __aarch64_swp1_relax() align(16) callconv(.Naked) void {
36 @setRuntimeSafety(false);
37 asm volatile (
38 \\ cbz w16, 8f
39 \\ .inst 0x38208020 + 0x00000000 + 0x000000
40 \\ ret
41 \\8:
42 \\ mov w16, w0
43 \\0:
44 \\ ldxrb w0, [x1]
45 \\ stxrb w17, w16, [x1]
46 \\ cbnz w17, 0b
47 \\1:
48 \\ ret
49 :
50 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
51 : "w15", "w16", "w17", "memory"
52 );
53 unreachable;
54}
55fn __aarch64_ldadd1_relax() align(16) callconv(.Naked) void {
56 @setRuntimeSafety(false);
57 asm volatile (
58 \\ cbz w16, 8f
59 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
60 \\ ret
61 \\8:
62 \\ mov w16, w0
63 \\0:
64 \\ ldxrb w0, [x1]
65 \\ add w17, w0, w16
66 \\ stxrb w15, w17, [x1]
67 \\ cbnz w15, 0b
68 \\1:
69 \\ ret
70 :
71 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
72 : "w15", "w16", "w17", "memory"
73 );
74 unreachable;
75}
76fn __aarch64_ldclr1_relax() align(16) callconv(.Naked) void {
77 @setRuntimeSafety(false);
78 asm volatile (
79 \\ cbz w16, 8f
80 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
81 \\ ret
82 \\8:
83 \\ mov w16, w0
84 \\0:
85 \\ ldxrb w0, [x1]
86 \\ bic w17, w0, w16
87 \\ stxrb w15, w17, [x1]
88 \\ cbnz w15, 0b
89 \\1:
90 \\ ret
91 :
92 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
93 : "w15", "w16", "w17", "memory"
94 );
95 unreachable;
96}
97fn __aarch64_ldeor1_relax() align(16) callconv(.Naked) void {
98 @setRuntimeSafety(false);
99 asm volatile (
100 \\ cbz w16, 8f
101 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
102 \\ ret
103 \\8:
104 \\ mov w16, w0
105 \\0:
106 \\ ldxrb w0, [x1]
107 \\ eor w17, w0, w16
108 \\ stxrb w15, w17, [x1]
109 \\ cbnz w15, 0b
110 \\1:
111 \\ ret
112 :
113 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
114 : "w15", "w16", "w17", "memory"
115 );
116 unreachable;
117}
118fn __aarch64_ldset1_relax() align(16) callconv(.Naked) void {
119 @setRuntimeSafety(false);
120 asm volatile (
121 \\ cbz w16, 8f
122 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
123 \\ ret
124 \\8:
125 \\ mov w16, w0
126 \\0:
127 \\ ldxrb w0, [x1]
128 \\ orr w17, w0, w16
129 \\ stxrb w15, w17, [x1]
130 \\ cbnz w15, 0b
131 \\1:
132 \\ ret
133 :
134 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
135 : "w15", "w16", "w17", "memory"
136 );
137 unreachable;
138}
139fn __aarch64_cas1_acq() align(16) callconv(.Naked) void {
140 @setRuntimeSafety(false);
141 asm volatile (
142 \\ cbz w16, 8f
143 \\ .inst 0x08a07c41 + 0x00000000 + 0x400000
144 \\ ret
145 \\8:
146 \\ uxtb w16, w0
147 \\0:
148 \\ ldaxrb w0, [x2]
149 \\ cmp w0, w16
150 \\ bne 1f
151 \\ stxrb w17, w1, [x2]
152 \\ cbnz w17, 0b
153 \\1:
154 \\ ret
155 :
156 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
157 : "w15", "w16", "w17", "memory"
158 );
159 unreachable;
160}
161fn __aarch64_swp1_acq() align(16) callconv(.Naked) void {
162 @setRuntimeSafety(false);
163 asm volatile (
164 \\ cbz w16, 8f
165 \\ .inst 0x38208020 + 0x00000000 + 0x800000
166 \\ ret
167 \\8:
168 \\ mov w16, w0
169 \\0:
170 \\ ldaxrb w0, [x1]
171 \\ stxrb w17, w16, [x1]
172 \\ cbnz w17, 0b
173 \\1:
174 \\ ret
175 :
176 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
177 : "w15", "w16", "w17", "memory"
178 );
179 unreachable;
180}
181fn __aarch64_ldadd1_acq() align(16) callconv(.Naked) void {
182 @setRuntimeSafety(false);
183 asm volatile (
184 \\ cbz w16, 8f
185 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
186 \\ ret
187 \\8:
188 \\ mov w16, w0
189 \\0:
190 \\ ldaxrb w0, [x1]
191 \\ add w17, w0, w16
192 \\ stxrb w15, w17, [x1]
193 \\ cbnz w15, 0b
194 \\1:
195 \\ ret
196 :
197 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
198 : "w15", "w16", "w17", "memory"
199 );
200 unreachable;
201}
202fn __aarch64_ldclr1_acq() align(16) callconv(.Naked) void {
203 @setRuntimeSafety(false);
204 asm volatile (
205 \\ cbz w16, 8f
206 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
207 \\ ret
208 \\8:
209 \\ mov w16, w0
210 \\0:
211 \\ ldaxrb w0, [x1]
212 \\ bic w17, w0, w16
213 \\ stxrb w15, w17, [x1]
214 \\ cbnz w15, 0b
215 \\1:
216 \\ ret
217 :
218 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
219 : "w15", "w16", "w17", "memory"
220 );
221 unreachable;
222}
223fn __aarch64_ldeor1_acq() align(16) callconv(.Naked) void {
224 @setRuntimeSafety(false);
225 asm volatile (
226 \\ cbz w16, 8f
227 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
228 \\ ret
229 \\8:
230 \\ mov w16, w0
231 \\0:
232 \\ ldaxrb w0, [x1]
233 \\ eor w17, w0, w16
234 \\ stxrb w15, w17, [x1]
235 \\ cbnz w15, 0b
236 \\1:
237 \\ ret
238 :
239 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
240 : "w15", "w16", "w17", "memory"
241 );
242 unreachable;
243}
244fn __aarch64_ldset1_acq() align(16) callconv(.Naked) void {
245 @setRuntimeSafety(false);
246 asm volatile (
247 \\ cbz w16, 8f
248 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
249 \\ ret
250 \\8:
251 \\ mov w16, w0
252 \\0:
253 \\ ldaxrb w0, [x1]
254 \\ orr w17, w0, w16
255 \\ stxrb w15, w17, [x1]
256 \\ cbnz w15, 0b
257 \\1:
258 \\ ret
259 :
260 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
261 : "w15", "w16", "w17", "memory"
262 );
263 unreachable;
264}
265fn __aarch64_cas1_rel() align(16) callconv(.Naked) void {
266 @setRuntimeSafety(false);
267 asm volatile (
268 \\ cbz w16, 8f
269 \\ .inst 0x08a07c41 + 0x00000000 + 0x008000
270 \\ ret
271 \\8:
272 \\ uxtb w16, w0
273 \\0:
274 \\ ldxrb w0, [x2]
275 \\ cmp w0, w16
276 \\ bne 1f
277 \\ stlxrb w17, w1, [x2]
278 \\ cbnz w17, 0b
279 \\1:
280 \\ ret
281 :
282 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
283 : "w15", "w16", "w17", "memory"
284 );
285 unreachable;
286}
287fn __aarch64_swp1_rel() align(16) callconv(.Naked) void {
288 @setRuntimeSafety(false);
289 asm volatile (
290 \\ cbz w16, 8f
291 \\ .inst 0x38208020 + 0x00000000 + 0x400000
292 \\ ret
293 \\8:
294 \\ mov w16, w0
295 \\0:
296 \\ ldxrb w0, [x1]
297 \\ stlxrb w17, w16, [x1]
298 \\ cbnz w17, 0b
299 \\1:
300 \\ ret
301 :
302 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
303 : "w15", "w16", "w17", "memory"
304 );
305 unreachable;
306}
307fn __aarch64_ldadd1_rel() align(16) callconv(.Naked) void {
308 @setRuntimeSafety(false);
309 asm volatile (
310 \\ cbz w16, 8f
311 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
312 \\ ret
313 \\8:
314 \\ mov w16, w0
315 \\0:
316 \\ ldxrb w0, [x1]
317 \\ add w17, w0, w16
318 \\ stlxrb w15, w17, [x1]
319 \\ cbnz w15, 0b
320 \\1:
321 \\ ret
322 :
323 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
324 : "w15", "w16", "w17", "memory"
325 );
326 unreachable;
327}
328fn __aarch64_ldclr1_rel() align(16) callconv(.Naked) void {
329 @setRuntimeSafety(false);
330 asm volatile (
331 \\ cbz w16, 8f
332 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
333 \\ ret
334 \\8:
335 \\ mov w16, w0
336 \\0:
337 \\ ldxrb w0, [x1]
338 \\ bic w17, w0, w16
339 \\ stlxrb w15, w17, [x1]
340 \\ cbnz w15, 0b
341 \\1:
342 \\ ret
343 :
344 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
345 : "w15", "w16", "w17", "memory"
346 );
347 unreachable;
348}
349fn __aarch64_ldeor1_rel() align(16) callconv(.Naked) void {
350 @setRuntimeSafety(false);
351 asm volatile (
352 \\ cbz w16, 8f
353 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
354 \\ ret
355 \\8:
356 \\ mov w16, w0
357 \\0:
358 \\ ldxrb w0, [x1]
359 \\ eor w17, w0, w16
360 \\ stlxrb w15, w17, [x1]
361 \\ cbnz w15, 0b
362 \\1:
363 \\ ret
364 :
365 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
366 : "w15", "w16", "w17", "memory"
367 );
368 unreachable;
369}
370fn __aarch64_ldset1_rel() align(16) callconv(.Naked) void {
371 @setRuntimeSafety(false);
372 asm volatile (
373 \\ cbz w16, 8f
374 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
375 \\ ret
376 \\8:
377 \\ mov w16, w0
378 \\0:
379 \\ ldxrb w0, [x1]
380 \\ orr w17, w0, w16
381 \\ stlxrb w15, w17, [x1]
382 \\ cbnz w15, 0b
383 \\1:
384 \\ ret
385 :
386 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
387 : "w15", "w16", "w17", "memory"
388 );
389 unreachable;
390}
391fn __aarch64_cas1_acq_rel() align(16) callconv(.Naked) void {
392 @setRuntimeSafety(false);
393 asm volatile (
394 \\ cbz w16, 8f
395 \\ .inst 0x08a07c41 + 0x00000000 + 0x408000
396 \\ ret
397 \\8:
398 \\ uxtb w16, w0
399 \\0:
400 \\ ldaxrb w0, [x2]
401 \\ cmp w0, w16
402 \\ bne 1f
403 \\ stlxrb w17, w1, [x2]
404 \\ cbnz w17, 0b
405 \\1:
406 \\ ret
407 :
408 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
409 : "w15", "w16", "w17", "memory"
410 );
411 unreachable;
412}
413fn __aarch64_swp1_acq_rel() align(16) callconv(.Naked) void {
414 @setRuntimeSafety(false);
415 asm volatile (
416 \\ cbz w16, 8f
417 \\ .inst 0x38208020 + 0x00000000 + 0xc00000
418 \\ ret
419 \\8:
420 \\ mov w16, w0
421 \\0:
422 \\ ldaxrb w0, [x1]
423 \\ stlxrb w17, w16, [x1]
424 \\ cbnz w17, 0b
425 \\1:
426 \\ ret
427 :
428 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
429 : "w15", "w16", "w17", "memory"
430 );
431 unreachable;
432}
433fn __aarch64_ldadd1_acq_rel() align(16) callconv(.Naked) void {
434 @setRuntimeSafety(false);
435 asm volatile (
436 \\ cbz w16, 8f
437 \\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
438 \\ ret
439 \\8:
440 \\ mov w16, w0
441 \\0:
442 \\ ldaxrb w0, [x1]
443 \\ add w17, w0, w16
444 \\ stlxrb w15, w17, [x1]
445 \\ cbnz w15, 0b
446 \\1:
447 \\ ret
448 :
449 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
450 : "w15", "w16", "w17", "memory"
451 );
452 unreachable;
453}
454fn __aarch64_ldclr1_acq_rel() align(16) callconv(.Naked) void {
455 @setRuntimeSafety(false);
456 asm volatile (
457 \\ cbz w16, 8f
458 \\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
459 \\ ret
460 \\8:
461 \\ mov w16, w0
462 \\0:
463 \\ ldaxrb w0, [x1]
464 \\ bic w17, w0, w16
465 \\ stlxrb w15, w17, [x1]
466 \\ cbnz w15, 0b
467 \\1:
468 \\ ret
469 :
470 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
471 : "w15", "w16", "w17", "memory"
472 );
473 unreachable;
474}
475fn __aarch64_ldeor1_acq_rel() align(16) callconv(.Naked) void {
476 @setRuntimeSafety(false);
477 asm volatile (
478 \\ cbz w16, 8f
479 \\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
480 \\ ret
481 \\8:
482 \\ mov w16, w0
483 \\0:
484 \\ ldaxrb w0, [x1]
485 \\ eor w17, w0, w16
486 \\ stlxrb w15, w17, [x1]
487 \\ cbnz w15, 0b
488 \\1:
489 \\ ret
490 :
491 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
492 : "w15", "w16", "w17", "memory"
493 );
494 unreachable;
495}
496fn __aarch64_ldset1_acq_rel() align(16) callconv(.Naked) void {
497 @setRuntimeSafety(false);
498 asm volatile (
499 \\ cbz w16, 8f
500 \\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
501 \\ ret
502 \\8:
503 \\ mov w16, w0
504 \\0:
505 \\ ldaxrb w0, [x1]
506 \\ orr w17, w0, w16
507 \\ stlxrb w15, w17, [x1]
508 \\ cbnz w15, 0b
509 \\1:
510 \\ ret
511 :
512 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
513 : "w15", "w16", "w17", "memory"
514 );
515 unreachable;
516}
517fn __aarch64_cas2_relax() align(16) callconv(.Naked) void {
518 @setRuntimeSafety(false);
519 asm volatile (
520 \\ cbz w16, 8f
521 \\ .inst 0x08a07c41 + 0x40000000 + 0x000000
522 \\ ret
523 \\8:
524 \\ uxth w16, w0
525 \\0:
526 \\ ldxrh w0, [x2]
527 \\ cmp w0, w16
528 \\ bne 1f
529 \\ stxrh w17, w1, [x2]
530 \\ cbnz w17, 0b
531 \\1:
532 \\ ret
533 :
534 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
535 : "w15", "w16", "w17", "memory"
536 );
537 unreachable;
538}
539fn __aarch64_swp2_relax() align(16) callconv(.Naked) void {
540 @setRuntimeSafety(false);
541 asm volatile (
542 \\ cbz w16, 8f
543 \\ .inst 0x38208020 + 0x40000000 + 0x000000
544 \\ ret
545 \\8:
546 \\ mov w16, w0
547 \\0:
548 \\ ldxrh w0, [x1]
549 \\ stxrh w17, w16, [x1]
550 \\ cbnz w17, 0b
551 \\1:
552 \\ ret
553 :
554 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
555 : "w15", "w16", "w17", "memory"
556 );
557 unreachable;
558}
559fn __aarch64_ldadd2_relax() align(16) callconv(.Naked) void {
560 @setRuntimeSafety(false);
561 asm volatile (
562 \\ cbz w16, 8f
563 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
564 \\ ret
565 \\8:
566 \\ mov w16, w0
567 \\0:
568 \\ ldxrh w0, [x1]
569 \\ add w17, w0, w16
570 \\ stxrh w15, w17, [x1]
571 \\ cbnz w15, 0b
572 \\1:
573 \\ ret
574 :
575 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
576 : "w15", "w16", "w17", "memory"
577 );
578 unreachable;
579}
580fn __aarch64_ldclr2_relax() align(16) callconv(.Naked) void {
581 @setRuntimeSafety(false);
582 asm volatile (
583 \\ cbz w16, 8f
584 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
585 \\ ret
586 \\8:
587 \\ mov w16, w0
588 \\0:
589 \\ ldxrh w0, [x1]
590 \\ bic w17, w0, w16
591 \\ stxrh w15, w17, [x1]
592 \\ cbnz w15, 0b
593 \\1:
594 \\ ret
595 :
596 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
597 : "w15", "w16", "w17", "memory"
598 );
599 unreachable;
600}
601fn __aarch64_ldeor2_relax() align(16) callconv(.Naked) void {
602 @setRuntimeSafety(false);
603 asm volatile (
604 \\ cbz w16, 8f
605 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
606 \\ ret
607 \\8:
608 \\ mov w16, w0
609 \\0:
610 \\ ldxrh w0, [x1]
611 \\ eor w17, w0, w16
612 \\ stxrh w15, w17, [x1]
613 \\ cbnz w15, 0b
614 \\1:
615 \\ ret
616 :
617 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
618 : "w15", "w16", "w17", "memory"
619 );
620 unreachable;
621}
622fn __aarch64_ldset2_relax() align(16) callconv(.Naked) void {
623 @setRuntimeSafety(false);
624 asm volatile (
625 \\ cbz w16, 8f
626 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
627 \\ ret
628 \\8:
629 \\ mov w16, w0
630 \\0:
631 \\ ldxrh w0, [x1]
632 \\ orr w17, w0, w16
633 \\ stxrh w15, w17, [x1]
634 \\ cbnz w15, 0b
635 \\1:
636 \\ ret
637 :
638 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
639 : "w15", "w16", "w17", "memory"
640 );
641 unreachable;
642}
643fn __aarch64_cas2_acq() align(16) callconv(.Naked) void {
644 @setRuntimeSafety(false);
645 asm volatile (
646 \\ cbz w16, 8f
647 \\ .inst 0x08a07c41 + 0x40000000 + 0x400000
648 \\ ret
649 \\8:
650 \\ uxth w16, w0
651 \\0:
652 \\ ldaxrh w0, [x2]
653 \\ cmp w0, w16
654 \\ bne 1f
655 \\ stxrh w17, w1, [x2]
656 \\ cbnz w17, 0b
657 \\1:
658 \\ ret
659 :
660 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
661 : "w15", "w16", "w17", "memory"
662 );
663 unreachable;
664}
665fn __aarch64_swp2_acq() align(16) callconv(.Naked) void {
666 @setRuntimeSafety(false);
667 asm volatile (
668 \\ cbz w16, 8f
669 \\ .inst 0x38208020 + 0x40000000 + 0x800000
670 \\ ret
671 \\8:
672 \\ mov w16, w0
673 \\0:
674 \\ ldaxrh w0, [x1]
675 \\ stxrh w17, w16, [x1]
676 \\ cbnz w17, 0b
677 \\1:
678 \\ ret
679 :
680 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
681 : "w15", "w16", "w17", "memory"
682 );
683 unreachable;
684}
685fn __aarch64_ldadd2_acq() align(16) callconv(.Naked) void {
686 @setRuntimeSafety(false);
687 asm volatile (
688 \\ cbz w16, 8f
689 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
690 \\ ret
691 \\8:
692 \\ mov w16, w0
693 \\0:
694 \\ ldaxrh w0, [x1]
695 \\ add w17, w0, w16
696 \\ stxrh w15, w17, [x1]
697 \\ cbnz w15, 0b
698 \\1:
699 \\ ret
700 :
701 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
702 : "w15", "w16", "w17", "memory"
703 );
704 unreachable;
705}
706fn __aarch64_ldclr2_acq() align(16) callconv(.Naked) void {
707 @setRuntimeSafety(false);
708 asm volatile (
709 \\ cbz w16, 8f
710 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
711 \\ ret
712 \\8:
713 \\ mov w16, w0
714 \\0:
715 \\ ldaxrh w0, [x1]
716 \\ bic w17, w0, w16
717 \\ stxrh w15, w17, [x1]
718 \\ cbnz w15, 0b
719 \\1:
720 \\ ret
721 :
722 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
723 : "w15", "w16", "w17", "memory"
724 );
725 unreachable;
726}
727fn __aarch64_ldeor2_acq() align(16) callconv(.Naked) void {
728 @setRuntimeSafety(false);
729 asm volatile (
730 \\ cbz w16, 8f
731 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
732 \\ ret
733 \\8:
734 \\ mov w16, w0
735 \\0:
736 \\ ldaxrh w0, [x1]
737 \\ eor w17, w0, w16
738 \\ stxrh w15, w17, [x1]
739 \\ cbnz w15, 0b
740 \\1:
741 \\ ret
742 :
743 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
744 : "w15", "w16", "w17", "memory"
745 );
746 unreachable;
747}
748fn __aarch64_ldset2_acq() align(16) callconv(.Naked) void {
749 @setRuntimeSafety(false);
750 asm volatile (
751 \\ cbz w16, 8f
752 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
753 \\ ret
754 \\8:
755 \\ mov w16, w0
756 \\0:
757 \\ ldaxrh w0, [x1]
758 \\ orr w17, w0, w16
759 \\ stxrh w15, w17, [x1]
760 \\ cbnz w15, 0b
761 \\1:
762 \\ ret
763 :
764 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
765 : "w15", "w16", "w17", "memory"
766 );
767 unreachable;
768}
769fn __aarch64_cas2_rel() align(16) callconv(.Naked) void {
770 @setRuntimeSafety(false);
771 asm volatile (
772 \\ cbz w16, 8f
773 \\ .inst 0x08a07c41 + 0x40000000 + 0x008000
774 \\ ret
775 \\8:
776 \\ uxth w16, w0
777 \\0:
778 \\ ldxrh w0, [x2]
779 \\ cmp w0, w16
780 \\ bne 1f
781 \\ stlxrh w17, w1, [x2]
782 \\ cbnz w17, 0b
783 \\1:
784 \\ ret
785 :
786 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
787 : "w15", "w16", "w17", "memory"
788 );
789 unreachable;
790}
791fn __aarch64_swp2_rel() align(16) callconv(.Naked) void {
792 @setRuntimeSafety(false);
793 asm volatile (
794 \\ cbz w16, 8f
795 \\ .inst 0x38208020 + 0x40000000 + 0x400000
796 \\ ret
797 \\8:
798 \\ mov w16, w0
799 \\0:
800 \\ ldxrh w0, [x1]
801 \\ stlxrh w17, w16, [x1]
802 \\ cbnz w17, 0b
803 \\1:
804 \\ ret
805 :
806 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
807 : "w15", "w16", "w17", "memory"
808 );
809 unreachable;
810}
811fn __aarch64_ldadd2_rel() align(16) callconv(.Naked) void {
812 @setRuntimeSafety(false);
813 asm volatile (
814 \\ cbz w16, 8f
815 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
816 \\ ret
817 \\8:
818 \\ mov w16, w0
819 \\0:
820 \\ ldxrh w0, [x1]
821 \\ add w17, w0, w16
822 \\ stlxrh w15, w17, [x1]
823 \\ cbnz w15, 0b
824 \\1:
825 \\ ret
826 :
827 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
828 : "w15", "w16", "w17", "memory"
829 );
830 unreachable;
831}
832fn __aarch64_ldclr2_rel() align(16) callconv(.Naked) void {
833 @setRuntimeSafety(false);
834 asm volatile (
835 \\ cbz w16, 8f
836 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
837 \\ ret
838 \\8:
839 \\ mov w16, w0
840 \\0:
841 \\ ldxrh w0, [x1]
842 \\ bic w17, w0, w16
843 \\ stlxrh w15, w17, [x1]
844 \\ cbnz w15, 0b
845 \\1:
846 \\ ret
847 :
848 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
849 : "w15", "w16", "w17", "memory"
850 );
851 unreachable;
852}
853fn __aarch64_ldeor2_rel() align(16) callconv(.Naked) void {
854 @setRuntimeSafety(false);
855 asm volatile (
856 \\ cbz w16, 8f
857 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
858 \\ ret
859 \\8:
860 \\ mov w16, w0
861 \\0:
862 \\ ldxrh w0, [x1]
863 \\ eor w17, w0, w16
864 \\ stlxrh w15, w17, [x1]
865 \\ cbnz w15, 0b
866 \\1:
867 \\ ret
868 :
869 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
870 : "w15", "w16", "w17", "memory"
871 );
872 unreachable;
873}
874fn __aarch64_ldset2_rel() align(16) callconv(.Naked) void {
875 @setRuntimeSafety(false);
876 asm volatile (
877 \\ cbz w16, 8f
878 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
879 \\ ret
880 \\8:
881 \\ mov w16, w0
882 \\0:
883 \\ ldxrh w0, [x1]
884 \\ orr w17, w0, w16
885 \\ stlxrh w15, w17, [x1]
886 \\ cbnz w15, 0b
887 \\1:
888 \\ ret
889 :
890 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
891 : "w15", "w16", "w17", "memory"
892 );
893 unreachable;
894}
895fn __aarch64_cas2_acq_rel() align(16) callconv(.Naked) void {
896 @setRuntimeSafety(false);
897 asm volatile (
898 \\ cbz w16, 8f
899 \\ .inst 0x08a07c41 + 0x40000000 + 0x408000
900 \\ ret
901 \\8:
902 \\ uxth w16, w0
903 \\0:
904 \\ ldaxrh w0, [x2]
905 \\ cmp w0, w16
906 \\ bne 1f
907 \\ stlxrh w17, w1, [x2]
908 \\ cbnz w17, 0b
909 \\1:
910 \\ ret
911 :
912 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
913 : "w15", "w16", "w17", "memory"
914 );
915 unreachable;
916}
917fn __aarch64_swp2_acq_rel() align(16) callconv(.Naked) void {
918 @setRuntimeSafety(false);
919 asm volatile (
920 \\ cbz w16, 8f
921 \\ .inst 0x38208020 + 0x40000000 + 0xc00000
922 \\ ret
923 \\8:
924 \\ mov w16, w0
925 \\0:
926 \\ ldaxrh w0, [x1]
927 \\ stlxrh w17, w16, [x1]
928 \\ cbnz w17, 0b
929 \\1:
930 \\ ret
931 :
932 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
933 : "w15", "w16", "w17", "memory"
934 );
935 unreachable;
936}
937fn __aarch64_ldadd2_acq_rel() align(16) callconv(.Naked) void {
938 @setRuntimeSafety(false);
939 asm volatile (
940 \\ cbz w16, 8f
941 \\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
942 \\ ret
943 \\8:
944 \\ mov w16, w0
945 \\0:
946 \\ ldaxrh w0, [x1]
947 \\ add w17, w0, w16
948 \\ stlxrh w15, w17, [x1]
949 \\ cbnz w15, 0b
950 \\1:
951 \\ ret
952 :
953 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
954 : "w15", "w16", "w17", "memory"
955 );
956 unreachable;
957}
958fn __aarch64_ldclr2_acq_rel() align(16) callconv(.Naked) void {
959 @setRuntimeSafety(false);
960 asm volatile (
961 \\ cbz w16, 8f
962 \\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
963 \\ ret
964 \\8:
965 \\ mov w16, w0
966 \\0:
967 \\ ldaxrh w0, [x1]
968 \\ bic w17, w0, w16
969 \\ stlxrh w15, w17, [x1]
970 \\ cbnz w15, 0b
971 \\1:
972 \\ ret
973 :
974 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
975 : "w15", "w16", "w17", "memory"
976 );
977 unreachable;
978}
979fn __aarch64_ldeor2_acq_rel() align(16) callconv(.Naked) void {
980 @setRuntimeSafety(false);
981 asm volatile (
982 \\ cbz w16, 8f
983 \\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
984 \\ ret
985 \\8:
986 \\ mov w16, w0
987 \\0:
988 \\ ldaxrh w0, [x1]
989 \\ eor w17, w0, w16
990 \\ stlxrh w15, w17, [x1]
991 \\ cbnz w15, 0b
992 \\1:
993 \\ ret
994 :
995 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
996 : "w15", "w16", "w17", "memory"
997 );
998 unreachable;
999}
1000fn __aarch64_ldset2_acq_rel() align(16) callconv(.Naked) void {
1001 @setRuntimeSafety(false);
1002 asm volatile (
1003 \\ cbz w16, 8f
1004 \\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
1005 \\ ret
1006 \\8:
1007 \\ mov w16, w0
1008 \\0:
1009 \\ ldaxrh w0, [x1]
1010 \\ orr w17, w0, w16
1011 \\ stlxrh w15, w17, [x1]
1012 \\ cbnz w15, 0b
1013 \\1:
1014 \\ ret
1015 :
1016 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1017 : "w15", "w16", "w17", "memory"
1018 );
1019 unreachable;
1020}
1021fn __aarch64_cas4_relax() align(16) callconv(.Naked) void {
1022 @setRuntimeSafety(false);
1023 asm volatile (
1024 \\ cbz w16, 8f
1025 \\ .inst 0x08a07c41 + 0x80000000 + 0x000000
1026 \\ ret
1027 \\8:
1028 \\ mov w16, w0
1029 \\0:
1030 \\ ldxr w0, [x2]
1031 \\ cmp w0, w16
1032 \\ bne 1f
1033 \\ stxr w17, w1, [x2]
1034 \\ cbnz w17, 0b
1035 \\1:
1036 \\ ret
1037 :
1038 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1039 : "w15", "w16", "w17", "memory"
1040 );
1041 unreachable;
1042}
1043fn __aarch64_swp4_relax() align(16) callconv(.Naked) void {
1044 @setRuntimeSafety(false);
1045 asm volatile (
1046 \\ cbz w16, 8f
1047 \\ .inst 0x38208020 + 0x80000000 + 0x000000
1048 \\ ret
1049 \\8:
1050 \\ mov w16, w0
1051 \\0:
1052 \\ ldxr w0, [x1]
1053 \\ stxr w17, w16, [x1]
1054 \\ cbnz w17, 0b
1055 \\1:
1056 \\ ret
1057 :
1058 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1059 : "w15", "w16", "w17", "memory"
1060 );
1061 unreachable;
1062}
1063fn __aarch64_ldadd4_relax() align(16) callconv(.Naked) void {
1064 @setRuntimeSafety(false);
1065 asm volatile (
1066 \\ cbz w16, 8f
1067 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
1068 \\ ret
1069 \\8:
1070 \\ mov w16, w0
1071 \\0:
1072 \\ ldxr w0, [x1]
1073 \\ add w17, w0, w16
1074 \\ stxr w15, w17, [x1]
1075 \\ cbnz w15, 0b
1076 \\1:
1077 \\ ret
1078 :
1079 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1080 : "w15", "w16", "w17", "memory"
1081 );
1082 unreachable;
1083}
1084fn __aarch64_ldclr4_relax() align(16) callconv(.Naked) void {
1085 @setRuntimeSafety(false);
1086 asm volatile (
1087 \\ cbz w16, 8f
1088 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
1089 \\ ret
1090 \\8:
1091 \\ mov w16, w0
1092 \\0:
1093 \\ ldxr w0, [x1]
1094 \\ bic w17, w0, w16
1095 \\ stxr w15, w17, [x1]
1096 \\ cbnz w15, 0b
1097 \\1:
1098 \\ ret
1099 :
1100 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1101 : "w15", "w16", "w17", "memory"
1102 );
1103 unreachable;
1104}
1105fn __aarch64_ldeor4_relax() align(16) callconv(.Naked) void {
1106 @setRuntimeSafety(false);
1107 asm volatile (
1108 \\ cbz w16, 8f
1109 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
1110 \\ ret
1111 \\8:
1112 \\ mov w16, w0
1113 \\0:
1114 \\ ldxr w0, [x1]
1115 \\ eor w17, w0, w16
1116 \\ stxr w15, w17, [x1]
1117 \\ cbnz w15, 0b
1118 \\1:
1119 \\ ret
1120 :
1121 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1122 : "w15", "w16", "w17", "memory"
1123 );
1124 unreachable;
1125}
1126fn __aarch64_ldset4_relax() align(16) callconv(.Naked) void {
1127 @setRuntimeSafety(false);
1128 asm volatile (
1129 \\ cbz w16, 8f
1130 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
1131 \\ ret
1132 \\8:
1133 \\ mov w16, w0
1134 \\0:
1135 \\ ldxr w0, [x1]
1136 \\ orr w17, w0, w16
1137 \\ stxr w15, w17, [x1]
1138 \\ cbnz w15, 0b
1139 \\1:
1140 \\ ret
1141 :
1142 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1143 : "w15", "w16", "w17", "memory"
1144 );
1145 unreachable;
1146}
1147fn __aarch64_cas4_acq() align(16) callconv(.Naked) void {
1148 @setRuntimeSafety(false);
1149 asm volatile (
1150 \\ cbz w16, 8f
1151 \\ .inst 0x08a07c41 + 0x80000000 + 0x400000
1152 \\ ret
1153 \\8:
1154 \\ mov w16, w0
1155 \\0:
1156 \\ ldaxr w0, [x2]
1157 \\ cmp w0, w16
1158 \\ bne 1f
1159 \\ stxr w17, w1, [x2]
1160 \\ cbnz w17, 0b
1161 \\1:
1162 \\ ret
1163 :
1164 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1165 : "w15", "w16", "w17", "memory"
1166 );
1167 unreachable;
1168}
1169fn __aarch64_swp4_acq() align(16) callconv(.Naked) void {
1170 @setRuntimeSafety(false);
1171 asm volatile (
1172 \\ cbz w16, 8f
1173 \\ .inst 0x38208020 + 0x80000000 + 0x800000
1174 \\ ret
1175 \\8:
1176 \\ mov w16, w0
1177 \\0:
1178 \\ ldaxr w0, [x1]
1179 \\ stxr w17, w16, [x1]
1180 \\ cbnz w17, 0b
1181 \\1:
1182 \\ ret
1183 :
1184 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1185 : "w15", "w16", "w17", "memory"
1186 );
1187 unreachable;
1188}
1189fn __aarch64_ldadd4_acq() align(16) callconv(.Naked) void {
1190 @setRuntimeSafety(false);
1191 asm volatile (
1192 \\ cbz w16, 8f
1193 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
1194 \\ ret
1195 \\8:
1196 \\ mov w16, w0
1197 \\0:
1198 \\ ldaxr w0, [x1]
1199 \\ add w17, w0, w16
1200 \\ stxr w15, w17, [x1]
1201 \\ cbnz w15, 0b
1202 \\1:
1203 \\ ret
1204 :
1205 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1206 : "w15", "w16", "w17", "memory"
1207 );
1208 unreachable;
1209}
1210fn __aarch64_ldclr4_acq() align(16) callconv(.Naked) void {
1211 @setRuntimeSafety(false);
1212 asm volatile (
1213 \\ cbz w16, 8f
1214 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
1215 \\ ret
1216 \\8:
1217 \\ mov w16, w0
1218 \\0:
1219 \\ ldaxr w0, [x1]
1220 \\ bic w17, w0, w16
1221 \\ stxr w15, w17, [x1]
1222 \\ cbnz w15, 0b
1223 \\1:
1224 \\ ret
1225 :
1226 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1227 : "w15", "w16", "w17", "memory"
1228 );
1229 unreachable;
1230}
1231fn __aarch64_ldeor4_acq() align(16) callconv(.Naked) void {
1232 @setRuntimeSafety(false);
1233 asm volatile (
1234 \\ cbz w16, 8f
1235 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
1236 \\ ret
1237 \\8:
1238 \\ mov w16, w0
1239 \\0:
1240 \\ ldaxr w0, [x1]
1241 \\ eor w17, w0, w16
1242 \\ stxr w15, w17, [x1]
1243 \\ cbnz w15, 0b
1244 \\1:
1245 \\ ret
1246 :
1247 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1248 : "w15", "w16", "w17", "memory"
1249 );
1250 unreachable;
1251}
1252fn __aarch64_ldset4_acq() align(16) callconv(.Naked) void {
1253 @setRuntimeSafety(false);
1254 asm volatile (
1255 \\ cbz w16, 8f
1256 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
1257 \\ ret
1258 \\8:
1259 \\ mov w16, w0
1260 \\0:
1261 \\ ldaxr w0, [x1]
1262 \\ orr w17, w0, w16
1263 \\ stxr w15, w17, [x1]
1264 \\ cbnz w15, 0b
1265 \\1:
1266 \\ ret
1267 :
1268 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1269 : "w15", "w16", "w17", "memory"
1270 );
1271 unreachable;
1272}
1273fn __aarch64_cas4_rel() align(16) callconv(.Naked) void {
1274 @setRuntimeSafety(false);
1275 asm volatile (
1276 \\ cbz w16, 8f
1277 \\ .inst 0x08a07c41 + 0x80000000 + 0x008000
1278 \\ ret
1279 \\8:
1280 \\ mov w16, w0
1281 \\0:
1282 \\ ldxr w0, [x2]
1283 \\ cmp w0, w16
1284 \\ bne 1f
1285 \\ stlxr w17, w1, [x2]
1286 \\ cbnz w17, 0b
1287 \\1:
1288 \\ ret
1289 :
1290 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1291 : "w15", "w16", "w17", "memory"
1292 );
1293 unreachable;
1294}
1295fn __aarch64_swp4_rel() align(16) callconv(.Naked) void {
1296 @setRuntimeSafety(false);
1297 asm volatile (
1298 \\ cbz w16, 8f
1299 \\ .inst 0x38208020 + 0x80000000 + 0x400000
1300 \\ ret
1301 \\8:
1302 \\ mov w16, w0
1303 \\0:
1304 \\ ldxr w0, [x1]
1305 \\ stlxr w17, w16, [x1]
1306 \\ cbnz w17, 0b
1307 \\1:
1308 \\ ret
1309 :
1310 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1311 : "w15", "w16", "w17", "memory"
1312 );
1313 unreachable;
1314}
1315fn __aarch64_ldadd4_rel() align(16) callconv(.Naked) void {
1316 @setRuntimeSafety(false);
1317 asm volatile (
1318 \\ cbz w16, 8f
1319 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
1320 \\ ret
1321 \\8:
1322 \\ mov w16, w0
1323 \\0:
1324 \\ ldxr w0, [x1]
1325 \\ add w17, w0, w16
1326 \\ stlxr w15, w17, [x1]
1327 \\ cbnz w15, 0b
1328 \\1:
1329 \\ ret
1330 :
1331 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1332 : "w15", "w16", "w17", "memory"
1333 );
1334 unreachable;
1335}
1336fn __aarch64_ldclr4_rel() align(16) callconv(.Naked) void {
1337 @setRuntimeSafety(false);
1338 asm volatile (
1339 \\ cbz w16, 8f
1340 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
1341 \\ ret
1342 \\8:
1343 \\ mov w16, w0
1344 \\0:
1345 \\ ldxr w0, [x1]
1346 \\ bic w17, w0, w16
1347 \\ stlxr w15, w17, [x1]
1348 \\ cbnz w15, 0b
1349 \\1:
1350 \\ ret
1351 :
1352 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1353 : "w15", "w16", "w17", "memory"
1354 );
1355 unreachable;
1356}
1357fn __aarch64_ldeor4_rel() align(16) callconv(.Naked) void {
1358 @setRuntimeSafety(false);
1359 asm volatile (
1360 \\ cbz w16, 8f
1361 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
1362 \\ ret
1363 \\8:
1364 \\ mov w16, w0
1365 \\0:
1366 \\ ldxr w0, [x1]
1367 \\ eor w17, w0, w16
1368 \\ stlxr w15, w17, [x1]
1369 \\ cbnz w15, 0b
1370 \\1:
1371 \\ ret
1372 :
1373 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1374 : "w15", "w16", "w17", "memory"
1375 );
1376 unreachable;
1377}
1378fn __aarch64_ldset4_rel() align(16) callconv(.Naked) void {
1379 @setRuntimeSafety(false);
1380 asm volatile (
1381 \\ cbz w16, 8f
1382 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
1383 \\ ret
1384 \\8:
1385 \\ mov w16, w0
1386 \\0:
1387 \\ ldxr w0, [x1]
1388 \\ orr w17, w0, w16
1389 \\ stlxr w15, w17, [x1]
1390 \\ cbnz w15, 0b
1391 \\1:
1392 \\ ret
1393 :
1394 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1395 : "w15", "w16", "w17", "memory"
1396 );
1397 unreachable;
1398}
1399fn __aarch64_cas4_acq_rel() align(16) callconv(.Naked) void {
1400 @setRuntimeSafety(false);
1401 asm volatile (
1402 \\ cbz w16, 8f
1403 \\ .inst 0x08a07c41 + 0x80000000 + 0x408000
1404 \\ ret
1405 \\8:
1406 \\ mov w16, w0
1407 \\0:
1408 \\ ldaxr w0, [x2]
1409 \\ cmp w0, w16
1410 \\ bne 1f
1411 \\ stlxr w17, w1, [x2]
1412 \\ cbnz w17, 0b
1413 \\1:
1414 \\ ret
1415 :
1416 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1417 : "w15", "w16", "w17", "memory"
1418 );
1419 unreachable;
1420}
1421fn __aarch64_swp4_acq_rel() align(16) callconv(.Naked) void {
1422 @setRuntimeSafety(false);
1423 asm volatile (
1424 \\ cbz w16, 8f
1425 \\ .inst 0x38208020 + 0x80000000 + 0xc00000
1426 \\ ret
1427 \\8:
1428 \\ mov w16, w0
1429 \\0:
1430 \\ ldaxr w0, [x1]
1431 \\ stlxr w17, w16, [x1]
1432 \\ cbnz w17, 0b
1433 \\1:
1434 \\ ret
1435 :
1436 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1437 : "w15", "w16", "w17", "memory"
1438 );
1439 unreachable;
1440}
1441fn __aarch64_ldadd4_acq_rel() align(16) callconv(.Naked) void {
1442 @setRuntimeSafety(false);
1443 asm volatile (
1444 \\ cbz w16, 8f
1445 \\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
1446 \\ ret
1447 \\8:
1448 \\ mov w16, w0
1449 \\0:
1450 \\ ldaxr w0, [x1]
1451 \\ add w17, w0, w16
1452 \\ stlxr w15, w17, [x1]
1453 \\ cbnz w15, 0b
1454 \\1:
1455 \\ ret
1456 :
1457 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1458 : "w15", "w16", "w17", "memory"
1459 );
1460 unreachable;
1461}
1462fn __aarch64_ldclr4_acq_rel() align(16) callconv(.Naked) void {
1463 @setRuntimeSafety(false);
1464 asm volatile (
1465 \\ cbz w16, 8f
1466 \\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
1467 \\ ret
1468 \\8:
1469 \\ mov w16, w0
1470 \\0:
1471 \\ ldaxr w0, [x1]
1472 \\ bic w17, w0, w16
1473 \\ stlxr w15, w17, [x1]
1474 \\ cbnz w15, 0b
1475 \\1:
1476 \\ ret
1477 :
1478 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1479 : "w15", "w16", "w17", "memory"
1480 );
1481 unreachable;
1482}
1483fn __aarch64_ldeor4_acq_rel() align(16) callconv(.Naked) void {
1484 @setRuntimeSafety(false);
1485 asm volatile (
1486 \\ cbz w16, 8f
1487 \\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
1488 \\ ret
1489 \\8:
1490 \\ mov w16, w0
1491 \\0:
1492 \\ ldaxr w0, [x1]
1493 \\ eor w17, w0, w16
1494 \\ stlxr w15, w17, [x1]
1495 \\ cbnz w15, 0b
1496 \\1:
1497 \\ ret
1498 :
1499 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1500 : "w15", "w16", "w17", "memory"
1501 );
1502 unreachable;
1503}
1504fn __aarch64_ldset4_acq_rel() align(16) callconv(.Naked) void {
1505 @setRuntimeSafety(false);
1506 asm volatile (
1507 \\ cbz w16, 8f
1508 \\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
1509 \\ ret
1510 \\8:
1511 \\ mov w16, w0
1512 \\0:
1513 \\ ldaxr w0, [x1]
1514 \\ orr w17, w0, w16
1515 \\ stlxr w15, w17, [x1]
1516 \\ cbnz w15, 0b
1517 \\1:
1518 \\ ret
1519 :
1520 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1521 : "w15", "w16", "w17", "memory"
1522 );
1523 unreachable;
1524}
1525fn __aarch64_cas8_relax() align(16) callconv(.Naked) void {
1526 @setRuntimeSafety(false);
1527 asm volatile (
1528 \\ cbz w16, 8f
1529 \\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
1530 \\ ret
1531 \\8:
1532 \\ mov x16, x0
1533 \\0:
1534 \\ ldxr x0, [x2]
1535 \\ cmp x0, x16
1536 \\ bne 1f
1537 \\ stxr w17, x1, [x2]
1538 \\ cbnz w17, 0b
1539 \\1:
1540 \\ ret
1541 :
1542 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1543 : "w15", "w16", "w17", "memory"
1544 );
1545 unreachable;
1546}
1547fn __aarch64_swp8_relax() align(16) callconv(.Naked) void {
1548 @setRuntimeSafety(false);
1549 asm volatile (
1550 \\ cbz w16, 8f
1551 \\ .inst 0x38208020 + 0xc0000000 + 0x000000
1552 \\ ret
1553 \\8:
1554 \\ mov x16, x0
1555 \\0:
1556 \\ ldxr x0, [x1]
1557 \\ stxr w17, x16, [x1]
1558 \\ cbnz w17, 0b
1559 \\1:
1560 \\ ret
1561 :
1562 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1563 : "w15", "w16", "w17", "memory"
1564 );
1565 unreachable;
1566}
1567fn __aarch64_ldadd8_relax() align(16) callconv(.Naked) void {
1568 @setRuntimeSafety(false);
1569 asm volatile (
1570 \\ cbz w16, 8f
1571 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
1572 \\ ret
1573 \\8:
1574 \\ mov x16, x0
1575 \\0:
1576 \\ ldxr x0, [x1]
1577 \\ add x17, x0, x16
1578 \\ stxr w15, x17, [x1]
1579 \\ cbnz w15, 0b
1580 \\1:
1581 \\ ret
1582 :
1583 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1584 : "w15", "w16", "w17", "memory"
1585 );
1586 unreachable;
1587}
1588fn __aarch64_ldclr8_relax() align(16) callconv(.Naked) void {
1589 @setRuntimeSafety(false);
1590 asm volatile (
1591 \\ cbz w16, 8f
1592 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
1593 \\ ret
1594 \\8:
1595 \\ mov x16, x0
1596 \\0:
1597 \\ ldxr x0, [x1]
1598 \\ bic x17, x0, x16
1599 \\ stxr w15, x17, [x1]
1600 \\ cbnz w15, 0b
1601 \\1:
1602 \\ ret
1603 :
1604 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1605 : "w15", "w16", "w17", "memory"
1606 );
1607 unreachable;
1608}
1609fn __aarch64_ldeor8_relax() align(16) callconv(.Naked) void {
1610 @setRuntimeSafety(false);
1611 asm volatile (
1612 \\ cbz w16, 8f
1613 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
1614 \\ ret
1615 \\8:
1616 \\ mov x16, x0
1617 \\0:
1618 \\ ldxr x0, [x1]
1619 \\ eor x17, x0, x16
1620 \\ stxr w15, x17, [x1]
1621 \\ cbnz w15, 0b
1622 \\1:
1623 \\ ret
1624 :
1625 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1626 : "w15", "w16", "w17", "memory"
1627 );
1628 unreachable;
1629}
1630fn __aarch64_ldset8_relax() align(16) callconv(.Naked) void {
1631 @setRuntimeSafety(false);
1632 asm volatile (
1633 \\ cbz w16, 8f
1634 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
1635 \\ ret
1636 \\8:
1637 \\ mov x16, x0
1638 \\0:
1639 \\ ldxr x0, [x1]
1640 \\ orr x17, x0, x16
1641 \\ stxr w15, x17, [x1]
1642 \\ cbnz w15, 0b
1643 \\1:
1644 \\ ret
1645 :
1646 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1647 : "w15", "w16", "w17", "memory"
1648 );
1649 unreachable;
1650}
1651fn __aarch64_cas8_acq() align(16) callconv(.Naked) void {
1652 @setRuntimeSafety(false);
1653 asm volatile (
1654 \\ cbz w16, 8f
1655 \\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
1656 \\ ret
1657 \\8:
1658 \\ mov x16, x0
1659 \\0:
1660 \\ ldaxr x0, [x2]
1661 \\ cmp x0, x16
1662 \\ bne 1f
1663 \\ stxr w17, x1, [x2]
1664 \\ cbnz w17, 0b
1665 \\1:
1666 \\ ret
1667 :
1668 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1669 : "w15", "w16", "w17", "memory"
1670 );
1671 unreachable;
1672}
1673fn __aarch64_swp8_acq() align(16) callconv(.Naked) void {
1674 @setRuntimeSafety(false);
1675 asm volatile (
1676 \\ cbz w16, 8f
1677 \\ .inst 0x38208020 + 0xc0000000 + 0x800000
1678 \\ ret
1679 \\8:
1680 \\ mov x16, x0
1681 \\0:
1682 \\ ldaxr x0, [x1]
1683 \\ stxr w17, x16, [x1]
1684 \\ cbnz w17, 0b
1685 \\1:
1686 \\ ret
1687 :
1688 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1689 : "w15", "w16", "w17", "memory"
1690 );
1691 unreachable;
1692}
1693fn __aarch64_ldadd8_acq() align(16) callconv(.Naked) void {
1694 @setRuntimeSafety(false);
1695 asm volatile (
1696 \\ cbz w16, 8f
1697 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
1698 \\ ret
1699 \\8:
1700 \\ mov x16, x0
1701 \\0:
1702 \\ ldaxr x0, [x1]
1703 \\ add x17, x0, x16
1704 \\ stxr w15, x17, [x1]
1705 \\ cbnz w15, 0b
1706 \\1:
1707 \\ ret
1708 :
1709 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1710 : "w15", "w16", "w17", "memory"
1711 );
1712 unreachable;
1713}
1714fn __aarch64_ldclr8_acq() align(16) callconv(.Naked) void {
1715 @setRuntimeSafety(false);
1716 asm volatile (
1717 \\ cbz w16, 8f
1718 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
1719 \\ ret
1720 \\8:
1721 \\ mov x16, x0
1722 \\0:
1723 \\ ldaxr x0, [x1]
1724 \\ bic x17, x0, x16
1725 \\ stxr w15, x17, [x1]
1726 \\ cbnz w15, 0b
1727 \\1:
1728 \\ ret
1729 :
1730 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1731 : "w15", "w16", "w17", "memory"
1732 );
1733 unreachable;
1734}
1735fn __aarch64_ldeor8_acq() align(16) callconv(.Naked) void {
1736 @setRuntimeSafety(false);
1737 asm volatile (
1738 \\ cbz w16, 8f
1739 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
1740 \\ ret
1741 \\8:
1742 \\ mov x16, x0
1743 \\0:
1744 \\ ldaxr x0, [x1]
1745 \\ eor x17, x0, x16
1746 \\ stxr w15, x17, [x1]
1747 \\ cbnz w15, 0b
1748 \\1:
1749 \\ ret
1750 :
1751 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1752 : "w15", "w16", "w17", "memory"
1753 );
1754 unreachable;
1755}
1756fn __aarch64_ldset8_acq() align(16) callconv(.Naked) void {
1757 @setRuntimeSafety(false);
1758 asm volatile (
1759 \\ cbz w16, 8f
1760 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
1761 \\ ret
1762 \\8:
1763 \\ mov x16, x0
1764 \\0:
1765 \\ ldaxr x0, [x1]
1766 \\ orr x17, x0, x16
1767 \\ stxr w15, x17, [x1]
1768 \\ cbnz w15, 0b
1769 \\1:
1770 \\ ret
1771 :
1772 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1773 : "w15", "w16", "w17", "memory"
1774 );
1775 unreachable;
1776}
1777fn __aarch64_cas8_rel() align(16) callconv(.Naked) void {
1778 @setRuntimeSafety(false);
1779 asm volatile (
1780 \\ cbz w16, 8f
1781 \\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
1782 \\ ret
1783 \\8:
1784 \\ mov x16, x0
1785 \\0:
1786 \\ ldxr x0, [x2]
1787 \\ cmp x0, x16
1788 \\ bne 1f
1789 \\ stlxr w17, x1, [x2]
1790 \\ cbnz w17, 0b
1791 \\1:
1792 \\ ret
1793 :
1794 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1795 : "w15", "w16", "w17", "memory"
1796 );
1797 unreachable;
1798}
1799fn __aarch64_swp8_rel() align(16) callconv(.Naked) void {
1800 @setRuntimeSafety(false);
1801 asm volatile (
1802 \\ cbz w16, 8f
1803 \\ .inst 0x38208020 + 0xc0000000 + 0x400000
1804 \\ ret
1805 \\8:
1806 \\ mov x16, x0
1807 \\0:
1808 \\ ldxr x0, [x1]
1809 \\ stlxr w17, x16, [x1]
1810 \\ cbnz w17, 0b
1811 \\1:
1812 \\ ret
1813 :
1814 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1815 : "w15", "w16", "w17", "memory"
1816 );
1817 unreachable;
1818}
1819fn __aarch64_ldadd8_rel() align(16) callconv(.Naked) void {
1820 @setRuntimeSafety(false);
1821 asm volatile (
1822 \\ cbz w16, 8f
1823 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
1824 \\ ret
1825 \\8:
1826 \\ mov x16, x0
1827 \\0:
1828 \\ ldxr x0, [x1]
1829 \\ add x17, x0, x16
1830 \\ stlxr w15, x17, [x1]
1831 \\ cbnz w15, 0b
1832 \\1:
1833 \\ ret
1834 :
1835 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1836 : "w15", "w16", "w17", "memory"
1837 );
1838 unreachable;
1839}
1840fn __aarch64_ldclr8_rel() align(16) callconv(.Naked) void {
1841 @setRuntimeSafety(false);
1842 asm volatile (
1843 \\ cbz w16, 8f
1844 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
1845 \\ ret
1846 \\8:
1847 \\ mov x16, x0
1848 \\0:
1849 \\ ldxr x0, [x1]
1850 \\ bic x17, x0, x16
1851 \\ stlxr w15, x17, [x1]
1852 \\ cbnz w15, 0b
1853 \\1:
1854 \\ ret
1855 :
1856 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1857 : "w15", "w16", "w17", "memory"
1858 );
1859 unreachable;
1860}
1861fn __aarch64_ldeor8_rel() align(16) callconv(.Naked) void {
1862 @setRuntimeSafety(false);
1863 asm volatile (
1864 \\ cbz w16, 8f
1865 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
1866 \\ ret
1867 \\8:
1868 \\ mov x16, x0
1869 \\0:
1870 \\ ldxr x0, [x1]
1871 \\ eor x17, x0, x16
1872 \\ stlxr w15, x17, [x1]
1873 \\ cbnz w15, 0b
1874 \\1:
1875 \\ ret
1876 :
1877 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1878 : "w15", "w16", "w17", "memory"
1879 );
1880 unreachable;
1881}
1882fn __aarch64_ldset8_rel() align(16) callconv(.Naked) void {
1883 @setRuntimeSafety(false);
1884 asm volatile (
1885 \\ cbz w16, 8f
1886 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
1887 \\ ret
1888 \\8:
1889 \\ mov x16, x0
1890 \\0:
1891 \\ ldxr x0, [x1]
1892 \\ orr x17, x0, x16
1893 \\ stlxr w15, x17, [x1]
1894 \\ cbnz w15, 0b
1895 \\1:
1896 \\ ret
1897 :
1898 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1899 : "w15", "w16", "w17", "memory"
1900 );
1901 unreachable;
1902}
1903fn __aarch64_cas8_acq_rel() align(16) callconv(.Naked) void {
1904 @setRuntimeSafety(false);
1905 asm volatile (
1906 \\ cbz w16, 8f
1907 \\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
1908 \\ ret
1909 \\8:
1910 \\ mov x16, x0
1911 \\0:
1912 \\ ldaxr x0, [x2]
1913 \\ cmp x0, x16
1914 \\ bne 1f
1915 \\ stlxr w17, x1, [x2]
1916 \\ cbnz w17, 0b
1917 \\1:
1918 \\ ret
1919 :
1920 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1921 : "w15", "w16", "w17", "memory"
1922 );
1923 unreachable;
1924}
1925fn __aarch64_swp8_acq_rel() align(16) callconv(.Naked) void {
1926 @setRuntimeSafety(false);
1927 asm volatile (
1928 \\ cbz w16, 8f
1929 \\ .inst 0x38208020 + 0xc0000000 + 0xc00000
1930 \\ ret
1931 \\8:
1932 \\ mov x16, x0
1933 \\0:
1934 \\ ldaxr x0, [x1]
1935 \\ stlxr w17, x16, [x1]
1936 \\ cbnz w17, 0b
1937 \\1:
1938 \\ ret
1939 :
1940 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1941 : "w15", "w16", "w17", "memory"
1942 );
1943 unreachable;
1944}
1945fn __aarch64_ldadd8_acq_rel() align(16) callconv(.Naked) void {
1946 @setRuntimeSafety(false);
1947 asm volatile (
1948 \\ cbz w16, 8f
1949 \\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
1950 \\ ret
1951 \\8:
1952 \\ mov x16, x0
1953 \\0:
1954 \\ ldaxr x0, [x1]
1955 \\ add x17, x0, x16
1956 \\ stlxr w15, x17, [x1]
1957 \\ cbnz w15, 0b
1958 \\1:
1959 \\ ret
1960 :
1961 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1962 : "w15", "w16", "w17", "memory"
1963 );
1964 unreachable;
1965}
1966fn __aarch64_ldclr8_acq_rel() align(16) callconv(.Naked) void {
1967 @setRuntimeSafety(false);
1968 asm volatile (
1969 \\ cbz w16, 8f
1970 \\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
1971 \\ ret
1972 \\8:
1973 \\ mov x16, x0
1974 \\0:
1975 \\ ldaxr x0, [x1]
1976 \\ bic x17, x0, x16
1977 \\ stlxr w15, x17, [x1]
1978 \\ cbnz w15, 0b
1979 \\1:
1980 \\ ret
1981 :
1982 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1983 : "w15", "w16", "w17", "memory"
1984 );
1985 unreachable;
1986}
1987fn __aarch64_ldeor8_acq_rel() align(16) callconv(.Naked) void {
1988 @setRuntimeSafety(false);
1989 asm volatile (
1990 \\ cbz w16, 8f
1991 \\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
1992 \\ ret
1993 \\8:
1994 \\ mov x16, x0
1995 \\0:
1996 \\ ldaxr x0, [x1]
1997 \\ eor x17, x0, x16
1998 \\ stlxr w15, x17, [x1]
1999 \\ cbnz w15, 0b
2000 \\1:
2001 \\ ret
2002 :
2003 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2004 : "w15", "w16", "w17", "memory"
2005 );
2006 unreachable;
2007}
2008fn __aarch64_ldset8_acq_rel() align(16) callconv(.Naked) void {
2009 @setRuntimeSafety(false);
2010 asm volatile (
2011 \\ cbz w16, 8f
2012 \\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
2013 \\ ret
2014 \\8:
2015 \\ mov x16, x0
2016 \\0:
2017 \\ ldaxr x0, [x1]
2018 \\ orr x17, x0, x16
2019 \\ stlxr w15, x17, [x1]
2020 \\ cbnz w15, 0b
2021 \\1:
2022 \\ ret
2023 :
2024 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2025 : "w15", "w16", "w17", "memory"
2026 );
2027 unreachable;
2028}
2029fn __aarch64_cas16_relax() align(16) callconv(.Naked) void {
2030 @setRuntimeSafety(false);
2031 asm volatile (
2032 \\ cbz w16, 8f
2033 \\ .inst 0x48207c82 + 0x000000
2034 \\ ret
2035 \\8:
2036 \\ mov x16, x0
2037 \\ mov x17, x1
2038 \\0:
2039 \\ ldxp x0, x1, [x4]
2040 \\ cmp x0, x16
2041 \\ ccmp x1, x17, #0, eq
2042 \\ bne 1f
2043 \\ stxp w15, x2, x3, [x4]
2044 \\ cbnz w15, 0b
2045 \\1:
2046 \\ ret
2047 :
2048 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2049 : "w15", "w16", "w17", "memory"
2050 );
2051 unreachable;
2052}
2053fn __aarch64_cas16_acq() align(16) callconv(.Naked) void {
2054 @setRuntimeSafety(false);
2055 asm volatile (
2056 \\ cbz w16, 8f
2057 \\ .inst 0x48207c82 + 0x400000
2058 \\ ret
2059 \\8:
2060 \\ mov x16, x0
2061 \\ mov x17, x1
2062 \\0:
2063 \\ ldaxp x0, x1, [x4]
2064 \\ cmp x0, x16
2065 \\ ccmp x1, x17, #0, eq
2066 \\ bne 1f
2067 \\ stxp w15, x2, x3, [x4]
2068 \\ cbnz w15, 0b
2069 \\1:
2070 \\ ret
2071 :
2072 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2073 : "w15", "w16", "w17", "memory"
2074 );
2075 unreachable;
2076}
2077fn __aarch64_cas16_rel() align(16) callconv(.Naked) void {
2078 @setRuntimeSafety(false);
2079 asm volatile (
2080 \\ cbz w16, 8f
2081 \\ .inst 0x48207c82 + 0x008000
2082 \\ ret
2083 \\8:
2084 \\ mov x16, x0
2085 \\ mov x17, x1
2086 \\0:
2087 \\ ldxp x0, x1, [x4]
2088 \\ cmp x0, x16
2089 \\ ccmp x1, x17, #0, eq
2090 \\ bne 1f
2091 \\ stlxp w15, x2, x3, [x4]
2092 \\ cbnz w15, 0b
2093 \\1:
2094 \\ ret
2095 :
2096 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2097 : "w15", "w16", "w17", "memory"
2098 );
2099 unreachable;
2100}
2101fn __aarch64_cas16_acq_rel() align(16) callconv(.Naked) void {
2102 @setRuntimeSafety(false);
2103 asm volatile (
2104 \\ cbz w16, 8f
2105 \\ .inst 0x48207c82 + 0x408000
2106 \\ ret
2107 \\8:
2108 \\ mov x16, x0
2109 \\ mov x17, x1
2110 \\0:
2111 \\ ldaxp x0, x1, [x4]
2112 \\ cmp x0, x16
2113 \\ ccmp x1, x17, #0, eq
2114 \\ bne 1f
2115 \\ stlxp w15, x2, x3, [x4]
2116 \\ cbnz w15, 0b
2117 \\1:
2118 \\ ret
2119 :
2120 : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2121 : "w15", "w16", "w17", "memory"
2122 );
2123 unreachable;
2124}
2125
2126comptime {
2127 @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage });
2128 @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage });
2129 @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage });
2130 @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage });
2131 @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage });
2132 @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage });
2133 @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage });
2134 @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage });
2135 @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage });
2136 @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage });
2137 @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage });
2138 @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage });
2139 @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage });
2140 @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage });
2141 @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage });
2142 @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage });
2143 @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage });
2144 @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage });
2145 @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage });
2146 @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage });
2147 @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage });
2148 @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage });
2149 @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage });
2150 @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage });
2151 @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage });
2152 @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage });
2153 @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage });
2154 @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage });
2155 @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage });
2156 @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage });
2157 @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage });
2158 @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage });
2159 @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage });
2160 @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage });
2161 @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage });
2162 @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage });
2163 @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage });
2164 @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage });
2165 @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage });
2166 @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage });
2167 @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage });
2168 @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage });
2169 @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage });
2170 @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage });
2171 @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage });
2172 @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage });
2173 @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage });
2174 @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage });
2175 @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage });
2176 @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage });
2177 @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage });
2178 @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage });
2179 @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage });
2180 @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage });
2181 @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage });
2182 @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage });
2183 @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage });
2184 @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage });
2185 @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage });
2186 @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage });
2187 @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage });
2188 @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage });
2189 @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage });
2190 @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage });
2191 @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage });
2192 @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage });
2193 @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage });
2194 @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage });
2195 @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage });
2196 @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage });
2197 @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage });
2198 @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage });
2199 @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage });
2200 @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage });
2201 @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage });
2202 @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage });
2203 @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage });
2204 @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage });
2205 @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage });
2206 @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage });
2207 @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage });
2208 @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage });
2209 @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage });
2210 @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage });
2211 @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage });
2212 @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage });
2213 @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage });
2214 @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage });
2215 @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage });
2216 @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage });
2217 @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage });
2218 @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage });
2219 @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage });
2220 @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage });
2221 @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage });
2222 @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage });
2223 @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage });
2224 @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage });
2225 @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage });
2226 @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage });
2227}
tools/gen_outline_atomics.zig created+338
...@@ -0,0 +1,338 @@
1const std = @import("std");
2const Allocator = std.mem.Allocator;
3
4const AtomicOp = enum {
5 cas,
6 swp,
7 ldadd,
8 ldclr,
9 ldeor,
10 ldset,
11};
12
13pub fn main() !void {
14 var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);
15 defer arena_instance.deinit();
16 const arena = arena_instance.allocator();
17
18 //const args = try std.process.argsAlloc(arena);
19
20 var bw = std.io.bufferedWriter(std.io.getStdOut().writer());
21 const w = bw.writer();
22
23 try w.writeAll(
24 \\//! This file is generated by tools/gen_outline_atomics.zig.
25 \\const builtin = @import("builtin");
26 \\const std = @import("std");
27 \\const linkage = @import("./common.zig").linkage;
28 \\const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .lse);
29 \\
30 \\/// This default is overridden at runtime after inspecting CPU properties.
31 \\/// It is intentionally not exported in order to make the machine code that
32 \\/// uses it a statically predicted direct branch rather than using the PLT,
33 \\/// which ARM is concerned would have too much overhead.
34 \\var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);
35 \\
36 \\
37 );
38
39 var footer = std.ArrayList(u8).init(arena);
40 try footer.appendSlice("\ncomptime {\n");
41
42 for ([_]N{ .one, .two, .four, .eight, .sixteen }) |n| {
43 for ([_]Ordering{ .relax, .acq, .rel, .acq_rel }) |order| {
44 for ([_]AtomicOp{ .cas, .swp, .ldadd, .ldclr, .ldeor, .ldset }) |op| {
45 if (n == .sixteen and op != .cas) continue;
46
47 const name = try std.fmt.allocPrint(arena, "__aarch64_{s}{d}_{s}", .{
48 @tagName(op), n.toBytes(), @tagName(order),
49 });
50 try writeFunction(arena, w, name, op, n, order);
51 try footer.writer().print(" @export({s}, .{{ .name = \"{s}\", .linkage = linkage }});\n", .{
52 name, name,
53 });
54 }
55 }
56 }
57
58 try w.writeAll(footer.items);
59 try w.writeAll("}\n");
60 try bw.flush();
61}
62
63fn writeFunction(
64 arena: Allocator,
65 w: anytype,
66 name: []const u8,
67 op: AtomicOp,
68 n: N,
69 order: Ordering,
70) !void {
71 const body = switch (op) {
72 .cas => try generateCas(arena, n, order),
73 .swp => try generateSwp(arena, n, order),
74 .ldadd => try generateLd(arena, n, order, .ldadd),
75 .ldclr => try generateLd(arena, n, order, .ldclr),
76 .ldeor => try generateLd(arena, n, order, .ldeor),
77 .ldset => try generateLd(arena, n, order, .ldset),
78 };
79 const fn_sig = try std.fmt.allocPrint(
80 arena,
81 "fn {[name]s}() align(16) callconv(.Naked) void {{",
82 .{ .name = name },
83 );
84 try w.writeAll(fn_sig);
85 try w.writeAll(
86 \\
87 \\ @setRuntimeSafety(false);
88 \\ asm volatile (
89 \\
90 );
91 var iter = std.mem.split(u8, body, "\n");
92 while (iter.next()) |line| {
93 try w.writeAll(" \\\\");
94 try w.writeAll(line);
95 try w.writeAll("\n");
96 }
97 try w.writeAll(
98 \\ :
99 \\ : [__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
100 \\ : "w15", "w16", "w17", "memory"
101 \\ );
102 \\ unreachable;
103 \\}
104 \\
105 );
106}
107
108const N = enum(u8) {
109 one = 1,
110 two = 2,
111 four = 4,
112 eight = 8,
113 sixteen = 16,
114
115 const Defines = struct {
116 s: []const u8,
117 uxt: []const u8,
118 b: []const u8,
119 };
120
121 fn defines(n: N) Defines {
122 const s = switch (n) {
123 .one => "b",
124 .two => "h",
125 else => "",
126 };
127 const uxt = switch (n) {
128 .one => "uxtb",
129 .two => "uxth",
130 .four, .eight, .sixteen => "mov",
131 };
132 const b = switch (n) {
133 .one => "0x00000000",
134 .two => "0x40000000",
135 .four => "0x80000000",
136 .eight => "0xc0000000",
137 else => "0x00000000",
138 };
139 return Defines{
140 .s = s,
141 .uxt = uxt,
142 .b = b,
143 };
144 }
145
146 fn register(n: N) []const u8 {
147 return if (@enumToInt(n) < 8) "w" else "x";
148 }
149
150 fn toBytes(n: N) u8 {
151 return @enumToInt(n);
152 }
153
154 fn toBits(n: N) u8 {
155 return n.toBytes() * 8;
156 }
157};
158
159const Ordering = enum {
160 relax,
161 acq,
162 rel,
163 acq_rel,
164
165 const Defines = struct {
166 suff: []const u8,
167 a: []const u8,
168 l: []const u8,
169 m: []const u8,
170 n: []const u8,
171 };
172 fn defines(self: @This()) Defines {
173 const suff = switch (self) {
174 .relax => "_relax",
175 .acq => "_acq",
176 .rel => "_rel",
177 .acq_rel => "_acq_rel",
178 };
179 const a = switch (self) {
180 .relax => "",
181 .acq => "a",
182 .rel => "",
183 .acq_rel => "a",
184 };
185 const l = switch (self) {
186 .relax => "",
187 .acq => "",
188 .rel => "l",
189 .acq_rel => "l",
190 };
191 const m = switch (self) {
192 .relax => "0x000000",
193 .acq => "0x400000",
194 .rel => "0x008000",
195 .acq_rel => "0x408000",
196 };
197 const n = switch (self) {
198 .relax => "0x000000",
199 .acq => "0x800000",
200 .rel => "0x400000",
201 .acq_rel => "0xc00000",
202 };
203 return .{ .suff = suff, .a = a, .l = l, .m = m, .n = n };
204 }
205};
206
207const LdName = enum { ldadd, ldclr, ldeor, ldset };
208
209fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
210 const s_def = n.defines();
211 const o_def = order.defines();
212
213 const reg = n.register();
214
215 if (@enumToInt(n) < 16) {
216 const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}", .{ s_def.b, o_def.m });
217 const ldxr = try std.fmt.allocPrint(arena, "ld{s}xr{s}", .{ o_def.a, s_def.s });
218 const stxr = try std.fmt.allocPrint(arena, "st{s}xr{s}", .{ o_def.l, s_def.s });
219
220 return try std.fmt.allocPrint(arena,
221 \\ cbz w16, 8f
222 \\ {[cas]s}
223 \\ ret
224 \\8:
225 \\ {[uxt]s} {[reg]s}16, {[reg]s}0
226 \\0:
227 \\ {[ldxr]s} {[reg]s}0, [x2]
228 \\ cmp {[reg]s}0, {[reg]s}16
229 \\ bne 1f
230 \\ {[stxr]s} w17, {[reg]s}1, [x2]
231 \\ cbnz w17, 0b
232 \\1:
233 \\ ret
234 , .{
235 .cas = cas,
236 .uxt = s_def.uxt,
237 .ldxr = ldxr,
238 .stxr = stxr,
239 .reg = reg,
240 });
241 } else {
242 const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}", .{o_def.m});
243 const ldxp = try std.fmt.allocPrint(arena, "ld{s}xp", .{o_def.a});
244 const stxp = try std.fmt.allocPrint(arena, "st{s}xp", .{o_def.l});
245
246 return try std.fmt.allocPrint(arena,
247 \\ cbz w16, 8f
248 \\ {[casp]s}
249 \\ ret
250 \\8:
251 \\ mov x16, x0
252 \\ mov x17, x1
253 \\0:
254 \\ {[ldxp]s} x0, x1, [x4]
255 \\ cmp x0, x16
256 \\ ccmp x1, x17, #0, eq
257 \\ bne 1f
258 \\ {[stxp]s} w15, x2, x3, [x4]
259 \\ cbnz w15, 0b
260 \\1:
261 \\ ret
262 , .{
263 .casp = casp,
264 .ldxp = ldxp,
265 .stxp = stxp,
266 });
267 }
268}
269
270fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {
271 const s_def = n.defines();
272 const o_def = order.defines();
273 const reg = n.register();
274
275 return try std.fmt.allocPrint(arena,
276 \\ cbz w16, 8f
277 \\ .inst 0x38208020 + {[b]s} + {[n]s}
278 \\ ret
279 \\8:
280 \\ mov {[reg]s}16, {[reg]s}0
281 \\0:
282 \\ ld{[a]s}xr{[s]s} {[reg]s}0, [x1]
283 \\ st{[l]s}xr{[s]s} w17, {[reg]s}16, [x1]
284 \\ cbnz w17, 0b
285 \\1:
286 \\ ret
287 , .{
288 .b = s_def.b,
289 .n = o_def.n,
290 .reg = reg,
291 .s = s_def.s,
292 .a = o_def.a,
293 .l = o_def.l,
294 });
295}
296
297fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {
298 const s_def = n.defines();
299 const o_def = order.defines();
300 const op = switch (ld) {
301 .ldadd => "add",
302 .ldclr => "bic",
303 .ldeor => "eor",
304 .ldset => "orr",
305 };
306 const op_n = switch (ld) {
307 .ldadd => "0x0000",
308 .ldclr => "0x1000",
309 .ldeor => "0x2000",
310 .ldset => "0x3000",
311 };
312
313 const reg = n.register();
314
315 return try std.fmt.allocPrint(arena,
316 \\ cbz w16, 8f
317 \\ .inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}
318 \\ ret
319 \\8:
320 \\ mov {[reg]s}16, {[reg]s}0
321 \\0:
322 \\ ld{[a]s}xr{[s]s} {[reg]s}0, [x1]
323 \\ {[op]s} {[reg]s}17, {[reg]s}0, {[reg]s}16
324 \\ st{[l]s}xr{[s]s} w15, {[reg]s}17, [x1]
325 \\ cbnz w15, 0b
326 \\1:
327 \\ ret
328 , .{
329 .op_n = op_n,
330 .b = s_def.b,
331 .n = o_def.n,
332 .s = s_def.s,
333 .a = o_def.a,
334 .l = o_def.l,
335 .op = op,
336 .reg = reg,
337 });
338}