Commit 95ee8ab77da6b76729beee6fc6ffccb07c9a2167 | zig

author	Andrew Kelley <andrew@ziglang.org> 2022-11-21 12:36:26-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2022-11-21 17:17:02-07:00
log	95ee8ab77da6b76729beee6fc6ffccb07c9a2167
tree	c8b3b3565a384701ac7951a5ebddd7b4dc56c1aa
parent	a8f2d00ec461522ce090fcfc09c80f013a70c51d

simplify outline atomics

 * Rely on libSystem when targeting macOS.
 * Make tools/gen_outline_atomics.zig more idiomatic.
 * Remove the CPU detection / auxval checking from compiler_rt. This
   functionality belongs in a different component. Zig's compiler_rt
   must not rely on constructors. Instead it will export a symbol for
   setting the value, and start code can detect and activate it.
 * Remove the separate logic for inline assembly when the target does or
   does not have lse support. `.inst` works in both cases.

3 files changed, 875 insertions(+), 9266 deletions(-)

lib
- compiler_rt.zig
- compiler_rt
  - lse_atomics.zig
tools
- gen_outline_atomics.zig

lib/compiler_rt.zig+7-1

...	...	@@ -1,8 +1,14 @@
	1	const builtin = @import("builtin");
	2
1	3	pub const panic = @import("compiler_rt/common.zig").panic;
2	4
3	5	comptime {
4	6	_ = @import("compiler_rt/atomics.zig");
5		_ = @import("compiler_rt/lse_atomics.zig");
	7
	8	// macOS has these functions inside libSystem.
	9	if (builtin.cpu.arch.isAARCH64() and !builtin.os.tag.isDarwin()) {
	10	_ = @import("compiler_rt/lse_atomics.zig");
	11	}
6	12
7	13	_ = @import("compiler_rt/addf3.zig");
8	14	_ = @import("compiler_rt/addhf3.zig");

lib/compiler_rt/lse_atomics.zig+755-8923

...	...	@@ -1,23 +1,20 @@
1		const std = @import("std");
	1	//! This file is generated by tools/gen_outline_atomics.zig.
2	2	const builtin = @import("builtin");
3		const arch = builtin.cpu.arch;
4		const is_test = builtin.is_test;
5		const target = std.Target;
6		const os_tag = builtin.os.tag;
7		const is_darwin = target.Os.Tag.isDarwin(os_tag);
8		const has_lse = target.aarch64.featureSetHas(builtin.target.cpu.features, .lse);
9		const linkage = if (is_test)
10		std.builtin.GlobalLinkage.Internal
11		else
12		std.builtin.GlobalLinkage.Strong;
	3	const std = @import("std");
	4	const linkage = @import("./common.zig").linkage;
	5	const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .lse);
13	6
14		fn cas1RelaxDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
15		@setRuntimeSafety(false);
16		__init_aarch64_have_lse_atomics();
	7	/// This default is overridden at runtime after inspecting CPU properties.
	8	/// It is intentionally not exported in order to make the machine code that
	9	/// uses it a statically predicted direct branch rather than using the PLT,
	10	/// which ARM is concerned would have too much overhead.
	11	var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);
17	12
	13	fn __aarch64_cas1_relax(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	14	@setRuntimeSafety(false);
18	15	return asm volatile (
19	16	\\ cbz w16, 8f
20		\\ casb w0, w1, [x2]
	17	\\ .inst 0x08a07c41 + 0x00000000 + 0x000000
21	18	\\
22	19	\\ cbz wzr, 1f
23	20	\\8:
...	...	@@ -37,107 +34,81 @@ fn cas1RelaxDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
37	34	: "w15", "w16", "w17", "memory"
38	35	);
39	36	}
40		fn cas1RelaxDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	37	fn __aarch64_swp1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
41	38	@setRuntimeSafety(false);
42		__init_aarch64_have_lse_atomics();
43
44	39	return asm volatile (
45	40	\\ cbz w16, 8f
46		\\ .inst 0x08a07c41 + 0x00000000 + 0x000000
47		\\
	41	\\ .inst 0x38208020 + 0x00000000 + 0x000000
48	42	\\ cbz wzr, 1f
49	43	\\8:
50		\\ uxtb w16, w0
	44	\\ mov w16, w0
51	45	\\0:
52		\\ ldxrb w0, [x2]
53		\\ cmp w0, w16
54		\\ bne 1f
55		\\ stxrb w17, w1, [x2]
	46	\\ ldxrb w0, [x1]
	47	\\ stxrb w17, w16, [x1]
56	48	\\ cbnz w17, 0b
57	49	\\1:
58	50	: [ret] "={w0}" (-> u8),
59		: [expected] "{w0}" (expected),
60		[desired] "{w1}" (desired),
61		[ptr] "{x2}" (ptr),
	51	: [val] "{w0}" (val),
	52	[ptr] "{x1}" (ptr),
62	53	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
63	54	: "w15", "w16", "w17", "memory"
64	55	);
65	56	}
66		fn cas1RelaxNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	57	fn __aarch64_ldadd1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
67	58	@setRuntimeSafety(false);
68		__init_aarch64_have_lse_atomics();
69
70	59	return asm volatile (
71	60	\\ cbz w16, 8f
72		\\ casb w0, w1, [x2]
73		\\
	61	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
74	62	\\ cbz wzr, 1f
75	63	\\8:
76		\\ uxtb w16, w0
	64	\\ mov w16, w0
77	65	\\0:
78		\\ ldxrb w0, [x2]
79		\\ cmp w0, w16
80		\\ bne 1f
81		\\ stxrb w17, w1, [x2]
82		\\ cbnz w17, 0b
	66	\\ ldxrb w0, [x1]
	67	\\ add w17, w0, w16
	68	\\ stxrb w15, w17, [x1]
	69	\\ cbnz w15, 0b
83	70	\\1:
84	71	: [ret] "={w0}" (-> u8),
85		: [expected] "{w0}" (expected),
86		[desired] "{w1}" (desired),
87		[ptr] "{x2}" (ptr),
	72	: [val] "{w0}" (val),
	73	[ptr] "{x1}" (ptr),
88	74	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
89	75	: "w15", "w16", "w17", "memory"
90	76	);
91	77	}
92		fn cas1RelaxNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	78	fn __aarch64_ldclr1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
93	79	@setRuntimeSafety(false);
94		__init_aarch64_have_lse_atomics();
95
96	80	return asm volatile (
97	81	\\ cbz w16, 8f
98		\\ .inst 0x08a07c41 + 0x00000000 + 0x000000
99		\\
	82	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
100	83	\\ cbz wzr, 1f
101	84	\\8:
102		\\ uxtb w16, w0
	85	\\ mov w16, w0
103	86	\\0:
104		\\ ldxrb w0, [x2]
105		\\ cmp w0, w16
106		\\ bne 1f
107		\\ stxrb w17, w1, [x2]
108		\\ cbnz w17, 0b
	87	\\ ldxrb w0, [x1]
	88	\\ bic w17, w0, w16
	89	\\ stxrb w15, w17, [x1]
	90	\\ cbnz w15, 0b
109	91	\\1:
110	92	: [ret] "={w0}" (-> u8),
111		: [expected] "{w0}" (expected),
112		[desired] "{w1}" (desired),
113		[ptr] "{x2}" (ptr),
	93	: [val] "{w0}" (val),
	94	[ptr] "{x1}" (ptr),
114	95	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
115	96	: "w15", "w16", "w17", "memory"
116	97	);
117	98	}
118		const __aarch64_cas1_relax = if (is_darwin)
119		if (has_lse)
120		cas1RelaxDarwinLse
121		else
122		cas1RelaxDarwinNolse
123		else if (has_lse)
124		cas1RelaxNondarwinLse
125		else
126		cas1RelaxNondarwinNolse;
127		fn swp1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	99	fn __aarch64_ldeor1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
128	100	@setRuntimeSafety(false);
129		__init_aarch64_have_lse_atomics();
130
131	101	return asm volatile (
132	102	\\ cbz w16, 8f
133		\\ swpb w0, w0, [x1]
	103	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
134	104	\\ cbz wzr, 1f
135	105	\\8:
136	106	\\ mov w16, w0
137	107	\\0:
138	108	\\ ldxrb w0, [x1]
139		\\ stxrb w17, w16, [x1]
140		\\ cbnz w17, 0b
	109	\\ eor w17, w0, w16
	110	\\ stxrb w15, w17, [x1]
	111	\\ cbnz w15, 0b
141	112	\\1:
142	113	: [ret] "={w0}" (-> u8),
143	114	: [val] "{w0}" (val),
...	...	@@ -146,20 +117,19 @@ fn swp1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
146	117	: "w15", "w16", "w17", "memory"
147	118	);
148	119	}
149		fn swp1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	120	fn __aarch64_ldset1_relax(val: u8, ptr: *u8) callconv(.C) u8 {
150	121	@setRuntimeSafety(false);
151		__init_aarch64_have_lse_atomics();
152
153	122	return asm volatile (
154	123	\\ cbz w16, 8f
155		\\ .inst 0x38208020 + 0x00000000 + 0x000000
	124	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
156	125	\\ cbz wzr, 1f
157	126	\\8:
158	127	\\ mov w16, w0
159	128	\\0:
160	129	\\ ldxrb w0, [x1]
161		\\ stxrb w17, w16, [x1]
162		\\ cbnz w17, 0b
	130	\\ orr w17, w0, w16
	131	\\ stxrb w15, w17, [x1]
	132	\\ cbnz w15, 0b
163	133	\\1:
164	134	: [ret] "={w0}" (-> u8),
165	135	: [val] "{w0}" (val),
...	...	@@ -168,40 +138,40 @@ fn swp1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
168	138	: "w15", "w16", "w17", "memory"
169	139	);
170	140	}
171		fn swp1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	141	fn __aarch64_cas1_acq(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
172	142	@setRuntimeSafety(false);
173		__init_aarch64_have_lse_atomics();
174
175	143	return asm volatile (
176	144	\\ cbz w16, 8f
177		\\ swpb w0, w0, [x1]
	145	\\ .inst 0x08a07c41 + 0x00000000 + 0x400000
	146	\\
178	147	\\ cbz wzr, 1f
179	148	\\8:
180		\\ mov w16, w0
	149	\\ uxtb w16, w0
181	150	\\0:
182		\\ ldxrb w0, [x1]
183		\\ stxrb w17, w16, [x1]
	151	\\ ldaxrb w0, [x2]
	152	\\ cmp w0, w16
	153	\\ bne 1f
	154	\\ stxrb w17, w1, [x2]
184	155	\\ cbnz w17, 0b
185	156	\\1:
186	157	: [ret] "={w0}" (-> u8),
187		: [val] "{w0}" (val),
188		[ptr] "{x1}" (ptr),
	158	: [expected] "{w0}" (expected),
	159	[desired] "{w1}" (desired),
	160	[ptr] "{x2}" (ptr),
189	161	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
190	162	: "w15", "w16", "w17", "memory"
191	163	);
192	164	}
193		fn swp1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	165	fn __aarch64_swp1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
194	166	@setRuntimeSafety(false);
195		__init_aarch64_have_lse_atomics();
196
197	167	return asm volatile (
198	168	\\ cbz w16, 8f
199		\\ .inst 0x38208020 + 0x00000000 + 0x000000
	169	\\ .inst 0x38208020 + 0x00000000 + 0x800000
200	170	\\ cbz wzr, 1f
201	171	\\8:
202	172	\\ mov w16, w0
203	173	\\0:
204		\\ ldxrb w0, [x1]
	174	\\ ldaxrb w0, [x1]
205	175	\\ stxrb w17, w16, [x1]
206	176	\\ cbnz w17, 0b
207	177	\\1:
...	...	@@ -212,27 +182,16 @@ fn swp1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
212	182	: "w15", "w16", "w17", "memory"
213	183	);
214	184	}
215		const __aarch64_swp1_relax = if (is_darwin)
216		if (has_lse)
217		swp1RelaxDarwinLse
218		else
219		swp1RelaxDarwinNolse
220		else if (has_lse)
221		swp1RelaxNondarwinLse
222		else
223		swp1RelaxNondarwinNolse;
224		fn ldadd1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	185	fn __aarch64_ldadd1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
225	186	@setRuntimeSafety(false);
226		__init_aarch64_have_lse_atomics();
227
228	187	return asm volatile (
229	188	\\ cbz w16, 8f
230		\\ ldaddb w0, w0, [x1]
	189	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
231	190	\\ cbz wzr, 1f
232	191	\\8:
233	192	\\ mov w16, w0
234	193	\\0:
235		\\ ldxrb w0, [x1]
	194	\\ ldaxrb w0, [x1]
236	195	\\ add w17, w0, w16
237	196	\\ stxrb w15, w17, [x1]
238	197	\\ cbnz w15, 0b
...	...	@@ -244,19 +203,17 @@ fn ldadd1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
244	203	: "w15", "w16", "w17", "memory"
245	204	);
246	205	}
247		fn ldadd1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	206	fn __aarch64_ldclr1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
248	207	@setRuntimeSafety(false);
249		__init_aarch64_have_lse_atomics();
250
251	208	return asm volatile (
252	209	\\ cbz w16, 8f
253		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
	210	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
254	211	\\ cbz wzr, 1f
255	212	\\8:
256	213	\\ mov w16, w0
257	214	\\0:
258		\\ ldxrb w0, [x1]
259		\\ add w17, w0, w16
	215	\\ ldaxrb w0, [x1]
	216	\\ bic w17, w0, w16
260	217	\\ stxrb w15, w17, [x1]
261	218	\\ cbnz w15, 0b
262	219	\\1:
...	...	@@ -267,19 +224,17 @@ fn ldadd1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
267	224	: "w15", "w16", "w17", "memory"
268	225	);
269	226	}
270		fn ldadd1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	227	fn __aarch64_ldeor1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
271	228	@setRuntimeSafety(false);
272		__init_aarch64_have_lse_atomics();
273
274	229	return asm volatile (
275	230	\\ cbz w16, 8f
276		\\ ldaddb w0, w0, [x1]
	231	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
277	232	\\ cbz wzr, 1f
278	233	\\8:
279	234	\\ mov w16, w0
280	235	\\0:
281		\\ ldxrb w0, [x1]
282		\\ add w17, w0, w16
	236	\\ ldaxrb w0, [x1]
	237	\\ eor w17, w0, w16
283	238	\\ stxrb w15, w17, [x1]
284	239	\\ cbnz w15, 0b
285	240	\\1:
...	...	@@ -290,19 +245,17 @@ fn ldadd1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
290	245	: "w15", "w16", "w17", "memory"
291	246	);
292	247	}
293		fn ldadd1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	248	fn __aarch64_ldset1_acq(val: u8, ptr: *u8) callconv(.C) u8 {
294	249	@setRuntimeSafety(false);
295		__init_aarch64_have_lse_atomics();
296
297	250	return asm volatile (
298	251	\\ cbz w16, 8f
299		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x000000
	252	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
300	253	\\ cbz wzr, 1f
301	254	\\8:
302	255	\\ mov w16, w0
303	256	\\0:
304		\\ ldxrb w0, [x1]
305		\\ add w17, w0, w16
	257	\\ ldaxrb w0, [x1]
	258	\\ orr w17, w0, w16
306	259	\\ stxrb w15, w17, [x1]
307	260	\\ cbnz w15, 0b
308	261	\\1:
...	...	@@ -313,53 +266,42 @@ fn ldadd1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
313	266	: "w15", "w16", "w17", "memory"
314	267	);
315	268	}
316		const __aarch64_ldadd1_relax = if (is_darwin)
317		if (has_lse)
318		ldadd1RelaxDarwinLse
319		else
320		ldadd1RelaxDarwinNolse
321		else if (has_lse)
322		ldadd1RelaxNondarwinLse
323		else
324		ldadd1RelaxNondarwinNolse;
325		fn ldclr1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	269	fn __aarch64_cas1_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
326	270	@setRuntimeSafety(false);
327		__init_aarch64_have_lse_atomics();
328
329	271	return asm volatile (
330	272	\\ cbz w16, 8f
331		\\ ldclrb w0, w0, [x1]
	273	\\ .inst 0x08a07c41 + 0x00000000 + 0x008000
	274	\\
332	275	\\ cbz wzr, 1f
333	276	\\8:
334		\\ mov w16, w0
	277	\\ uxtb w16, w0
335	278	\\0:
336		\\ ldxrb w0, [x1]
337		\\ bic w17, w0, w16
338		\\ stxrb w15, w17, [x1]
339		\\ cbnz w15, 0b
	279	\\ ldxrb w0, [x2]
	280	\\ cmp w0, w16
	281	\\ bne 1f
	282	\\ stlxrb w17, w1, [x2]
	283	\\ cbnz w17, 0b
340	284	\\1:
341	285	: [ret] "={w0}" (-> u8),
342		: [val] "{w0}" (val),
343		[ptr] "{x1}" (ptr),
	286	: [expected] "{w0}" (expected),
	287	[desired] "{w1}" (desired),
	288	[ptr] "{x2}" (ptr),
344	289	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
345	290	: "w15", "w16", "w17", "memory"
346	291	);
347	292	}
348		fn ldclr1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	293	fn __aarch64_swp1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
349	294	@setRuntimeSafety(false);
350		__init_aarch64_have_lse_atomics();
351
352	295	return asm volatile (
353	296	\\ cbz w16, 8f
354		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
	297	\\ .inst 0x38208020 + 0x00000000 + 0x400000
355	298	\\ cbz wzr, 1f
356	299	\\8:
357	300	\\ mov w16, w0
358	301	\\0:
359	302	\\ ldxrb w0, [x1]
360		\\ bic w17, w0, w16
361		\\ stxrb w15, w17, [x1]
362		\\ cbnz w15, 0b
	303	\\ stlxrb w17, w16, [x1]
	304	\\ cbnz w17, 0b
363	305	\\1:
364	306	: [ret] "={w0}" (-> u8),
365	307	: [val] "{w0}" (val),
...	...	@@ -368,20 +310,18 @@ fn ldclr1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
368	310	: "w15", "w16", "w17", "memory"
369	311	);
370	312	}
371		fn ldclr1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	313	fn __aarch64_ldadd1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
372	314	@setRuntimeSafety(false);
373		__init_aarch64_have_lse_atomics();
374
375	315	return asm volatile (
376	316	\\ cbz w16, 8f
377		\\ ldclrb w0, w0, [x1]
	317	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
378	318	\\ cbz wzr, 1f
379	319	\\8:
380	320	\\ mov w16, w0
381	321	\\0:
382	322	\\ ldxrb w0, [x1]
383		\\ bic w17, w0, w16
384		\\ stxrb w15, w17, [x1]
	323	\\ add w17, w0, w16
	324	\\ stlxrb w15, w17, [x1]
385	325	\\ cbnz w15, 0b
386	326	\\1:
387	327	: [ret] "={w0}" (-> u8),
...	...	@@ -391,20 +331,18 @@ fn ldclr1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
391	331	: "w15", "w16", "w17", "memory"
392	332	);
393	333	}
394		fn ldclr1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	334	fn __aarch64_ldclr1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
395	335	@setRuntimeSafety(false);
396		__init_aarch64_have_lse_atomics();
397
398	336	return asm volatile (
399	337	\\ cbz w16, 8f
400		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x000000
	338	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
401	339	\\ cbz wzr, 1f
402	340	\\8:
403	341	\\ mov w16, w0
404	342	\\0:
405	343	\\ ldxrb w0, [x1]
406	344	\\ bic w17, w0, w16
407		\\ stxrb w15, w17, [x1]
	345	\\ stlxrb w15, w17, [x1]
408	346	\\ cbnz w15, 0b
409	347	\\1:
410	348	: [ret] "={w0}" (-> u8),
...	...	@@ -414,29 +352,18 @@ fn ldclr1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
414	352	: "w15", "w16", "w17", "memory"
415	353	);
416	354	}
417		const __aarch64_ldclr1_relax = if (is_darwin)
418		if (has_lse)
419		ldclr1RelaxDarwinLse
420		else
421		ldclr1RelaxDarwinNolse
422		else if (has_lse)
423		ldclr1RelaxNondarwinLse
424		else
425		ldclr1RelaxNondarwinNolse;
426		fn ldeor1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	355	fn __aarch64_ldeor1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
427	356	@setRuntimeSafety(false);
428		__init_aarch64_have_lse_atomics();
429
430	357	return asm volatile (
431	358	\\ cbz w16, 8f
432		\\ ldeorb w0, w0, [x1]
	359	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
433	360	\\ cbz wzr, 1f
434	361	\\8:
435	362	\\ mov w16, w0
436	363	\\0:
437	364	\\ ldxrb w0, [x1]
438	365	\\ eor w17, w0, w16
439		\\ stxrb w15, w17, [x1]
	366	\\ stlxrb w15, w17, [x1]
440	367	\\ cbnz w15, 0b
441	368	\\1:
442	369	: [ret] "={w0}" (-> u8),
...	...	@@ -446,20 +373,18 @@ fn ldeor1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
446	373	: "w15", "w16", "w17", "memory"
447	374	);
448	375	}
449		fn ldeor1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	376	fn __aarch64_ldset1_rel(val: u8, ptr: *u8) callconv(.C) u8 {
450	377	@setRuntimeSafety(false);
451		__init_aarch64_have_lse_atomics();
452
453	378	return asm volatile (
454	379	\\ cbz w16, 8f
455		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
	380	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
456	381	\\ cbz wzr, 1f
457	382	\\8:
458	383	\\ mov w16, w0
459	384	\\0:
460	385	\\ ldxrb w0, [x1]
461		\\ eor w17, w0, w16
462		\\ stxrb w15, w17, [x1]
	386	\\ orr w17, w0, w16
	387	\\ stlxrb w15, w17, [x1]
463	388	\\ cbnz w15, 0b
464	389	\\1:
465	390	: [ret] "={w0}" (-> u8),
...	...	@@ -469,44 +394,42 @@ fn ldeor1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
469	394	: "w15", "w16", "w17", "memory"
470	395	);
471	396	}
472		fn ldeor1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	397	fn __aarch64_cas1_acq_rel(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
473	398	@setRuntimeSafety(false);
474		__init_aarch64_have_lse_atomics();
475
476	399	return asm volatile (
477	400	\\ cbz w16, 8f
478		\\ ldeorb w0, w0, [x1]
	401	\\ .inst 0x08a07c41 + 0x00000000 + 0x408000
	402	\\
479	403	\\ cbz wzr, 1f
480	404	\\8:
481		\\ mov w16, w0
	405	\\ uxtb w16, w0
482	406	\\0:
483		\\ ldxrb w0, [x1]
484		\\ eor w17, w0, w16
485		\\ stxrb w15, w17, [x1]
486		\\ cbnz w15, 0b
	407	\\ ldaxrb w0, [x2]
	408	\\ cmp w0, w16
	409	\\ bne 1f
	410	\\ stlxrb w17, w1, [x2]
	411	\\ cbnz w17, 0b
487	412	\\1:
488	413	: [ret] "={w0}" (-> u8),
489		: [val] "{w0}" (val),
490		[ptr] "{x1}" (ptr),
	414	: [expected] "{w0}" (expected),
	415	[desired] "{w1}" (desired),
	416	[ptr] "{x2}" (ptr),
491	417	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
492	418	: "w15", "w16", "w17", "memory"
493	419	);
494	420	}
495		fn ldeor1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	421	fn __aarch64_swp1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
496	422	@setRuntimeSafety(false);
497		__init_aarch64_have_lse_atomics();
498
499	423	return asm volatile (
500	424	\\ cbz w16, 8f
501		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x000000
	425	\\ .inst 0x38208020 + 0x00000000 + 0xc00000
502	426	\\ cbz wzr, 1f
503	427	\\8:
504	428	\\ mov w16, w0
505	429	\\0:
506		\\ ldxrb w0, [x1]
507		\\ eor w17, w0, w16
508		\\ stxrb w15, w17, [x1]
509		\\ cbnz w15, 0b
	430	\\ ldaxrb w0, [x1]
	431	\\ stlxrb w17, w16, [x1]
	432	\\ cbnz w17, 0b
510	433	\\1:
511	434	: [ret] "={w0}" (-> u8),
512	435	: [val] "{w0}" (val),
...	...	@@ -515,29 +438,18 @@ fn ldeor1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
515	438	: "w15", "w16", "w17", "memory"
516	439	);
517	440	}
518		const __aarch64_ldeor1_relax = if (is_darwin)
519		if (has_lse)
520		ldeor1RelaxDarwinLse
521		else
522		ldeor1RelaxDarwinNolse
523		else if (has_lse)
524		ldeor1RelaxNondarwinLse
525		else
526		ldeor1RelaxNondarwinNolse;
527		fn ldset1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	441	fn __aarch64_ldadd1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
528	442	@setRuntimeSafety(false);
529		__init_aarch64_have_lse_atomics();
530
531	443	return asm volatile (
532	444	\\ cbz w16, 8f
533		\\ ldsetb w0, w0, [x1]
	445	\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
534	446	\\ cbz wzr, 1f
535	447	\\8:
536	448	\\ mov w16, w0
537	449	\\0:
538		\\ ldxrb w0, [x1]
539		\\ orr w17, w0, w16
540		\\ stxrb w15, w17, [x1]
	450	\\ ldaxrb w0, [x1]
	451	\\ add w17, w0, w16
	452	\\ stlxrb w15, w17, [x1]
541	453	\\ cbnz w15, 0b
542	454	\\1:
543	455	: [ret] "={w0}" (-> u8),
...	...	@@ -547,20 +459,18 @@ fn ldset1RelaxDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
547	459	: "w15", "w16", "w17", "memory"
548	460	);
549	461	}
550		fn ldset1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	462	fn __aarch64_ldclr1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
551	463	@setRuntimeSafety(false);
552		__init_aarch64_have_lse_atomics();
553
554	464	return asm volatile (
555	465	\\ cbz w16, 8f
556		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
	466	\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
557	467	\\ cbz wzr, 1f
558	468	\\8:
559	469	\\ mov w16, w0
560	470	\\0:
561		\\ ldxrb w0, [x1]
562		\\ orr w17, w0, w16
563		\\ stxrb w15, w17, [x1]
	471	\\ ldaxrb w0, [x1]
	472	\\ bic w17, w0, w16
	473	\\ stlxrb w15, w17, [x1]
564	474	\\ cbnz w15, 0b
565	475	\\1:
566	476	: [ret] "={w0}" (-> u8),
...	...	@@ -570,20 +480,18 @@ fn ldset1RelaxDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
570	480	: "w15", "w16", "w17", "memory"
571	481	);
572	482	}
573		fn ldset1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	483	fn __aarch64_ldeor1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
574	484	@setRuntimeSafety(false);
575		__init_aarch64_have_lse_atomics();
576
577	485	return asm volatile (
578	486	\\ cbz w16, 8f
579		\\ ldsetb w0, w0, [x1]
	487	\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
580	488	\\ cbz wzr, 1f
581	489	\\8:
582	490	\\ mov w16, w0
583	491	\\0:
584		\\ ldxrb w0, [x1]
585		\\ orr w17, w0, w16
586		\\ stxrb w15, w17, [x1]
	492	\\ ldaxrb w0, [x1]
	493	\\ eor w17, w0, w16
	494	\\ stlxrb w15, w17, [x1]
587	495	\\ cbnz w15, 0b
588	496	\\1:
589	497	: [ret] "={w0}" (-> u8),
...	...	@@ -593,20 +501,18 @@ fn ldset1RelaxNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
593	501	: "w15", "w16", "w17", "memory"
594	502	);
595	503	}
596		fn ldset1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	504	fn __aarch64_ldset1_acq_rel(val: u8, ptr: *u8) callconv(.C) u8 {
597	505	@setRuntimeSafety(false);
598		__init_aarch64_have_lse_atomics();
599
600	506	return asm volatile (
601	507	\\ cbz w16, 8f
602		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x000000
	508	\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
603	509	\\ cbz wzr, 1f
604	510	\\8:
605	511	\\ mov w16, w0
606	512	\\0:
607		\\ ldxrb w0, [x1]
	513	\\ ldaxrb w0, [x1]
608	514	\\ orr w17, w0, w16
609		\\ stxrb w15, w17, [x1]
	515	\\ stlxrb w15, w17, [x1]
610	516	\\ cbnz w15, 0b
611	517	\\1:
612	518	: [ret] "={w0}" (-> u8),
...	...	@@ -616,34 +522,23 @@ fn ldset1RelaxNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
616	522	: "w15", "w16", "w17", "memory"
617	523	);
618	524	}
619		const __aarch64_ldset1_relax = if (is_darwin)
620		if (has_lse)
621		ldset1RelaxDarwinLse
622		else
623		ldset1RelaxDarwinNolse
624		else if (has_lse)
625		ldset1RelaxNondarwinLse
626		else
627		ldset1RelaxNondarwinNolse;
628		fn cas1AcqDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	525	fn __aarch64_cas2_relax(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
629	526	@setRuntimeSafety(false);
630		__init_aarch64_have_lse_atomics();
631
632	527	return asm volatile (
633	528	\\ cbz w16, 8f
634		\\ casab w0, w1, [x2]
	529	\\ .inst 0x08a07c41 + 0x40000000 + 0x000000
635	530	\\
636	531	\\ cbz wzr, 1f
637	532	\\8:
638		\\ uxtb w16, w0
	533	\\ uxth w16, w0
639	534	\\0:
640		\\ ldaxrb w0, [x2]
	535	\\ ldxrh w0, [x2]
641	536	\\ cmp w0, w16
642	537	\\ bne 1f
643		\\ stxrb w17, w1, [x2]
	538	\\ stxrh w17, w1, [x2]
644	539	\\ cbnz w17, 0b
645	540	\\1:
646		: [ret] "={w0}" (-> u8),
	541	: [ret] "={w0}" (-> u16),
647	542	: [expected] "{w0}" (expected),
648	543	[desired] "{w1}" (desired),
649	544	[ptr] "{x2}" (ptr),
...	...	@@ -651,613 +546,511 @@ fn cas1AcqDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
651	546	: "w15", "w16", "w17", "memory"
652	547	);
653	548	}
654		fn cas1AcqDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	549	fn __aarch64_swp2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
655	550	@setRuntimeSafety(false);
656		__init_aarch64_have_lse_atomics();
657
658	551	return asm volatile (
659	552	\\ cbz w16, 8f
660		\\ .inst 0x08a07c41 + 0x00000000 + 0x400000
661		\\
	553	\\ .inst 0x38208020 + 0x40000000 + 0x000000
662	554	\\ cbz wzr, 1f
663	555	\\8:
664		\\ uxtb w16, w0
	556	\\ mov w16, w0
665	557	\\0:
666		\\ ldaxrb w0, [x2]
667		\\ cmp w0, w16
668		\\ bne 1f
669		\\ stxrb w17, w1, [x2]
	558	\\ ldxrh w0, [x1]
	559	\\ stxrh w17, w16, [x1]
670	560	\\ cbnz w17, 0b
671	561	\\1:
672		: [ret] "={w0}" (-> u8),
673		: [expected] "{w0}" (expected),
674		[desired] "{w1}" (desired),
675		[ptr] "{x2}" (ptr),
	562	: [ret] "={w0}" (-> u16),
	563	: [val] "{w0}" (val),
	564	[ptr] "{x1}" (ptr),
676	565	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
677	566	: "w15", "w16", "w17", "memory"
678	567	);
679	568	}
680		fn cas1AcqNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	569	fn __aarch64_ldadd2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
681	570	@setRuntimeSafety(false);
682		__init_aarch64_have_lse_atomics();
683
684	571	return asm volatile (
685	572	\\ cbz w16, 8f
686		\\ casab w0, w1, [x2]
687		\\
	573	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
688	574	\\ cbz wzr, 1f
689	575	\\8:
690		\\ uxtb w16, w0
	576	\\ mov w16, w0
691	577	\\0:
692		\\ ldaxrb w0, [x2]
693		\\ cmp w0, w16
694		\\ bne 1f
695		\\ stxrb w17, w1, [x2]
696		\\ cbnz w17, 0b
	578	\\ ldxrh w0, [x1]
	579	\\ add w17, w0, w16
	580	\\ stxrh w15, w17, [x1]
	581	\\ cbnz w15, 0b
697	582	\\1:
698		: [ret] "={w0}" (-> u8),
699		: [expected] "{w0}" (expected),
700		[desired] "{w1}" (desired),
701		[ptr] "{x2}" (ptr),
	583	: [ret] "={w0}" (-> u16),
	584	: [val] "{w0}" (val),
	585	[ptr] "{x1}" (ptr),
702	586	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
703	587	: "w15", "w16", "w17", "memory"
704	588	);
705	589	}
706		fn cas1AcqNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	590	fn __aarch64_ldclr2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
707	591	@setRuntimeSafety(false);
708		__init_aarch64_have_lse_atomics();
709
710	592	return asm volatile (
711	593	\\ cbz w16, 8f
712		\\ .inst 0x08a07c41 + 0x00000000 + 0x400000
713		\\
	594	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
714	595	\\ cbz wzr, 1f
715	596	\\8:
716		\\ uxtb w16, w0
	597	\\ mov w16, w0
717	598	\\0:
718		\\ ldaxrb w0, [x2]
719		\\ cmp w0, w16
720		\\ bne 1f
721		\\ stxrb w17, w1, [x2]
722		\\ cbnz w17, 0b
	599	\\ ldxrh w0, [x1]
	600	\\ bic w17, w0, w16
	601	\\ stxrh w15, w17, [x1]
	602	\\ cbnz w15, 0b
723	603	\\1:
724		: [ret] "={w0}" (-> u8),
725		: [expected] "{w0}" (expected),
726		[desired] "{w1}" (desired),
727		[ptr] "{x2}" (ptr),
	604	: [ret] "={w0}" (-> u16),
	605	: [val] "{w0}" (val),
	606	[ptr] "{x1}" (ptr),
728	607	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
729	608	: "w15", "w16", "w17", "memory"
730	609	);
731	610	}
732		const __aarch64_cas1_acq = if (is_darwin)
733		if (has_lse)
734		cas1AcqDarwinLse
735		else
736		cas1AcqDarwinNolse
737		else if (has_lse)
738		cas1AcqNondarwinLse
739		else
740		cas1AcqNondarwinNolse;
741		fn swp1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	611	fn __aarch64_ldeor2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
742	612	@setRuntimeSafety(false);
743		__init_aarch64_have_lse_atomics();
744
745	613	return asm volatile (
746	614	\\ cbz w16, 8f
747		\\ swpab w0, w0, [x1]
	615	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
748	616	\\ cbz wzr, 1f
749	617	\\8:
750	618	\\ mov w16, w0
751	619	\\0:
752		\\ ldaxrb w0, [x1]
753		\\ stxrb w17, w16, [x1]
754		\\ cbnz w17, 0b
	620	\\ ldxrh w0, [x1]
	621	\\ eor w17, w0, w16
	622	\\ stxrh w15, w17, [x1]
	623	\\ cbnz w15, 0b
755	624	\\1:
756		: [ret] "={w0}" (-> u8),
	625	: [ret] "={w0}" (-> u16),
757	626	: [val] "{w0}" (val),
758	627	[ptr] "{x1}" (ptr),
759	628	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
760	629	: "w15", "w16", "w17", "memory"
761	630	);
762	631	}
763		fn swp1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	632	fn __aarch64_ldset2_relax(val: u16, ptr: *u16) callconv(.C) u16 {
764	633	@setRuntimeSafety(false);
765		__init_aarch64_have_lse_atomics();
766
767	634	return asm volatile (
768	635	\\ cbz w16, 8f
769		\\ .inst 0x38208020 + 0x00000000 + 0x800000
	636	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
770	637	\\ cbz wzr, 1f
771	638	\\8:
772	639	\\ mov w16, w0
773	640	\\0:
774		\\ ldaxrb w0, [x1]
775		\\ stxrb w17, w16, [x1]
776		\\ cbnz w17, 0b
	641	\\ ldxrh w0, [x1]
	642	\\ orr w17, w0, w16
	643	\\ stxrh w15, w17, [x1]
	644	\\ cbnz w15, 0b
777	645	\\1:
778		: [ret] "={w0}" (-> u8),
	646	: [ret] "={w0}" (-> u16),
779	647	: [val] "{w0}" (val),
780	648	[ptr] "{x1}" (ptr),
781	649	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
782	650	: "w15", "w16", "w17", "memory"
783	651	);
784	652	}
785		fn swp1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	653	fn __aarch64_cas2_acq(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
786	654	@setRuntimeSafety(false);
787		__init_aarch64_have_lse_atomics();
788
789	655	return asm volatile (
790	656	\\ cbz w16, 8f
791		\\ swpab w0, w0, [x1]
	657	\\ .inst 0x08a07c41 + 0x40000000 + 0x400000
	658	\\
792	659	\\ cbz wzr, 1f
793	660	\\8:
794		\\ mov w16, w0
	661	\\ uxth w16, w0
795	662	\\0:
796		\\ ldaxrb w0, [x1]
797		\\ stxrb w17, w16, [x1]
	663	\\ ldaxrh w0, [x2]
	664	\\ cmp w0, w16
	665	\\ bne 1f
	666	\\ stxrh w17, w1, [x2]
798	667	\\ cbnz w17, 0b
799	668	\\1:
800		: [ret] "={w0}" (-> u8),
801		: [val] "{w0}" (val),
802		[ptr] "{x1}" (ptr),
	669	: [ret] "={w0}" (-> u16),
	670	: [expected] "{w0}" (expected),
	671	[desired] "{w1}" (desired),
	672	[ptr] "{x2}" (ptr),
803	673	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
804	674	: "w15", "w16", "w17", "memory"
805	675	);
806	676	}
807		fn swp1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	677	fn __aarch64_swp2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
808	678	@setRuntimeSafety(false);
809		__init_aarch64_have_lse_atomics();
810
811	679	return asm volatile (
812	680	\\ cbz w16, 8f
813		\\ .inst 0x38208020 + 0x00000000 + 0x800000
	681	\\ .inst 0x38208020 + 0x40000000 + 0x800000
814	682	\\ cbz wzr, 1f
815	683	\\8:
816	684	\\ mov w16, w0
817	685	\\0:
818		\\ ldaxrb w0, [x1]
819		\\ stxrb w17, w16, [x1]
	686	\\ ldaxrh w0, [x1]
	687	\\ stxrh w17, w16, [x1]
820	688	\\ cbnz w17, 0b
821	689	\\1:
822		: [ret] "={w0}" (-> u8),
	690	: [ret] "={w0}" (-> u16),
823	691	: [val] "{w0}" (val),
824	692	[ptr] "{x1}" (ptr),
825	693	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
826	694	: "w15", "w16", "w17", "memory"
827	695	);
828	696	}
829		const __aarch64_swp1_acq = if (is_darwin)
830		if (has_lse)
831		swp1AcqDarwinLse
832		else
833		swp1AcqDarwinNolse
834		else if (has_lse)
835		swp1AcqNondarwinLse
836		else
837		swp1AcqNondarwinNolse;
838		fn ldadd1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	697	fn __aarch64_ldadd2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
839	698	@setRuntimeSafety(false);
840		__init_aarch64_have_lse_atomics();
841
842	699	return asm volatile (
843	700	\\ cbz w16, 8f
844		\\ ldaddab w0, w0, [x1]
	701	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
845	702	\\ cbz wzr, 1f
846	703	\\8:
847	704	\\ mov w16, w0
848	705	\\0:
849		\\ ldaxrb w0, [x1]
	706	\\ ldaxrh w0, [x1]
850	707	\\ add w17, w0, w16
851		\\ stxrb w15, w17, [x1]
	708	\\ stxrh w15, w17, [x1]
852	709	\\ cbnz w15, 0b
853	710	\\1:
854		: [ret] "={w0}" (-> u8),
	711	: [ret] "={w0}" (-> u16),
855	712	: [val] "{w0}" (val),
856	713	[ptr] "{x1}" (ptr),
857	714	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
858	715	: "w15", "w16", "w17", "memory"
859	716	);
860	717	}
861		fn ldadd1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	718	fn __aarch64_ldclr2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
862	719	@setRuntimeSafety(false);
863		__init_aarch64_have_lse_atomics();
864
865	720	return asm volatile (
866	721	\\ cbz w16, 8f
867		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
	722	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
868	723	\\ cbz wzr, 1f
869	724	\\8:
870	725	\\ mov w16, w0
871	726	\\0:
872		\\ ldaxrb w0, [x1]
873		\\ add w17, w0, w16
874		\\ stxrb w15, w17, [x1]
	727	\\ ldaxrh w0, [x1]
	728	\\ bic w17, w0, w16
	729	\\ stxrh w15, w17, [x1]
875	730	\\ cbnz w15, 0b
876	731	\\1:
877		: [ret] "={w0}" (-> u8),
	732	: [ret] "={w0}" (-> u16),
878	733	: [val] "{w0}" (val),
879	734	[ptr] "{x1}" (ptr),
880	735	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
881	736	: "w15", "w16", "w17", "memory"
882	737	);
883	738	}
884		fn ldadd1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	739	fn __aarch64_ldeor2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
885	740	@setRuntimeSafety(false);
886		__init_aarch64_have_lse_atomics();
887
888	741	return asm volatile (
889	742	\\ cbz w16, 8f
890		\\ ldaddab w0, w0, [x1]
	743	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
891	744	\\ cbz wzr, 1f
892	745	\\8:
893	746	\\ mov w16, w0
894	747	\\0:
895		\\ ldaxrb w0, [x1]
896		\\ add w17, w0, w16
897		\\ stxrb w15, w17, [x1]
	748	\\ ldaxrh w0, [x1]
	749	\\ eor w17, w0, w16
	750	\\ stxrh w15, w17, [x1]
898	751	\\ cbnz w15, 0b
899	752	\\1:
900		: [ret] "={w0}" (-> u8),
	753	: [ret] "={w0}" (-> u16),
901	754	: [val] "{w0}" (val),
902	755	[ptr] "{x1}" (ptr),
903	756	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
904	757	: "w15", "w16", "w17", "memory"
905	758	);
906	759	}
907		fn ldadd1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	760	fn __aarch64_ldset2_acq(val: u16, ptr: *u16) callconv(.C) u16 {
908	761	@setRuntimeSafety(false);
909		__init_aarch64_have_lse_atomics();
910
911	762	return asm volatile (
912	763	\\ cbz w16, 8f
913		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x800000
	764	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
914	765	\\ cbz wzr, 1f
915	766	\\8:
916	767	\\ mov w16, w0
917	768	\\0:
918		\\ ldaxrb w0, [x1]
919		\\ add w17, w0, w16
920		\\ stxrb w15, w17, [x1]
	769	\\ ldaxrh w0, [x1]
	770	\\ orr w17, w0, w16
	771	\\ stxrh w15, w17, [x1]
921	772	\\ cbnz w15, 0b
922	773	\\1:
923		: [ret] "={w0}" (-> u8),
	774	: [ret] "={w0}" (-> u16),
924	775	: [val] "{w0}" (val),
925	776	[ptr] "{x1}" (ptr),
926	777	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
927	778	: "w15", "w16", "w17", "memory"
928	779	);
929	780	}
930		const __aarch64_ldadd1_acq = if (is_darwin)
931		if (has_lse)
932		ldadd1AcqDarwinLse
933		else
934		ldadd1AcqDarwinNolse
935		else if (has_lse)
936		ldadd1AcqNondarwinLse
937		else
938		ldadd1AcqNondarwinNolse;
939		fn ldclr1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	781	fn __aarch64_cas2_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
940	782	@setRuntimeSafety(false);
941		__init_aarch64_have_lse_atomics();
942
943	783	return asm volatile (
944	784	\\ cbz w16, 8f
945		\\ ldclrab w0, w0, [x1]
	785	\\ .inst 0x08a07c41 + 0x40000000 + 0x008000
	786	\\
946	787	\\ cbz wzr, 1f
947	788	\\8:
948		\\ mov w16, w0
	789	\\ uxth w16, w0
949	790	\\0:
950		\\ ldaxrb w0, [x1]
951		\\ bic w17, w0, w16
952		\\ stxrb w15, w17, [x1]
953		\\ cbnz w15, 0b
	791	\\ ldxrh w0, [x2]
	792	\\ cmp w0, w16
	793	\\ bne 1f
	794	\\ stlxrh w17, w1, [x2]
	795	\\ cbnz w17, 0b
954	796	\\1:
955		: [ret] "={w0}" (-> u8),
956		: [val] "{w0}" (val),
957		[ptr] "{x1}" (ptr),
	797	: [ret] "={w0}" (-> u16),
	798	: [expected] "{w0}" (expected),
	799	[desired] "{w1}" (desired),
	800	[ptr] "{x2}" (ptr),
958	801	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
959	802	: "w15", "w16", "w17", "memory"
960	803	);
961	804	}
962		fn ldclr1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	805	fn __aarch64_swp2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
963	806	@setRuntimeSafety(false);
964		__init_aarch64_have_lse_atomics();
965
966	807	return asm volatile (
967	808	\\ cbz w16, 8f
968		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
	809	\\ .inst 0x38208020 + 0x40000000 + 0x400000
969	810	\\ cbz wzr, 1f
970	811	\\8:
971	812	\\ mov w16, w0
972	813	\\0:
973		\\ ldaxrb w0, [x1]
974		\\ bic w17, w0, w16
975		\\ stxrb w15, w17, [x1]
976		\\ cbnz w15, 0b
	814	\\ ldxrh w0, [x1]
	815	\\ stlxrh w17, w16, [x1]
	816	\\ cbnz w17, 0b
977	817	\\1:
978		: [ret] "={w0}" (-> u8),
	818	: [ret] "={w0}" (-> u16),
979	819	: [val] "{w0}" (val),
980	820	[ptr] "{x1}" (ptr),
981	821	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
982	822	: "w15", "w16", "w17", "memory"
983	823	);
984	824	}
985		fn ldclr1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	825	fn __aarch64_ldadd2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
986	826	@setRuntimeSafety(false);
987		__init_aarch64_have_lse_atomics();
988
989	827	return asm volatile (
990	828	\\ cbz w16, 8f
991		\\ ldclrab w0, w0, [x1]
	829	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
992	830	\\ cbz wzr, 1f
993	831	\\8:
994	832	\\ mov w16, w0
995	833	\\0:
996		\\ ldaxrb w0, [x1]
997		\\ bic w17, w0, w16
998		\\ stxrb w15, w17, [x1]
	834	\\ ldxrh w0, [x1]
	835	\\ add w17, w0, w16
	836	\\ stlxrh w15, w17, [x1]
999	837	\\ cbnz w15, 0b
1000	838	\\1:
1001		: [ret] "={w0}" (-> u8),
	839	: [ret] "={w0}" (-> u16),
1002	840	: [val] "{w0}" (val),
1003	841	[ptr] "{x1}" (ptr),
1004	842	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1005	843	: "w15", "w16", "w17", "memory"
1006	844	);
1007	845	}
1008		fn ldclr1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	846	fn __aarch64_ldclr2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1009	847	@setRuntimeSafety(false);
1010		__init_aarch64_have_lse_atomics();
1011
1012	848	return asm volatile (
1013	849	\\ cbz w16, 8f
1014		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x800000
	850	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
1015	851	\\ cbz wzr, 1f
1016	852	\\8:
1017	853	\\ mov w16, w0
1018	854	\\0:
1019		\\ ldaxrb w0, [x1]
	855	\\ ldxrh w0, [x1]
1020	856	\\ bic w17, w0, w16
1021		\\ stxrb w15, w17, [x1]
	857	\\ stlxrh w15, w17, [x1]
1022	858	\\ cbnz w15, 0b
1023	859	\\1:
1024		: [ret] "={w0}" (-> u8),
	860	: [ret] "={w0}" (-> u16),
1025	861	: [val] "{w0}" (val),
1026	862	[ptr] "{x1}" (ptr),
1027	863	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1028	864	: "w15", "w16", "w17", "memory"
1029	865	);
1030	866	}
1031		const __aarch64_ldclr1_acq = if (is_darwin)
1032		if (has_lse)
1033		ldclr1AcqDarwinLse
1034		else
1035		ldclr1AcqDarwinNolse
1036		else if (has_lse)
1037		ldclr1AcqNondarwinLse
1038		else
1039		ldclr1AcqNondarwinNolse;
1040		fn ldeor1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	867	fn __aarch64_ldeor2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1041	868	@setRuntimeSafety(false);
1042		__init_aarch64_have_lse_atomics();
1043
1044	869	return asm volatile (
1045	870	\\ cbz w16, 8f
1046		\\ ldeorab w0, w0, [x1]
	871	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
1047	872	\\ cbz wzr, 1f
1048	873	\\8:
1049	874	\\ mov w16, w0
1050	875	\\0:
1051		\\ ldaxrb w0, [x1]
	876	\\ ldxrh w0, [x1]
1052	877	\\ eor w17, w0, w16
1053		\\ stxrb w15, w17, [x1]
	878	\\ stlxrh w15, w17, [x1]
1054	879	\\ cbnz w15, 0b
1055	880	\\1:
1056		: [ret] "={w0}" (-> u8),
	881	: [ret] "={w0}" (-> u16),
1057	882	: [val] "{w0}" (val),
1058	883	[ptr] "{x1}" (ptr),
1059	884	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1060	885	: "w15", "w16", "w17", "memory"
1061	886	);
1062	887	}
1063		fn ldeor1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	888	fn __aarch64_ldset2_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1064	889	@setRuntimeSafety(false);
1065		__init_aarch64_have_lse_atomics();
1066
1067	890	return asm volatile (
1068	891	\\ cbz w16, 8f
1069		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
	892	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
1070	893	\\ cbz wzr, 1f
1071	894	\\8:
1072	895	\\ mov w16, w0
1073	896	\\0:
1074		\\ ldaxrb w0, [x1]
1075		\\ eor w17, w0, w16
1076		\\ stxrb w15, w17, [x1]
	897	\\ ldxrh w0, [x1]
	898	\\ orr w17, w0, w16
	899	\\ stlxrh w15, w17, [x1]
1077	900	\\ cbnz w15, 0b
1078	901	\\1:
1079		: [ret] "={w0}" (-> u8),
	902	: [ret] "={w0}" (-> u16),
1080	903	: [val] "{w0}" (val),
1081	904	[ptr] "{x1}" (ptr),
1082	905	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1083	906	: "w15", "w16", "w17", "memory"
1084	907	);
1085	908	}
1086		fn ldeor1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	909	fn __aarch64_cas2_acq_rel(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
1087	910	@setRuntimeSafety(false);
1088		__init_aarch64_have_lse_atomics();
1089
1090	911	return asm volatile (
1091	912	\\ cbz w16, 8f
1092		\\ ldeorab w0, w0, [x1]
	913	\\ .inst 0x08a07c41 + 0x40000000 + 0x408000
	914	\\
1093	915	\\ cbz wzr, 1f
1094	916	\\8:
1095		\\ mov w16, w0
	917	\\ uxth w16, w0
1096	918	\\0:
1097		\\ ldaxrb w0, [x1]
1098		\\ eor w17, w0, w16
1099		\\ stxrb w15, w17, [x1]
1100		\\ cbnz w15, 0b
	919	\\ ldaxrh w0, [x2]
	920	\\ cmp w0, w16
	921	\\ bne 1f
	922	\\ stlxrh w17, w1, [x2]
	923	\\ cbnz w17, 0b
1101	924	\\1:
1102		: [ret] "={w0}" (-> u8),
1103		: [val] "{w0}" (val),
1104		[ptr] "{x1}" (ptr),
	925	: [ret] "={w0}" (-> u16),
	926	: [expected] "{w0}" (expected),
	927	[desired] "{w1}" (desired),
	928	[ptr] "{x2}" (ptr),
1105	929	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1106	930	: "w15", "w16", "w17", "memory"
1107	931	);
1108	932	}
1109		fn ldeor1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	933	fn __aarch64_swp2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1110	934	@setRuntimeSafety(false);
1111		__init_aarch64_have_lse_atomics();
1112
1113	935	return asm volatile (
1114	936	\\ cbz w16, 8f
1115		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x800000
	937	\\ .inst 0x38208020 + 0x40000000 + 0xc00000
1116	938	\\ cbz wzr, 1f
1117	939	\\8:
1118	940	\\ mov w16, w0
1119	941	\\0:
1120		\\ ldaxrb w0, [x1]
1121		\\ eor w17, w0, w16
1122		\\ stxrb w15, w17, [x1]
1123		\\ cbnz w15, 0b
	942	\\ ldaxrh w0, [x1]
	943	\\ stlxrh w17, w16, [x1]
	944	\\ cbnz w17, 0b
1124	945	\\1:
1125		: [ret] "={w0}" (-> u8),
	946	: [ret] "={w0}" (-> u16),
1126	947	: [val] "{w0}" (val),
1127	948	[ptr] "{x1}" (ptr),
1128	949	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1129	950	: "w15", "w16", "w17", "memory"
1130	951	);
1131	952	}
1132		const __aarch64_ldeor1_acq = if (is_darwin)
1133		if (has_lse)
1134		ldeor1AcqDarwinLse
1135		else
1136		ldeor1AcqDarwinNolse
1137		else if (has_lse)
1138		ldeor1AcqNondarwinLse
1139		else
1140		ldeor1AcqNondarwinNolse;
1141		fn ldset1AcqDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	953	fn __aarch64_ldadd2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1142	954	@setRuntimeSafety(false);
1143		__init_aarch64_have_lse_atomics();
1144
1145	955	return asm volatile (
1146	956	\\ cbz w16, 8f
1147		\\ ldsetab w0, w0, [x1]
	957	\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
1148	958	\\ cbz wzr, 1f
1149	959	\\8:
1150	960	\\ mov w16, w0
1151	961	\\0:
1152		\\ ldaxrb w0, [x1]
1153		\\ orr w17, w0, w16
1154		\\ stxrb w15, w17, [x1]
	962	\\ ldaxrh w0, [x1]
	963	\\ add w17, w0, w16
	964	\\ stlxrh w15, w17, [x1]
1155	965	\\ cbnz w15, 0b
1156	966	\\1:
1157		: [ret] "={w0}" (-> u8),
	967	: [ret] "={w0}" (-> u16),
1158	968	: [val] "{w0}" (val),
1159	969	[ptr] "{x1}" (ptr),
1160	970	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1161	971	: "w15", "w16", "w17", "memory"
1162	972	);
1163	973	}
1164		fn ldset1AcqDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	974	fn __aarch64_ldclr2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1165	975	@setRuntimeSafety(false);
1166		__init_aarch64_have_lse_atomics();
1167
1168	976	return asm volatile (
1169	977	\\ cbz w16, 8f
1170		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
	978	\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
1171	979	\\ cbz wzr, 1f
1172	980	\\8:
1173	981	\\ mov w16, w0
1174	982	\\0:
1175		\\ ldaxrb w0, [x1]
1176		\\ orr w17, w0, w16
1177		\\ stxrb w15, w17, [x1]
	983	\\ ldaxrh w0, [x1]
	984	\\ bic w17, w0, w16
	985	\\ stlxrh w15, w17, [x1]
1178	986	\\ cbnz w15, 0b
1179	987	\\1:
1180		: [ret] "={w0}" (-> u8),
	988	: [ret] "={w0}" (-> u16),
1181	989	: [val] "{w0}" (val),
1182	990	[ptr] "{x1}" (ptr),
1183	991	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1184	992	: "w15", "w16", "w17", "memory"
1185	993	);
1186	994	}
1187		fn ldset1AcqNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	995	fn __aarch64_ldeor2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1188	996	@setRuntimeSafety(false);
1189		__init_aarch64_have_lse_atomics();
1190
1191	997	return asm volatile (
1192	998	\\ cbz w16, 8f
1193		\\ ldsetab w0, w0, [x1]
	999	\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
1194	1000	\\ cbz wzr, 1f
1195	1001	\\8:
1196	1002	\\ mov w16, w0
1197	1003	\\0:
1198		\\ ldaxrb w0, [x1]
1199		\\ orr w17, w0, w16
1200		\\ stxrb w15, w17, [x1]
	1004	\\ ldaxrh w0, [x1]
	1005	\\ eor w17, w0, w16
	1006	\\ stlxrh w15, w17, [x1]
1201	1007	\\ cbnz w15, 0b
1202	1008	\\1:
1203		: [ret] "={w0}" (-> u8),
	1009	: [ret] "={w0}" (-> u16),
1204	1010	: [val] "{w0}" (val),
1205	1011	[ptr] "{x1}" (ptr),
1206	1012	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1207	1013	: "w15", "w16", "w17", "memory"
1208	1014	);
1209	1015	}
1210		fn ldset1AcqNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1016	fn __aarch64_ldset2_acq_rel(val: u16, ptr: *u16) callconv(.C) u16 {
1211	1017	@setRuntimeSafety(false);
1212		__init_aarch64_have_lse_atomics();
1213
1214	1018	return asm volatile (
1215	1019	\\ cbz w16, 8f
1216		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x800000
	1020	\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
1217	1021	\\ cbz wzr, 1f
1218	1022	\\8:
1219	1023	\\ mov w16, w0
1220	1024	\\0:
1221		\\ ldaxrb w0, [x1]
	1025	\\ ldaxrh w0, [x1]
1222	1026	\\ orr w17, w0, w16
1223		\\ stxrb w15, w17, [x1]
	1027	\\ stlxrh w15, w17, [x1]
1224	1028	\\ cbnz w15, 0b
1225	1029	\\1:
1226		: [ret] "={w0}" (-> u8),
	1030	: [ret] "={w0}" (-> u16),
1227	1031	: [val] "{w0}" (val),
1228	1032	[ptr] "{x1}" (ptr),
1229	1033	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1230	1034	: "w15", "w16", "w17", "memory"
1231	1035	);
1232	1036	}
1233		const __aarch64_ldset1_acq = if (is_darwin)
1234		if (has_lse)
1235		ldset1AcqDarwinLse
1236		else
1237		ldset1AcqDarwinNolse
1238		else if (has_lse)
1239		ldset1AcqNondarwinLse
1240		else
1241		ldset1AcqNondarwinNolse;
1242		fn cas1RelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	1037	fn __aarch64_cas4_relax(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1243	1038	@setRuntimeSafety(false);
1244		__init_aarch64_have_lse_atomics();
1245
1246	1039	return asm volatile (
1247	1040	\\ cbz w16, 8f
1248		\\ caslb w0, w1, [x2]
	1041	\\ .inst 0x08a07c41 + 0x80000000 + 0x000000
1249	1042	\\
1250	1043	\\ cbz wzr, 1f
1251	1044	\\8:
1252		\\ uxtb w16, w0
	1045	\\ mov w16, w0
1253	1046	\\0:
1254		\\ ldxrb w0, [x2]
	1047	\\ ldxr w0, [x2]
1255	1048	\\ cmp w0, w16
1256	1049	\\ bne 1f
1257		\\ stlxrb w17, w1, [x2]
	1050	\\ stxr w17, w1, [x2]
1258	1051	\\ cbnz w17, 0b
1259	1052	\\1:
1260		: [ret] "={w0}" (-> u8),
	1053	: [ret] "={w0}" (-> u32),
1261	1054	: [expected] "{w0}" (expected),
1262	1055	[desired] "{w1}" (desired),
1263	1056	[ptr] "{x2}" (ptr),
...	...	@@ -1265,8311 +1058,530 @@ fn cas1RelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1265	1058	: "w15", "w16", "w17", "memory"
1266	1059	);
1267	1060	}
1268		fn cas1RelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	1061	fn __aarch64_swp4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1269	1062	@setRuntimeSafety(false);
1270		__init_aarch64_have_lse_atomics();
1271
1272	1063	return asm volatile (
1273	1064	\\ cbz w16, 8f
1274		\\ .inst 0x08a07c41 + 0x00000000 + 0x008000
1275		\\
	1065	\\ .inst 0x38208020 + 0x80000000 + 0x000000
1276	1066	\\ cbz wzr, 1f
1277	1067	\\8:
1278		\\ uxtb w16, w0
	1068	\\ mov w16, w0
1279	1069	\\0:
1280		\\ ldxrb w0, [x2]
1281		\\ cmp w0, w16
1282		\\ bne 1f
1283		\\ stlxrb w17, w1, [x2]
	1070	\\ ldxr w0, [x1]
	1071	\\ stxr w17, w16, [x1]
1284	1072	\\ cbnz w17, 0b
1285	1073	\\1:
1286		: [ret] "={w0}" (-> u8),
1287		: [expected] "{w0}" (expected),
1288		[desired] "{w1}" (desired),
1289		[ptr] "{x2}" (ptr),
	1074	: [ret] "={w0}" (-> u32),
	1075	: [val] "{w0}" (val),
	1076	[ptr] "{x1}" (ptr),
1290	1077	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1291	1078	: "w15", "w16", "w17", "memory"
1292	1079	);
1293	1080	}
1294		fn cas1RelNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	1081	fn __aarch64_ldadd4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1295	1082	@setRuntimeSafety(false);
1296		__init_aarch64_have_lse_atomics();
1297
1298	1083	return asm volatile (
1299	1084	\\ cbz w16, 8f
1300		\\ caslb w0, w1, [x2]
1301		\\
	1085	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
1302	1086	\\ cbz wzr, 1f
1303	1087	\\8:
1304		\\ uxtb w16, w0
	1088	\\ mov w16, w0
1305	1089	\\0:
1306		\\ ldxrb w0, [x2]
1307		\\ cmp w0, w16
1308		\\ bne 1f
1309		\\ stlxrb w17, w1, [x2]
1310		\\ cbnz w17, 0b
	1090	\\ ldxr w0, [x1]
	1091	\\ add w17, w0, w16
	1092	\\ stxr w15, w17, [x1]
	1093	\\ cbnz w15, 0b
1311	1094	\\1:
1312		: [ret] "={w0}" (-> u8),
1313		: [expected] "{w0}" (expected),
1314		[desired] "{w1}" (desired),
1315		[ptr] "{x2}" (ptr),
	1095	: [ret] "={w0}" (-> u32),
	1096	: [val] "{w0}" (val),
	1097	[ptr] "{x1}" (ptr),
1316	1098	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1317	1099	: "w15", "w16", "w17", "memory"
1318	1100	);
1319	1101	}
1320		fn cas1RelNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
	1102	fn __aarch64_ldclr4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1321	1103	@setRuntimeSafety(false);
1322		__init_aarch64_have_lse_atomics();
1323
1324	1104	return asm volatile (
1325	1105	\\ cbz w16, 8f
1326		\\ .inst 0x08a07c41 + 0x00000000 + 0x008000
1327		\\
	1106	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
1328	1107	\\ cbz wzr, 1f
1329	1108	\\8:
1330		\\ uxtb w16, w0
	1109	\\ mov w16, w0
1331	1110	\\0:
1332		\\ ldxrb w0, [x2]
1333		\\ cmp w0, w16
1334		\\ bne 1f
1335		\\ stlxrb w17, w1, [x2]
1336		\\ cbnz w17, 0b
	1111	\\ ldxr w0, [x1]
	1112	\\ bic w17, w0, w16
	1113	\\ stxr w15, w17, [x1]
	1114	\\ cbnz w15, 0b
1337	1115	\\1:
1338		: [ret] "={w0}" (-> u8),
1339		: [expected] "{w0}" (expected),
1340		[desired] "{w1}" (desired),
1341		[ptr] "{x2}" (ptr),
	1116	: [ret] "={w0}" (-> u32),
	1117	: [val] "{w0}" (val),
	1118	[ptr] "{x1}" (ptr),
1342	1119	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1343	1120	: "w15", "w16", "w17", "memory"
1344	1121	);
1345	1122	}
1346		const __aarch64_cas1_rel = if (is_darwin)
1347		if (has_lse)
1348		cas1RelDarwinLse
1349		else
1350		cas1RelDarwinNolse
1351		else if (has_lse)
1352		cas1RelNondarwinLse
1353		else
1354		cas1RelNondarwinNolse;
1355		fn swp1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1123	fn __aarch64_ldeor4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1356	1124	@setRuntimeSafety(false);
1357		__init_aarch64_have_lse_atomics();
1358
1359	1125	return asm volatile (
1360	1126	\\ cbz w16, 8f
1361		\\ swplb w0, w0, [x1]
	1127	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
1362	1128	\\ cbz wzr, 1f
1363	1129	\\8:
1364	1130	\\ mov w16, w0
1365	1131	\\0:
1366		\\ ldxrb w0, [x1]
1367		\\ stlxrb w17, w16, [x1]
1368		\\ cbnz w17, 0b
	1132	\\ ldxr w0, [x1]
	1133	\\ eor w17, w0, w16
	1134	\\ stxr w15, w17, [x1]
	1135	\\ cbnz w15, 0b
1369	1136	\\1:
1370		: [ret] "={w0}" (-> u8),
	1137	: [ret] "={w0}" (-> u32),
1371	1138	: [val] "{w0}" (val),
1372	1139	[ptr] "{x1}" (ptr),
1373	1140	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1374	1141	: "w15", "w16", "w17", "memory"
1375	1142	);
1376	1143	}
1377		fn swp1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1144	fn __aarch64_ldset4_relax(val: u32, ptr: *u32) callconv(.C) u32 {
1378	1145	@setRuntimeSafety(false);
1379		__init_aarch64_have_lse_atomics();
1380
1381	1146	return asm volatile (
1382	1147	\\ cbz w16, 8f
1383		\\ .inst 0x38208020 + 0x00000000 + 0x400000
	1148	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
1384	1149	\\ cbz wzr, 1f
1385	1150	\\8:
1386	1151	\\ mov w16, w0
1387	1152	\\0:
1388		\\ ldxrb w0, [x1]
1389		\\ stlxrb w17, w16, [x1]
1390		\\ cbnz w17, 0b
	1153	\\ ldxr w0, [x1]
	1154	\\ orr w17, w0, w16
	1155	\\ stxr w15, w17, [x1]
	1156	\\ cbnz w15, 0b
1391	1157	\\1:
1392		: [ret] "={w0}" (-> u8),
	1158	: [ret] "={w0}" (-> u32),
1393	1159	: [val] "{w0}" (val),
1394	1160	[ptr] "{x1}" (ptr),
1395	1161	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1396	1162	: "w15", "w16", "w17", "memory"
1397	1163	);
1398	1164	}
1399		fn swp1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1165	fn __aarch64_cas4_acq(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1400	1166	@setRuntimeSafety(false);
1401		__init_aarch64_have_lse_atomics();
1402
1403	1167	return asm volatile (
1404	1168	\\ cbz w16, 8f
1405		\\ swplb w0, w0, [x1]
	1169	\\ .inst 0x08a07c41 + 0x80000000 + 0x400000
	1170	\\
1406	1171	\\ cbz wzr, 1f
1407	1172	\\8:
1408	1173	\\ mov w16, w0
1409	1174	\\0:
1410		\\ ldxrb w0, [x1]
1411		\\ stlxrb w17, w16, [x1]
	1175	\\ ldaxr w0, [x2]
	1176	\\ cmp w0, w16
	1177	\\ bne 1f
	1178	\\ stxr w17, w1, [x2]
1412	1179	\\ cbnz w17, 0b
1413	1180	\\1:
1414		: [ret] "={w0}" (-> u8),
1415		: [val] "{w0}" (val),
1416		[ptr] "{x1}" (ptr),
	1181	: [ret] "={w0}" (-> u32),
	1182	: [expected] "{w0}" (expected),
	1183	[desired] "{w1}" (desired),
	1184	[ptr] "{x2}" (ptr),
1417	1185	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1418	1186	: "w15", "w16", "w17", "memory"
1419	1187	);
1420	1188	}
1421		fn swp1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1189	fn __aarch64_swp4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1422	1190	@setRuntimeSafety(false);
1423		__init_aarch64_have_lse_atomics();
1424
1425	1191	return asm volatile (
1426	1192	\\ cbz w16, 8f
1427		\\ .inst 0x38208020 + 0x00000000 + 0x400000
	1193	\\ .inst 0x38208020 + 0x80000000 + 0x800000
1428	1194	\\ cbz wzr, 1f
1429	1195	\\8:
1430	1196	\\ mov w16, w0
1431	1197	\\0:
1432		\\ ldxrb w0, [x1]
1433		\\ stlxrb w17, w16, [x1]
	1198	\\ ldaxr w0, [x1]
	1199	\\ stxr w17, w16, [x1]
1434	1200	\\ cbnz w17, 0b
1435	1201	\\1:
1436		: [ret] "={w0}" (-> u8),
	1202	: [ret] "={w0}" (-> u32),
1437	1203	: [val] "{w0}" (val),
1438	1204	[ptr] "{x1}" (ptr),
1439	1205	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1440	1206	: "w15", "w16", "w17", "memory"
1441	1207	);
1442	1208	}
1443		const __aarch64_swp1_rel = if (is_darwin)
1444		if (has_lse)
1445		swp1RelDarwinLse
1446		else
1447		swp1RelDarwinNolse
1448		else if (has_lse)
1449		swp1RelNondarwinLse
1450		else
1451		swp1RelNondarwinNolse;
1452		fn ldadd1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1209	fn __aarch64_ldadd4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1453	1210	@setRuntimeSafety(false);
1454		__init_aarch64_have_lse_atomics();
1455
1456	1211	return asm volatile (
1457	1212	\\ cbz w16, 8f
1458		\\ ldaddlb w0, w0, [x1]
	1213	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
1459	1214	\\ cbz wzr, 1f
1460	1215	\\8:
1461	1216	\\ mov w16, w0
1462	1217	\\0:
1463		\\ ldxrb w0, [x1]
	1218	\\ ldaxr w0, [x1]
1464	1219	\\ add w17, w0, w16
1465		\\ stlxrb w15, w17, [x1]
	1220	\\ stxr w15, w17, [x1]
1466	1221	\\ cbnz w15, 0b
1467	1222	\\1:
1468		: [ret] "={w0}" (-> u8),
	1223	: [ret] "={w0}" (-> u32),
1469	1224	: [val] "{w0}" (val),
1470	1225	[ptr] "{x1}" (ptr),
1471	1226	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1472	1227	: "w15", "w16", "w17", "memory"
1473	1228	);
1474	1229	}
1475		fn ldadd1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1230	fn __aarch64_ldclr4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1476	1231	@setRuntimeSafety(false);
1477		__init_aarch64_have_lse_atomics();
1478
1479	1232	return asm volatile (
1480	1233	\\ cbz w16, 8f
1481		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
	1234	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
1482	1235	\\ cbz wzr, 1f
1483	1236	\\8:
1484	1237	\\ mov w16, w0
1485	1238	\\0:
1486		\\ ldxrb w0, [x1]
1487		\\ add w17, w0, w16
1488		\\ stlxrb w15, w17, [x1]
	1239	\\ ldaxr w0, [x1]
	1240	\\ bic w17, w0, w16
	1241	\\ stxr w15, w17, [x1]
1489	1242	\\ cbnz w15, 0b
1490	1243	\\1:
1491		: [ret] "={w0}" (-> u8),
	1244	: [ret] "={w0}" (-> u32),
1492	1245	: [val] "{w0}" (val),
1493	1246	[ptr] "{x1}" (ptr),
1494	1247	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1495	1248	: "w15", "w16", "w17", "memory"
1496	1249	);
1497	1250	}
1498		fn ldadd1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1251	fn __aarch64_ldeor4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1499	1252	@setRuntimeSafety(false);
1500		__init_aarch64_have_lse_atomics();
1501
1502	1253	return asm volatile (
1503	1254	\\ cbz w16, 8f
1504		\\ ldaddlb w0, w0, [x1]
	1255	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
1505	1256	\\ cbz wzr, 1f
1506	1257	\\8:
1507	1258	\\ mov w16, w0
1508	1259	\\0:
1509		\\ ldxrb w0, [x1]
1510		\\ add w17, w0, w16
1511		\\ stlxrb w15, w17, [x1]
	1260	\\ ldaxr w0, [x1]
	1261	\\ eor w17, w0, w16
	1262	\\ stxr w15, w17, [x1]
1512	1263	\\ cbnz w15, 0b
1513	1264	\\1:
1514		: [ret] "={w0}" (-> u8),
	1265	: [ret] "={w0}" (-> u32),
1515	1266	: [val] "{w0}" (val),
1516	1267	[ptr] "{x1}" (ptr),
1517	1268	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1518	1269	: "w15", "w16", "w17", "memory"
1519	1270	);
1520	1271	}
1521		fn ldadd1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1272	fn __aarch64_ldset4_acq(val: u32, ptr: *u32) callconv(.C) u32 {
1522	1273	@setRuntimeSafety(false);
1523		__init_aarch64_have_lse_atomics();
1524
1525	1274	return asm volatile (
1526	1275	\\ cbz w16, 8f
1527		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0x400000
	1276	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
1528	1277	\\ cbz wzr, 1f
1529	1278	\\8:
1530	1279	\\ mov w16, w0
1531	1280	\\0:
1532		\\ ldxrb w0, [x1]
1533		\\ add w17, w0, w16
1534		\\ stlxrb w15, w17, [x1]
	1281	\\ ldaxr w0, [x1]
	1282	\\ orr w17, w0, w16
	1283	\\ stxr w15, w17, [x1]
1535	1284	\\ cbnz w15, 0b
1536	1285	\\1:
1537		: [ret] "={w0}" (-> u8),
	1286	: [ret] "={w0}" (-> u32),
1538	1287	: [val] "{w0}" (val),
1539	1288	[ptr] "{x1}" (ptr),
1540	1289	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1541	1290	: "w15", "w16", "w17", "memory"
1542	1291	);
1543	1292	}
1544		const __aarch64_ldadd1_rel = if (is_darwin)
1545		if (has_lse)
1546		ldadd1RelDarwinLse
1547		else
1548		ldadd1RelDarwinNolse
1549		else if (has_lse)
1550		ldadd1RelNondarwinLse
1551		else
1552		ldadd1RelNondarwinNolse;
1553		fn ldclr1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1293	fn __aarch64_cas4_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1554	1294	@setRuntimeSafety(false);
1555		__init_aarch64_have_lse_atomics();
1556
1557	1295	return asm volatile (
1558	1296	\\ cbz w16, 8f
1559		\\ ldclrlb w0, w0, [x1]
	1297	\\ .inst 0x08a07c41 + 0x80000000 + 0x008000
	1298	\\
1560	1299	\\ cbz wzr, 1f
1561	1300	\\8:
1562	1301	\\ mov w16, w0
1563	1302	\\0:
1564		\\ ldxrb w0, [x1]
1565		\\ bic w17, w0, w16
1566		\\ stlxrb w15, w17, [x1]
1567		\\ cbnz w15, 0b
	1303	\\ ldxr w0, [x2]
	1304	\\ cmp w0, w16
	1305	\\ bne 1f
	1306	\\ stlxr w17, w1, [x2]
	1307	\\ cbnz w17, 0b
1568	1308	\\1:
1569		: [ret] "={w0}" (-> u8),
1570		: [val] "{w0}" (val),
1571		[ptr] "{x1}" (ptr),
	1309	: [ret] "={w0}" (-> u32),
	1310	: [expected] "{w0}" (expected),
	1311	[desired] "{w1}" (desired),
	1312	[ptr] "{x2}" (ptr),
1572	1313	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1573	1314	: "w15", "w16", "w17", "memory"
1574	1315	);
1575	1316	}
1576		fn ldclr1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1317	fn __aarch64_swp4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1577	1318	@setRuntimeSafety(false);
1578		__init_aarch64_have_lse_atomics();
1579
1580	1319	return asm volatile (
1581	1320	\\ cbz w16, 8f
1582		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
	1321	\\ .inst 0x38208020 + 0x80000000 + 0x400000
1583	1322	\\ cbz wzr, 1f
1584	1323	\\8:
1585	1324	\\ mov w16, w0
1586	1325	\\0:
1587		\\ ldxrb w0, [x1]
1588		\\ bic w17, w0, w16
1589		\\ stlxrb w15, w17, [x1]
1590		\\ cbnz w15, 0b
	1326	\\ ldxr w0, [x1]
	1327	\\ stlxr w17, w16, [x1]
	1328	\\ cbnz w17, 0b
1591	1329	\\1:
1592		: [ret] "={w0}" (-> u8),
	1330	: [ret] "={w0}" (-> u32),
1593	1331	: [val] "{w0}" (val),
1594	1332	[ptr] "{x1}" (ptr),
1595	1333	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1596	1334	: "w15", "w16", "w17", "memory"
1597	1335	);
1598	1336	}
1599		fn ldclr1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1337	fn __aarch64_ldadd4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1600	1338	@setRuntimeSafety(false);
1601		__init_aarch64_have_lse_atomics();
1602
1603	1339	return asm volatile (
1604	1340	\\ cbz w16, 8f
1605		\\ ldclrlb w0, w0, [x1]
	1341	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
1606	1342	\\ cbz wzr, 1f
1607	1343	\\8:
1608	1344	\\ mov w16, w0
1609	1345	\\0:
1610		\\ ldxrb w0, [x1]
1611		\\ bic w17, w0, w16
1612		\\ stlxrb w15, w17, [x1]
	1346	\\ ldxr w0, [x1]
	1347	\\ add w17, w0, w16
	1348	\\ stlxr w15, w17, [x1]
1613	1349	\\ cbnz w15, 0b
1614	1350	\\1:
1615		: [ret] "={w0}" (-> u8),
	1351	: [ret] "={w0}" (-> u32),
1616	1352	: [val] "{w0}" (val),
1617	1353	[ptr] "{x1}" (ptr),
1618	1354	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1619	1355	: "w15", "w16", "w17", "memory"
1620	1356	);
1621	1357	}
1622		fn ldclr1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1358	fn __aarch64_ldclr4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1623	1359	@setRuntimeSafety(false);
1624		__init_aarch64_have_lse_atomics();
1625
1626	1360	return asm volatile (
1627	1361	\\ cbz w16, 8f
1628		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0x400000
	1362	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
1629	1363	\\ cbz wzr, 1f
1630	1364	\\8:
1631	1365	\\ mov w16, w0
1632	1366	\\0:
1633		\\ ldxrb w0, [x1]
	1367	\\ ldxr w0, [x1]
1634	1368	\\ bic w17, w0, w16
1635		\\ stlxrb w15, w17, [x1]
	1369	\\ stlxr w15, w17, [x1]
1636	1370	\\ cbnz w15, 0b
1637	1371	\\1:
1638		: [ret] "={w0}" (-> u8),
	1372	: [ret] "={w0}" (-> u32),
1639	1373	: [val] "{w0}" (val),
1640	1374	[ptr] "{x1}" (ptr),
1641	1375	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1642	1376	: "w15", "w16", "w17", "memory"
1643	1377	);
1644	1378	}
1645		const __aarch64_ldclr1_rel = if (is_darwin)
1646		if (has_lse)
1647		ldclr1RelDarwinLse
1648		else
1649		ldclr1RelDarwinNolse
1650		else if (has_lse)
1651		ldclr1RelNondarwinLse
1652		else
1653		ldclr1RelNondarwinNolse;
1654		fn ldeor1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1379	fn __aarch64_ldeor4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1655	1380	@setRuntimeSafety(false);
1656		__init_aarch64_have_lse_atomics();
1657
1658	1381	return asm volatile (
1659	1382	\\ cbz w16, 8f
1660		\\ ldeorlb w0, w0, [x1]
	1383	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
1661	1384	\\ cbz wzr, 1f
1662	1385	\\8:
1663	1386	\\ mov w16, w0
1664	1387	\\0:
1665		\\ ldxrb w0, [x1]
	1388	\\ ldxr w0, [x1]
1666	1389	\\ eor w17, w0, w16
1667		\\ stlxrb w15, w17, [x1]
	1390	\\ stlxr w15, w17, [x1]
1668	1391	\\ cbnz w15, 0b
1669	1392	\\1:
1670		: [ret] "={w0}" (-> u8),
	1393	: [ret] "={w0}" (-> u32),
1671	1394	: [val] "{w0}" (val),
1672	1395	[ptr] "{x1}" (ptr),
1673	1396	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1674	1397	: "w15", "w16", "w17", "memory"
1675	1398	);
1676	1399	}
1677		fn ldeor1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
	1400	fn __aarch64_ldset4_rel(val: u32, ptr: *u32) callconv(.C) u32 {
1678	1401	@setRuntimeSafety(false);
1679		__init_aarch64_have_lse_atomics();
1680
1681	1402	return asm volatile (
1682	1403	\\ cbz w16, 8f
1683		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
	1404	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
1684	1405	\\ cbz wzr, 1f
1685	1406	\\8:
1686	1407	\\ mov w16, w0
1687	1408	\\0:
1688		\\ ldxrb w0, [x1]
1689		\\ eor w17, w0, w16
1690		\\ stlxrb w15, w17, [x1]
	1409	\\ ldxr w0, [x1]
	1410	\\ orr w17, w0, w16
	1411	\\ stlxr w15, w17, [x1]
1691	1412	\\ cbnz w15, 0b
1692	1413	\\1:
1693		: [ret] "={w0}" (-> u8),
	1414	: [ret] "={w0}" (-> u32),
1694	1415	: [val] "{w0}" (val),
1695	1416	[ptr] "{x1}" (ptr),
1696	1417	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1697	1418	: "w15", "w16", "w17", "memory"
1698	1419	);
1699	1420	}
1700		fn ldeor1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
	1421	fn __aarch64_cas4_acq_rel(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
1701	1422	@setRuntimeSafety(false);
1702		__init_aarch64_have_lse_atomics();
1703
1704	1423	return asm volatile (
1705	1424	\\ cbz w16, 8f
1706		\\ ldeorlb w0, w0, [x1]
1707		\\ cbz wzr, 1f
1708		\\8:
1709		\\ mov w16, w0
1710		\\0:
1711		\\ ldxrb w0, [x1]
1712		\\ eor w17, w0, w16
1713		\\ stlxrb w15, w17, [x1]
1714		\\ cbnz w15, 0b
1715		\\1:
1716		: [ret] "={w0}" (-> u8),
1717		: [val] "{w0}" (val),
1718		[ptr] "{x1}" (ptr),
1719		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1720		: "w15", "w16", "w17", "memory"
1721		);
1722		}
1723		fn ldeor1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1724		@setRuntimeSafety(false);
1725		__init_aarch64_have_lse_atomics();
1726
1727		return asm volatile (
1728		\\ cbz w16, 8f
1729		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0x400000
1730		\\ cbz wzr, 1f
1731		\\8:
1732		\\ mov w16, w0
1733		\\0:
1734		\\ ldxrb w0, [x1]
1735		\\ eor w17, w0, w16
1736		\\ stlxrb w15, w17, [x1]
1737		\\ cbnz w15, 0b
1738		\\1:
1739		: [ret] "={w0}" (-> u8),
1740		: [val] "{w0}" (val),
1741		[ptr] "{x1}" (ptr),
1742		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1743		: "w15", "w16", "w17", "memory"
1744		);
1745		}
1746		const __aarch64_ldeor1_rel = if (is_darwin)
1747		if (has_lse)
1748		ldeor1RelDarwinLse
1749		else
1750		ldeor1RelDarwinNolse
1751		else if (has_lse)
1752		ldeor1RelNondarwinLse
1753		else
1754		ldeor1RelNondarwinNolse;
1755		fn ldset1RelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1756		@setRuntimeSafety(false);
1757		__init_aarch64_have_lse_atomics();
1758
1759		return asm volatile (
1760		\\ cbz w16, 8f
1761		\\ ldsetlb w0, w0, [x1]
1762		\\ cbz wzr, 1f
1763		\\8:
1764		\\ mov w16, w0
1765		\\0:
1766		\\ ldxrb w0, [x1]
1767		\\ orr w17, w0, w16
1768		\\ stlxrb w15, w17, [x1]
1769		\\ cbnz w15, 0b
1770		\\1:
1771		: [ret] "={w0}" (-> u8),
1772		: [val] "{w0}" (val),
1773		[ptr] "{x1}" (ptr),
1774		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1775		: "w15", "w16", "w17", "memory"
1776		);
1777		}
1778		fn ldset1RelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1779		@setRuntimeSafety(false);
1780		__init_aarch64_have_lse_atomics();
1781
1782		return asm volatile (
1783		\\ cbz w16, 8f
1784		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
1785		\\ cbz wzr, 1f
1786		\\8:
1787		\\ mov w16, w0
1788		\\0:
1789		\\ ldxrb w0, [x1]
1790		\\ orr w17, w0, w16
1791		\\ stlxrb w15, w17, [x1]
1792		\\ cbnz w15, 0b
1793		\\1:
1794		: [ret] "={w0}" (-> u8),
1795		: [val] "{w0}" (val),
1796		[ptr] "{x1}" (ptr),
1797		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1798		: "w15", "w16", "w17", "memory"
1799		);
1800		}
1801		fn ldset1RelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1802		@setRuntimeSafety(false);
1803		__init_aarch64_have_lse_atomics();
1804
1805		return asm volatile (
1806		\\ cbz w16, 8f
1807		\\ ldsetlb w0, w0, [x1]
1808		\\ cbz wzr, 1f
1809		\\8:
1810		\\ mov w16, w0
1811		\\0:
1812		\\ ldxrb w0, [x1]
1813		\\ orr w17, w0, w16
1814		\\ stlxrb w15, w17, [x1]
1815		\\ cbnz w15, 0b
1816		\\1:
1817		: [ret] "={w0}" (-> u8),
1818		: [val] "{w0}" (val),
1819		[ptr] "{x1}" (ptr),
1820		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1821		: "w15", "w16", "w17", "memory"
1822		);
1823		}
1824		fn ldset1RelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1825		@setRuntimeSafety(false);
1826		__init_aarch64_have_lse_atomics();
1827
1828		return asm volatile (
1829		\\ cbz w16, 8f
1830		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0x400000
1831		\\ cbz wzr, 1f
1832		\\8:
1833		\\ mov w16, w0
1834		\\0:
1835		\\ ldxrb w0, [x1]
1836		\\ orr w17, w0, w16
1837		\\ stlxrb w15, w17, [x1]
1838		\\ cbnz w15, 0b
1839		\\1:
1840		: [ret] "={w0}" (-> u8),
1841		: [val] "{w0}" (val),
1842		[ptr] "{x1}" (ptr),
1843		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1844		: "w15", "w16", "w17", "memory"
1845		);
1846		}
1847		const __aarch64_ldset1_rel = if (is_darwin)
1848		if (has_lse)
1849		ldset1RelDarwinLse
1850		else
1851		ldset1RelDarwinNolse
1852		else if (has_lse)
1853		ldset1RelNondarwinLse
1854		else
1855		ldset1RelNondarwinNolse;
1856		fn cas1AcqRelDarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1857		@setRuntimeSafety(false);
1858		__init_aarch64_have_lse_atomics();
1859
1860		return asm volatile (
1861		\\ cbz w16, 8f
1862		\\ casalb w0, w1, [x2]
1863		\\
1864		\\ cbz wzr, 1f
1865		\\8:
1866		\\ uxtb w16, w0
1867		\\0:
1868		\\ ldaxrb w0, [x2]
1869		\\ cmp w0, w16
1870		\\ bne 1f
1871		\\ stlxrb w17, w1, [x2]
1872		\\ cbnz w17, 0b
1873		\\1:
1874		: [ret] "={w0}" (-> u8),
1875		: [expected] "{w0}" (expected),
1876		[desired] "{w1}" (desired),
1877		[ptr] "{x2}" (ptr),
1878		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1879		: "w15", "w16", "w17", "memory"
1880		);
1881		}
1882		fn cas1AcqRelDarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1883		@setRuntimeSafety(false);
1884		__init_aarch64_have_lse_atomics();
1885
1886		return asm volatile (
1887		\\ cbz w16, 8f
1888		\\ .inst 0x08a07c41 + 0x00000000 + 0x408000
1889		\\
1890		\\ cbz wzr, 1f
1891		\\8:
1892		\\ uxtb w16, w0
1893		\\0:
1894		\\ ldaxrb w0, [x2]
1895		\\ cmp w0, w16
1896		\\ bne 1f
1897		\\ stlxrb w17, w1, [x2]
1898		\\ cbnz w17, 0b
1899		\\1:
1900		: [ret] "={w0}" (-> u8),
1901		: [expected] "{w0}" (expected),
1902		[desired] "{w1}" (desired),
1903		[ptr] "{x2}" (ptr),
1904		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1905		: "w15", "w16", "w17", "memory"
1906		);
1907		}
1908		fn cas1AcqRelNondarwinLse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1909		@setRuntimeSafety(false);
1910		__init_aarch64_have_lse_atomics();
1911
1912		return asm volatile (
1913		\\ cbz w16, 8f
1914		\\ casalb w0, w1, [x2]
1915		\\
1916		\\ cbz wzr, 1f
1917		\\8:
1918		\\ uxtb w16, w0
1919		\\0:
1920		\\ ldaxrb w0, [x2]
1921		\\ cmp w0, w16
1922		\\ bne 1f
1923		\\ stlxrb w17, w1, [x2]
1924		\\ cbnz w17, 0b
1925		\\1:
1926		: [ret] "={w0}" (-> u8),
1927		: [expected] "{w0}" (expected),
1928		[desired] "{w1}" (desired),
1929		[ptr] "{x2}" (ptr),
1930		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1931		: "w15", "w16", "w17", "memory"
1932		);
1933		}
1934		fn cas1AcqRelNondarwinNolse(expected: u8, desired: u8, ptr: *u8) callconv(.C) u8 {
1935		@setRuntimeSafety(false);
1936		__init_aarch64_have_lse_atomics();
1937
1938		return asm volatile (
1939		\\ cbz w16, 8f
1940		\\ .inst 0x08a07c41 + 0x00000000 + 0x408000
1941		\\
1942		\\ cbz wzr, 1f
1943		\\8:
1944		\\ uxtb w16, w0
1945		\\0:
1946		\\ ldaxrb w0, [x2]
1947		\\ cmp w0, w16
1948		\\ bne 1f
1949		\\ stlxrb w17, w1, [x2]
1950		\\ cbnz w17, 0b
1951		\\1:
1952		: [ret] "={w0}" (-> u8),
1953		: [expected] "{w0}" (expected),
1954		[desired] "{w1}" (desired),
1955		[ptr] "{x2}" (ptr),
1956		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1957		: "w15", "w16", "w17", "memory"
1958		);
1959		}
1960		const __aarch64_cas1_acq_rel = if (is_darwin)
1961		if (has_lse)
1962		cas1AcqRelDarwinLse
1963		else
1964		cas1AcqRelDarwinNolse
1965		else if (has_lse)
1966		cas1AcqRelNondarwinLse
1967		else
1968		cas1AcqRelNondarwinNolse;
1969		fn swp1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
1970		@setRuntimeSafety(false);
1971		__init_aarch64_have_lse_atomics();
1972
1973		return asm volatile (
1974		\\ cbz w16, 8f
1975		\\ swpalb w0, w0, [x1]
1976		\\ cbz wzr, 1f
1977		\\8:
1978		\\ mov w16, w0
1979		\\0:
1980		\\ ldaxrb w0, [x1]
1981		\\ stlxrb w17, w16, [x1]
1982		\\ cbnz w17, 0b
1983		\\1:
1984		: [ret] "={w0}" (-> u8),
1985		: [val] "{w0}" (val),
1986		[ptr] "{x1}" (ptr),
1987		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
1988		: "w15", "w16", "w17", "memory"
1989		);
1990		}
1991		fn swp1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
1992		@setRuntimeSafety(false);
1993		__init_aarch64_have_lse_atomics();
1994
1995		return asm volatile (
1996		\\ cbz w16, 8f
1997		\\ .inst 0x38208020 + 0x00000000 + 0xc00000
1998		\\ cbz wzr, 1f
1999		\\8:
2000		\\ mov w16, w0
2001		\\0:
2002		\\ ldaxrb w0, [x1]
2003		\\ stlxrb w17, w16, [x1]
2004		\\ cbnz w17, 0b
2005		\\1:
2006		: [ret] "={w0}" (-> u8),
2007		: [val] "{w0}" (val),
2008		[ptr] "{x1}" (ptr),
2009		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2010		: "w15", "w16", "w17", "memory"
2011		);
2012		}
2013		fn swp1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2014		@setRuntimeSafety(false);
2015		__init_aarch64_have_lse_atomics();
2016
2017		return asm volatile (
2018		\\ cbz w16, 8f
2019		\\ swpalb w0, w0, [x1]
2020		\\ cbz wzr, 1f
2021		\\8:
2022		\\ mov w16, w0
2023		\\0:
2024		\\ ldaxrb w0, [x1]
2025		\\ stlxrb w17, w16, [x1]
2026		\\ cbnz w17, 0b
2027		\\1:
2028		: [ret] "={w0}" (-> u8),
2029		: [val] "{w0}" (val),
2030		[ptr] "{x1}" (ptr),
2031		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2032		: "w15", "w16", "w17", "memory"
2033		);
2034		}
2035		fn swp1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2036		@setRuntimeSafety(false);
2037		__init_aarch64_have_lse_atomics();
2038
2039		return asm volatile (
2040		\\ cbz w16, 8f
2041		\\ .inst 0x38208020 + 0x00000000 + 0xc00000
2042		\\ cbz wzr, 1f
2043		\\8:
2044		\\ mov w16, w0
2045		\\0:
2046		\\ ldaxrb w0, [x1]
2047		\\ stlxrb w17, w16, [x1]
2048		\\ cbnz w17, 0b
2049		\\1:
2050		: [ret] "={w0}" (-> u8),
2051		: [val] "{w0}" (val),
2052		[ptr] "{x1}" (ptr),
2053		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2054		: "w15", "w16", "w17", "memory"
2055		);
2056		}
2057		const __aarch64_swp1_acq_rel = if (is_darwin)
2058		if (has_lse)
2059		swp1AcqRelDarwinLse
2060		else
2061		swp1AcqRelDarwinNolse
2062		else if (has_lse)
2063		swp1AcqRelNondarwinLse
2064		else
2065		swp1AcqRelNondarwinNolse;
2066		fn ldadd1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2067		@setRuntimeSafety(false);
2068		__init_aarch64_have_lse_atomics();
2069
2070		return asm volatile (
2071		\\ cbz w16, 8f
2072		\\ ldaddalb w0, w0, [x1]
2073		\\ cbz wzr, 1f
2074		\\8:
2075		\\ mov w16, w0
2076		\\0:
2077		\\ ldaxrb w0, [x1]
2078		\\ add w17, w0, w16
2079		\\ stlxrb w15, w17, [x1]
2080		\\ cbnz w15, 0b
2081		\\1:
2082		: [ret] "={w0}" (-> u8),
2083		: [val] "{w0}" (val),
2084		[ptr] "{x1}" (ptr),
2085		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2086		: "w15", "w16", "w17", "memory"
2087		);
2088		}
2089		fn ldadd1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2090		@setRuntimeSafety(false);
2091		__init_aarch64_have_lse_atomics();
2092
2093		return asm volatile (
2094		\\ cbz w16, 8f
2095		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
2096		\\ cbz wzr, 1f
2097		\\8:
2098		\\ mov w16, w0
2099		\\0:
2100		\\ ldaxrb w0, [x1]
2101		\\ add w17, w0, w16
2102		\\ stlxrb w15, w17, [x1]
2103		\\ cbnz w15, 0b
2104		\\1:
2105		: [ret] "={w0}" (-> u8),
2106		: [val] "{w0}" (val),
2107		[ptr] "{x1}" (ptr),
2108		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2109		: "w15", "w16", "w17", "memory"
2110		);
2111		}
2112		fn ldadd1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2113		@setRuntimeSafety(false);
2114		__init_aarch64_have_lse_atomics();
2115
2116		return asm volatile (
2117		\\ cbz w16, 8f
2118		\\ ldaddalb w0, w0, [x1]
2119		\\ cbz wzr, 1f
2120		\\8:
2121		\\ mov w16, w0
2122		\\0:
2123		\\ ldaxrb w0, [x1]
2124		\\ add w17, w0, w16
2125		\\ stlxrb w15, w17, [x1]
2126		\\ cbnz w15, 0b
2127		\\1:
2128		: [ret] "={w0}" (-> u8),
2129		: [val] "{w0}" (val),
2130		[ptr] "{x1}" (ptr),
2131		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2132		: "w15", "w16", "w17", "memory"
2133		);
2134		}
2135		fn ldadd1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2136		@setRuntimeSafety(false);
2137		__init_aarch64_have_lse_atomics();
2138
2139		return asm volatile (
2140		\\ cbz w16, 8f
2141		\\ .inst 0x38200020 + 0x0000 + 0x00000000 + 0xc00000
2142		\\ cbz wzr, 1f
2143		\\8:
2144		\\ mov w16, w0
2145		\\0:
2146		\\ ldaxrb w0, [x1]
2147		\\ add w17, w0, w16
2148		\\ stlxrb w15, w17, [x1]
2149		\\ cbnz w15, 0b
2150		\\1:
2151		: [ret] "={w0}" (-> u8),
2152		: [val] "{w0}" (val),
2153		[ptr] "{x1}" (ptr),
2154		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2155		: "w15", "w16", "w17", "memory"
2156		);
2157		}
2158		const __aarch64_ldadd1_acq_rel = if (is_darwin)
2159		if (has_lse)
2160		ldadd1AcqRelDarwinLse
2161		else
2162		ldadd1AcqRelDarwinNolse
2163		else if (has_lse)
2164		ldadd1AcqRelNondarwinLse
2165		else
2166		ldadd1AcqRelNondarwinNolse;
2167		fn ldclr1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2168		@setRuntimeSafety(false);
2169		__init_aarch64_have_lse_atomics();
2170
2171		return asm volatile (
2172		\\ cbz w16, 8f
2173		\\ ldclralb w0, w0, [x1]
2174		\\ cbz wzr, 1f
2175		\\8:
2176		\\ mov w16, w0
2177		\\0:
2178		\\ ldaxrb w0, [x1]
2179		\\ bic w17, w0, w16
2180		\\ stlxrb w15, w17, [x1]
2181		\\ cbnz w15, 0b
2182		\\1:
2183		: [ret] "={w0}" (-> u8),
2184		: [val] "{w0}" (val),
2185		[ptr] "{x1}" (ptr),
2186		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2187		: "w15", "w16", "w17", "memory"
2188		);
2189		}
2190		fn ldclr1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2191		@setRuntimeSafety(false);
2192		__init_aarch64_have_lse_atomics();
2193
2194		return asm volatile (
2195		\\ cbz w16, 8f
2196		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
2197		\\ cbz wzr, 1f
2198		\\8:
2199		\\ mov w16, w0
2200		\\0:
2201		\\ ldaxrb w0, [x1]
2202		\\ bic w17, w0, w16
2203		\\ stlxrb w15, w17, [x1]
2204		\\ cbnz w15, 0b
2205		\\1:
2206		: [ret] "={w0}" (-> u8),
2207		: [val] "{w0}" (val),
2208		[ptr] "{x1}" (ptr),
2209		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2210		: "w15", "w16", "w17", "memory"
2211		);
2212		}
2213		fn ldclr1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2214		@setRuntimeSafety(false);
2215		__init_aarch64_have_lse_atomics();
2216
2217		return asm volatile (
2218		\\ cbz w16, 8f
2219		\\ ldclralb w0, w0, [x1]
2220		\\ cbz wzr, 1f
2221		\\8:
2222		\\ mov w16, w0
2223		\\0:
2224		\\ ldaxrb w0, [x1]
2225		\\ bic w17, w0, w16
2226		\\ stlxrb w15, w17, [x1]
2227		\\ cbnz w15, 0b
2228		\\1:
2229		: [ret] "={w0}" (-> u8),
2230		: [val] "{w0}" (val),
2231		[ptr] "{x1}" (ptr),
2232		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2233		: "w15", "w16", "w17", "memory"
2234		);
2235		}
2236		fn ldclr1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2237		@setRuntimeSafety(false);
2238		__init_aarch64_have_lse_atomics();
2239
2240		return asm volatile (
2241		\\ cbz w16, 8f
2242		\\ .inst 0x38200020 + 0x1000 + 0x00000000 + 0xc00000
2243		\\ cbz wzr, 1f
2244		\\8:
2245		\\ mov w16, w0
2246		\\0:
2247		\\ ldaxrb w0, [x1]
2248		\\ bic w17, w0, w16
2249		\\ stlxrb w15, w17, [x1]
2250		\\ cbnz w15, 0b
2251		\\1:
2252		: [ret] "={w0}" (-> u8),
2253		: [val] "{w0}" (val),
2254		[ptr] "{x1}" (ptr),
2255		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2256		: "w15", "w16", "w17", "memory"
2257		);
2258		}
2259		const __aarch64_ldclr1_acq_rel = if (is_darwin)
2260		if (has_lse)
2261		ldclr1AcqRelDarwinLse
2262		else
2263		ldclr1AcqRelDarwinNolse
2264		else if (has_lse)
2265		ldclr1AcqRelNondarwinLse
2266		else
2267		ldclr1AcqRelNondarwinNolse;
2268		fn ldeor1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2269		@setRuntimeSafety(false);
2270		__init_aarch64_have_lse_atomics();
2271
2272		return asm volatile (
2273		\\ cbz w16, 8f
2274		\\ ldeoralb w0, w0, [x1]
2275		\\ cbz wzr, 1f
2276		\\8:
2277		\\ mov w16, w0
2278		\\0:
2279		\\ ldaxrb w0, [x1]
2280		\\ eor w17, w0, w16
2281		\\ stlxrb w15, w17, [x1]
2282		\\ cbnz w15, 0b
2283		\\1:
2284		: [ret] "={w0}" (-> u8),
2285		: [val] "{w0}" (val),
2286		[ptr] "{x1}" (ptr),
2287		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2288		: "w15", "w16", "w17", "memory"
2289		);
2290		}
2291		fn ldeor1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2292		@setRuntimeSafety(false);
2293		__init_aarch64_have_lse_atomics();
2294
2295		return asm volatile (
2296		\\ cbz w16, 8f
2297		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
2298		\\ cbz wzr, 1f
2299		\\8:
2300		\\ mov w16, w0
2301		\\0:
2302		\\ ldaxrb w0, [x1]
2303		\\ eor w17, w0, w16
2304		\\ stlxrb w15, w17, [x1]
2305		\\ cbnz w15, 0b
2306		\\1:
2307		: [ret] "={w0}" (-> u8),
2308		: [val] "{w0}" (val),
2309		[ptr] "{x1}" (ptr),
2310		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2311		: "w15", "w16", "w17", "memory"
2312		);
2313		}
2314		fn ldeor1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2315		@setRuntimeSafety(false);
2316		__init_aarch64_have_lse_atomics();
2317
2318		return asm volatile (
2319		\\ cbz w16, 8f
2320		\\ ldeoralb w0, w0, [x1]
2321		\\ cbz wzr, 1f
2322		\\8:
2323		\\ mov w16, w0
2324		\\0:
2325		\\ ldaxrb w0, [x1]
2326		\\ eor w17, w0, w16
2327		\\ stlxrb w15, w17, [x1]
2328		\\ cbnz w15, 0b
2329		\\1:
2330		: [ret] "={w0}" (-> u8),
2331		: [val] "{w0}" (val),
2332		[ptr] "{x1}" (ptr),
2333		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2334		: "w15", "w16", "w17", "memory"
2335		);
2336		}
2337		fn ldeor1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2338		@setRuntimeSafety(false);
2339		__init_aarch64_have_lse_atomics();
2340
2341		return asm volatile (
2342		\\ cbz w16, 8f
2343		\\ .inst 0x38200020 + 0x2000 + 0x00000000 + 0xc00000
2344		\\ cbz wzr, 1f
2345		\\8:
2346		\\ mov w16, w0
2347		\\0:
2348		\\ ldaxrb w0, [x1]
2349		\\ eor w17, w0, w16
2350		\\ stlxrb w15, w17, [x1]
2351		\\ cbnz w15, 0b
2352		\\1:
2353		: [ret] "={w0}" (-> u8),
2354		: [val] "{w0}" (val),
2355		[ptr] "{x1}" (ptr),
2356		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2357		: "w15", "w16", "w17", "memory"
2358		);
2359		}
2360		const __aarch64_ldeor1_acq_rel = if (is_darwin)
2361		if (has_lse)
2362		ldeor1AcqRelDarwinLse
2363		else
2364		ldeor1AcqRelDarwinNolse
2365		else if (has_lse)
2366		ldeor1AcqRelNondarwinLse
2367		else
2368		ldeor1AcqRelNondarwinNolse;
2369		fn ldset1AcqRelDarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2370		@setRuntimeSafety(false);
2371		__init_aarch64_have_lse_atomics();
2372
2373		return asm volatile (
2374		\\ cbz w16, 8f
2375		\\ ldsetalb w0, w0, [x1]
2376		\\ cbz wzr, 1f
2377		\\8:
2378		\\ mov w16, w0
2379		\\0:
2380		\\ ldaxrb w0, [x1]
2381		\\ orr w17, w0, w16
2382		\\ stlxrb w15, w17, [x1]
2383		\\ cbnz w15, 0b
2384		\\1:
2385		: [ret] "={w0}" (-> u8),
2386		: [val] "{w0}" (val),
2387		[ptr] "{x1}" (ptr),
2388		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2389		: "w15", "w16", "w17", "memory"
2390		);
2391		}
2392		fn ldset1AcqRelDarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2393		@setRuntimeSafety(false);
2394		__init_aarch64_have_lse_atomics();
2395
2396		return asm volatile (
2397		\\ cbz w16, 8f
2398		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
2399		\\ cbz wzr, 1f
2400		\\8:
2401		\\ mov w16, w0
2402		\\0:
2403		\\ ldaxrb w0, [x1]
2404		\\ orr w17, w0, w16
2405		\\ stlxrb w15, w17, [x1]
2406		\\ cbnz w15, 0b
2407		\\1:
2408		: [ret] "={w0}" (-> u8),
2409		: [val] "{w0}" (val),
2410		[ptr] "{x1}" (ptr),
2411		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2412		: "w15", "w16", "w17", "memory"
2413		);
2414		}
2415		fn ldset1AcqRelNondarwinLse(val: u8, ptr: *u8) callconv(.C) u8 {
2416		@setRuntimeSafety(false);
2417		__init_aarch64_have_lse_atomics();
2418
2419		return asm volatile (
2420		\\ cbz w16, 8f
2421		\\ ldsetalb w0, w0, [x1]
2422		\\ cbz wzr, 1f
2423		\\8:
2424		\\ mov w16, w0
2425		\\0:
2426		\\ ldaxrb w0, [x1]
2427		\\ orr w17, w0, w16
2428		\\ stlxrb w15, w17, [x1]
2429		\\ cbnz w15, 0b
2430		\\1:
2431		: [ret] "={w0}" (-> u8),
2432		: [val] "{w0}" (val),
2433		[ptr] "{x1}" (ptr),
2434		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2435		: "w15", "w16", "w17", "memory"
2436		);
2437		}
2438		fn ldset1AcqRelNondarwinNolse(val: u8, ptr: *u8) callconv(.C) u8 {
2439		@setRuntimeSafety(false);
2440		__init_aarch64_have_lse_atomics();
2441
2442		return asm volatile (
2443		\\ cbz w16, 8f
2444		\\ .inst 0x38200020 + 0x3000 + 0x00000000 + 0xc00000
2445		\\ cbz wzr, 1f
2446		\\8:
2447		\\ mov w16, w0
2448		\\0:
2449		\\ ldaxrb w0, [x1]
2450		\\ orr w17, w0, w16
2451		\\ stlxrb w15, w17, [x1]
2452		\\ cbnz w15, 0b
2453		\\1:
2454		: [ret] "={w0}" (-> u8),
2455		: [val] "{w0}" (val),
2456		[ptr] "{x1}" (ptr),
2457		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2458		: "w15", "w16", "w17", "memory"
2459		);
2460		}
2461		const __aarch64_ldset1_acq_rel = if (is_darwin)
2462		if (has_lse)
2463		ldset1AcqRelDarwinLse
2464		else
2465		ldset1AcqRelDarwinNolse
2466		else if (has_lse)
2467		ldset1AcqRelNondarwinLse
2468		else
2469		ldset1AcqRelNondarwinNolse;
2470		fn cas2RelaxDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2471		@setRuntimeSafety(false);
2472		__init_aarch64_have_lse_atomics();
2473
2474		return asm volatile (
2475		\\ cbz w16, 8f
2476		\\ cash w0, w1, [x2]
2477		\\
2478		\\ cbz wzr, 1f
2479		\\8:
2480		\\ uxth w16, w0
2481		\\0:
2482		\\ ldxrh w0, [x2]
2483		\\ cmp w0, w16
2484		\\ bne 1f
2485		\\ stxrh w17, w1, [x2]
2486		\\ cbnz w17, 0b
2487		\\1:
2488		: [ret] "={w0}" (-> u16),
2489		: [expected] "{w0}" (expected),
2490		[desired] "{w1}" (desired),
2491		[ptr] "{x2}" (ptr),
2492		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2493		: "w15", "w16", "w17", "memory"
2494		);
2495		}
2496		fn cas2RelaxDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2497		@setRuntimeSafety(false);
2498		__init_aarch64_have_lse_atomics();
2499
2500		return asm volatile (
2501		\\ cbz w16, 8f
2502		\\ .inst 0x08a07c41 + 0x40000000 + 0x000000
2503		\\
2504		\\ cbz wzr, 1f
2505		\\8:
2506		\\ uxth w16, w0
2507		\\0:
2508		\\ ldxrh w0, [x2]
2509		\\ cmp w0, w16
2510		\\ bne 1f
2511		\\ stxrh w17, w1, [x2]
2512		\\ cbnz w17, 0b
2513		\\1:
2514		: [ret] "={w0}" (-> u16),
2515		: [expected] "{w0}" (expected),
2516		[desired] "{w1}" (desired),
2517		[ptr] "{x2}" (ptr),
2518		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2519		: "w15", "w16", "w17", "memory"
2520		);
2521		}
2522		fn cas2RelaxNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2523		@setRuntimeSafety(false);
2524		__init_aarch64_have_lse_atomics();
2525
2526		return asm volatile (
2527		\\ cbz w16, 8f
2528		\\ cash w0, w1, [x2]
2529		\\
2530		\\ cbz wzr, 1f
2531		\\8:
2532		\\ uxth w16, w0
2533		\\0:
2534		\\ ldxrh w0, [x2]
2535		\\ cmp w0, w16
2536		\\ bne 1f
2537		\\ stxrh w17, w1, [x2]
2538		\\ cbnz w17, 0b
2539		\\1:
2540		: [ret] "={w0}" (-> u16),
2541		: [expected] "{w0}" (expected),
2542		[desired] "{w1}" (desired),
2543		[ptr] "{x2}" (ptr),
2544		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2545		: "w15", "w16", "w17", "memory"
2546		);
2547		}
2548		fn cas2RelaxNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
2549		@setRuntimeSafety(false);
2550		__init_aarch64_have_lse_atomics();
2551
2552		return asm volatile (
2553		\\ cbz w16, 8f
2554		\\ .inst 0x08a07c41 + 0x40000000 + 0x000000
2555		\\
2556		\\ cbz wzr, 1f
2557		\\8:
2558		\\ uxth w16, w0
2559		\\0:
2560		\\ ldxrh w0, [x2]
2561		\\ cmp w0, w16
2562		\\ bne 1f
2563		\\ stxrh w17, w1, [x2]
2564		\\ cbnz w17, 0b
2565		\\1:
2566		: [ret] "={w0}" (-> u16),
2567		: [expected] "{w0}" (expected),
2568		[desired] "{w1}" (desired),
2569		[ptr] "{x2}" (ptr),
2570		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2571		: "w15", "w16", "w17", "memory"
2572		);
2573		}
2574		const __aarch64_cas2_relax = if (is_darwin)
2575		if (has_lse)
2576		cas2RelaxDarwinLse
2577		else
2578		cas2RelaxDarwinNolse
2579		else if (has_lse)
2580		cas2RelaxNondarwinLse
2581		else
2582		cas2RelaxNondarwinNolse;
2583		fn swp2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2584		@setRuntimeSafety(false);
2585		__init_aarch64_have_lse_atomics();
2586
2587		return asm volatile (
2588		\\ cbz w16, 8f
2589		\\ swph w0, w0, [x1]
2590		\\ cbz wzr, 1f
2591		\\8:
2592		\\ mov w16, w0
2593		\\0:
2594		\\ ldxrh w0, [x1]
2595		\\ stxrh w17, w16, [x1]
2596		\\ cbnz w17, 0b
2597		\\1:
2598		: [ret] "={w0}" (-> u16),
2599		: [val] "{w0}" (val),
2600		[ptr] "{x1}" (ptr),
2601		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2602		: "w15", "w16", "w17", "memory"
2603		);
2604		}
2605		fn swp2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2606		@setRuntimeSafety(false);
2607		__init_aarch64_have_lse_atomics();
2608
2609		return asm volatile (
2610		\\ cbz w16, 8f
2611		\\ .inst 0x38208020 + 0x40000000 + 0x000000
2612		\\ cbz wzr, 1f
2613		\\8:
2614		\\ mov w16, w0
2615		\\0:
2616		\\ ldxrh w0, [x1]
2617		\\ stxrh w17, w16, [x1]
2618		\\ cbnz w17, 0b
2619		\\1:
2620		: [ret] "={w0}" (-> u16),
2621		: [val] "{w0}" (val),
2622		[ptr] "{x1}" (ptr),
2623		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2624		: "w15", "w16", "w17", "memory"
2625		);
2626		}
2627		fn swp2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2628		@setRuntimeSafety(false);
2629		__init_aarch64_have_lse_atomics();
2630
2631		return asm volatile (
2632		\\ cbz w16, 8f
2633		\\ swph w0, w0, [x1]
2634		\\ cbz wzr, 1f
2635		\\8:
2636		\\ mov w16, w0
2637		\\0:
2638		\\ ldxrh w0, [x1]
2639		\\ stxrh w17, w16, [x1]
2640		\\ cbnz w17, 0b
2641		\\1:
2642		: [ret] "={w0}" (-> u16),
2643		: [val] "{w0}" (val),
2644		[ptr] "{x1}" (ptr),
2645		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2646		: "w15", "w16", "w17", "memory"
2647		);
2648		}
2649		fn swp2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2650		@setRuntimeSafety(false);
2651		__init_aarch64_have_lse_atomics();
2652
2653		return asm volatile (
2654		\\ cbz w16, 8f
2655		\\ .inst 0x38208020 + 0x40000000 + 0x000000
2656		\\ cbz wzr, 1f
2657		\\8:
2658		\\ mov w16, w0
2659		\\0:
2660		\\ ldxrh w0, [x1]
2661		\\ stxrh w17, w16, [x1]
2662		\\ cbnz w17, 0b
2663		\\1:
2664		: [ret] "={w0}" (-> u16),
2665		: [val] "{w0}" (val),
2666		[ptr] "{x1}" (ptr),
2667		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2668		: "w15", "w16", "w17", "memory"
2669		);
2670		}
2671		const __aarch64_swp2_relax = if (is_darwin)
2672		if (has_lse)
2673		swp2RelaxDarwinLse
2674		else
2675		swp2RelaxDarwinNolse
2676		else if (has_lse)
2677		swp2RelaxNondarwinLse
2678		else
2679		swp2RelaxNondarwinNolse;
2680		fn ldadd2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2681		@setRuntimeSafety(false);
2682		__init_aarch64_have_lse_atomics();
2683
2684		return asm volatile (
2685		\\ cbz w16, 8f
2686		\\ ldaddh w0, w0, [x1]
2687		\\ cbz wzr, 1f
2688		\\8:
2689		\\ mov w16, w0
2690		\\0:
2691		\\ ldxrh w0, [x1]
2692		\\ add w17, w0, w16
2693		\\ stxrh w15, w17, [x1]
2694		\\ cbnz w15, 0b
2695		\\1:
2696		: [ret] "={w0}" (-> u16),
2697		: [val] "{w0}" (val),
2698		[ptr] "{x1}" (ptr),
2699		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2700		: "w15", "w16", "w17", "memory"
2701		);
2702		}
2703		fn ldadd2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2704		@setRuntimeSafety(false);
2705		__init_aarch64_have_lse_atomics();
2706
2707		return asm volatile (
2708		\\ cbz w16, 8f
2709		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
2710		\\ cbz wzr, 1f
2711		\\8:
2712		\\ mov w16, w0
2713		\\0:
2714		\\ ldxrh w0, [x1]
2715		\\ add w17, w0, w16
2716		\\ stxrh w15, w17, [x1]
2717		\\ cbnz w15, 0b
2718		\\1:
2719		: [ret] "={w0}" (-> u16),
2720		: [val] "{w0}" (val),
2721		[ptr] "{x1}" (ptr),
2722		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2723		: "w15", "w16", "w17", "memory"
2724		);
2725		}
2726		fn ldadd2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2727		@setRuntimeSafety(false);
2728		__init_aarch64_have_lse_atomics();
2729
2730		return asm volatile (
2731		\\ cbz w16, 8f
2732		\\ ldaddh w0, w0, [x1]
2733		\\ cbz wzr, 1f
2734		\\8:
2735		\\ mov w16, w0
2736		\\0:
2737		\\ ldxrh w0, [x1]
2738		\\ add w17, w0, w16
2739		\\ stxrh w15, w17, [x1]
2740		\\ cbnz w15, 0b
2741		\\1:
2742		: [ret] "={w0}" (-> u16),
2743		: [val] "{w0}" (val),
2744		[ptr] "{x1}" (ptr),
2745		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2746		: "w15", "w16", "w17", "memory"
2747		);
2748		}
2749		fn ldadd2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2750		@setRuntimeSafety(false);
2751		__init_aarch64_have_lse_atomics();
2752
2753		return asm volatile (
2754		\\ cbz w16, 8f
2755		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x000000
2756		\\ cbz wzr, 1f
2757		\\8:
2758		\\ mov w16, w0
2759		\\0:
2760		\\ ldxrh w0, [x1]
2761		\\ add w17, w0, w16
2762		\\ stxrh w15, w17, [x1]
2763		\\ cbnz w15, 0b
2764		\\1:
2765		: [ret] "={w0}" (-> u16),
2766		: [val] "{w0}" (val),
2767		[ptr] "{x1}" (ptr),
2768		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2769		: "w15", "w16", "w17", "memory"
2770		);
2771		}
2772		const __aarch64_ldadd2_relax = if (is_darwin)
2773		if (has_lse)
2774		ldadd2RelaxDarwinLse
2775		else
2776		ldadd2RelaxDarwinNolse
2777		else if (has_lse)
2778		ldadd2RelaxNondarwinLse
2779		else
2780		ldadd2RelaxNondarwinNolse;
2781		fn ldclr2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2782		@setRuntimeSafety(false);
2783		__init_aarch64_have_lse_atomics();
2784
2785		return asm volatile (
2786		\\ cbz w16, 8f
2787		\\ ldclrh w0, w0, [x1]
2788		\\ cbz wzr, 1f
2789		\\8:
2790		\\ mov w16, w0
2791		\\0:
2792		\\ ldxrh w0, [x1]
2793		\\ bic w17, w0, w16
2794		\\ stxrh w15, w17, [x1]
2795		\\ cbnz w15, 0b
2796		\\1:
2797		: [ret] "={w0}" (-> u16),
2798		: [val] "{w0}" (val),
2799		[ptr] "{x1}" (ptr),
2800		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2801		: "w15", "w16", "w17", "memory"
2802		);
2803		}
2804		fn ldclr2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2805		@setRuntimeSafety(false);
2806		__init_aarch64_have_lse_atomics();
2807
2808		return asm volatile (
2809		\\ cbz w16, 8f
2810		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
2811		\\ cbz wzr, 1f
2812		\\8:
2813		\\ mov w16, w0
2814		\\0:
2815		\\ ldxrh w0, [x1]
2816		\\ bic w17, w0, w16
2817		\\ stxrh w15, w17, [x1]
2818		\\ cbnz w15, 0b
2819		\\1:
2820		: [ret] "={w0}" (-> u16),
2821		: [val] "{w0}" (val),
2822		[ptr] "{x1}" (ptr),
2823		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2824		: "w15", "w16", "w17", "memory"
2825		);
2826		}
2827		fn ldclr2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2828		@setRuntimeSafety(false);
2829		__init_aarch64_have_lse_atomics();
2830
2831		return asm volatile (
2832		\\ cbz w16, 8f
2833		\\ ldclrh w0, w0, [x1]
2834		\\ cbz wzr, 1f
2835		\\8:
2836		\\ mov w16, w0
2837		\\0:
2838		\\ ldxrh w0, [x1]
2839		\\ bic w17, w0, w16
2840		\\ stxrh w15, w17, [x1]
2841		\\ cbnz w15, 0b
2842		\\1:
2843		: [ret] "={w0}" (-> u16),
2844		: [val] "{w0}" (val),
2845		[ptr] "{x1}" (ptr),
2846		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2847		: "w15", "w16", "w17", "memory"
2848		);
2849		}
2850		fn ldclr2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2851		@setRuntimeSafety(false);
2852		__init_aarch64_have_lse_atomics();
2853
2854		return asm volatile (
2855		\\ cbz w16, 8f
2856		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x000000
2857		\\ cbz wzr, 1f
2858		\\8:
2859		\\ mov w16, w0
2860		\\0:
2861		\\ ldxrh w0, [x1]
2862		\\ bic w17, w0, w16
2863		\\ stxrh w15, w17, [x1]
2864		\\ cbnz w15, 0b
2865		\\1:
2866		: [ret] "={w0}" (-> u16),
2867		: [val] "{w0}" (val),
2868		[ptr] "{x1}" (ptr),
2869		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2870		: "w15", "w16", "w17", "memory"
2871		);
2872		}
2873		const __aarch64_ldclr2_relax = if (is_darwin)
2874		if (has_lse)
2875		ldclr2RelaxDarwinLse
2876		else
2877		ldclr2RelaxDarwinNolse
2878		else if (has_lse)
2879		ldclr2RelaxNondarwinLse
2880		else
2881		ldclr2RelaxNondarwinNolse;
2882		fn ldeor2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2883		@setRuntimeSafety(false);
2884		__init_aarch64_have_lse_atomics();
2885
2886		return asm volatile (
2887		\\ cbz w16, 8f
2888		\\ ldeorh w0, w0, [x1]
2889		\\ cbz wzr, 1f
2890		\\8:
2891		\\ mov w16, w0
2892		\\0:
2893		\\ ldxrh w0, [x1]
2894		\\ eor w17, w0, w16
2895		\\ stxrh w15, w17, [x1]
2896		\\ cbnz w15, 0b
2897		\\1:
2898		: [ret] "={w0}" (-> u16),
2899		: [val] "{w0}" (val),
2900		[ptr] "{x1}" (ptr),
2901		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2902		: "w15", "w16", "w17", "memory"
2903		);
2904		}
2905		fn ldeor2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2906		@setRuntimeSafety(false);
2907		__init_aarch64_have_lse_atomics();
2908
2909		return asm volatile (
2910		\\ cbz w16, 8f
2911		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
2912		\\ cbz wzr, 1f
2913		\\8:
2914		\\ mov w16, w0
2915		\\0:
2916		\\ ldxrh w0, [x1]
2917		\\ eor w17, w0, w16
2918		\\ stxrh w15, w17, [x1]
2919		\\ cbnz w15, 0b
2920		\\1:
2921		: [ret] "={w0}" (-> u16),
2922		: [val] "{w0}" (val),
2923		[ptr] "{x1}" (ptr),
2924		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2925		: "w15", "w16", "w17", "memory"
2926		);
2927		}
2928		fn ldeor2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2929		@setRuntimeSafety(false);
2930		__init_aarch64_have_lse_atomics();
2931
2932		return asm volatile (
2933		\\ cbz w16, 8f
2934		\\ ldeorh w0, w0, [x1]
2935		\\ cbz wzr, 1f
2936		\\8:
2937		\\ mov w16, w0
2938		\\0:
2939		\\ ldxrh w0, [x1]
2940		\\ eor w17, w0, w16
2941		\\ stxrh w15, w17, [x1]
2942		\\ cbnz w15, 0b
2943		\\1:
2944		: [ret] "={w0}" (-> u16),
2945		: [val] "{w0}" (val),
2946		[ptr] "{x1}" (ptr),
2947		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2948		: "w15", "w16", "w17", "memory"
2949		);
2950		}
2951		fn ldeor2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
2952		@setRuntimeSafety(false);
2953		__init_aarch64_have_lse_atomics();
2954
2955		return asm volatile (
2956		\\ cbz w16, 8f
2957		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x000000
2958		\\ cbz wzr, 1f
2959		\\8:
2960		\\ mov w16, w0
2961		\\0:
2962		\\ ldxrh w0, [x1]
2963		\\ eor w17, w0, w16
2964		\\ stxrh w15, w17, [x1]
2965		\\ cbnz w15, 0b
2966		\\1:
2967		: [ret] "={w0}" (-> u16),
2968		: [val] "{w0}" (val),
2969		[ptr] "{x1}" (ptr),
2970		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
2971		: "w15", "w16", "w17", "memory"
2972		);
2973		}
2974		const __aarch64_ldeor2_relax = if (is_darwin)
2975		if (has_lse)
2976		ldeor2RelaxDarwinLse
2977		else
2978		ldeor2RelaxDarwinNolse
2979		else if (has_lse)
2980		ldeor2RelaxNondarwinLse
2981		else
2982		ldeor2RelaxNondarwinNolse;
2983		fn ldset2RelaxDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
2984		@setRuntimeSafety(false);
2985		__init_aarch64_have_lse_atomics();
2986
2987		return asm volatile (
2988		\\ cbz w16, 8f
2989		\\ ldseth w0, w0, [x1]
2990		\\ cbz wzr, 1f
2991		\\8:
2992		\\ mov w16, w0
2993		\\0:
2994		\\ ldxrh w0, [x1]
2995		\\ orr w17, w0, w16
2996		\\ stxrh w15, w17, [x1]
2997		\\ cbnz w15, 0b
2998		\\1:
2999		: [ret] "={w0}" (-> u16),
3000		: [val] "{w0}" (val),
3001		[ptr] "{x1}" (ptr),
3002		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3003		: "w15", "w16", "w17", "memory"
3004		);
3005		}
3006		fn ldset2RelaxDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3007		@setRuntimeSafety(false);
3008		__init_aarch64_have_lse_atomics();
3009
3010		return asm volatile (
3011		\\ cbz w16, 8f
3012		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
3013		\\ cbz wzr, 1f
3014		\\8:
3015		\\ mov w16, w0
3016		\\0:
3017		\\ ldxrh w0, [x1]
3018		\\ orr w17, w0, w16
3019		\\ stxrh w15, w17, [x1]
3020		\\ cbnz w15, 0b
3021		\\1:
3022		: [ret] "={w0}" (-> u16),
3023		: [val] "{w0}" (val),
3024		[ptr] "{x1}" (ptr),
3025		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3026		: "w15", "w16", "w17", "memory"
3027		);
3028		}
3029		fn ldset2RelaxNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3030		@setRuntimeSafety(false);
3031		__init_aarch64_have_lse_atomics();
3032
3033		return asm volatile (
3034		\\ cbz w16, 8f
3035		\\ ldseth w0, w0, [x1]
3036		\\ cbz wzr, 1f
3037		\\8:
3038		\\ mov w16, w0
3039		\\0:
3040		\\ ldxrh w0, [x1]
3041		\\ orr w17, w0, w16
3042		\\ stxrh w15, w17, [x1]
3043		\\ cbnz w15, 0b
3044		\\1:
3045		: [ret] "={w0}" (-> u16),
3046		: [val] "{w0}" (val),
3047		[ptr] "{x1}" (ptr),
3048		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3049		: "w15", "w16", "w17", "memory"
3050		);
3051		}
3052		fn ldset2RelaxNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3053		@setRuntimeSafety(false);
3054		__init_aarch64_have_lse_atomics();
3055
3056		return asm volatile (
3057		\\ cbz w16, 8f
3058		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x000000
3059		\\ cbz wzr, 1f
3060		\\8:
3061		\\ mov w16, w0
3062		\\0:
3063		\\ ldxrh w0, [x1]
3064		\\ orr w17, w0, w16
3065		\\ stxrh w15, w17, [x1]
3066		\\ cbnz w15, 0b
3067		\\1:
3068		: [ret] "={w0}" (-> u16),
3069		: [val] "{w0}" (val),
3070		[ptr] "{x1}" (ptr),
3071		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3072		: "w15", "w16", "w17", "memory"
3073		);
3074		}
3075		const __aarch64_ldset2_relax = if (is_darwin)
3076		if (has_lse)
3077		ldset2RelaxDarwinLse
3078		else
3079		ldset2RelaxDarwinNolse
3080		else if (has_lse)
3081		ldset2RelaxNondarwinLse
3082		else
3083		ldset2RelaxNondarwinNolse;
3084		fn cas2AcqDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3085		@setRuntimeSafety(false);
3086		__init_aarch64_have_lse_atomics();
3087
3088		return asm volatile (
3089		\\ cbz w16, 8f
3090		\\ casah w0, w1, [x2]
3091		\\
3092		\\ cbz wzr, 1f
3093		\\8:
3094		\\ uxth w16, w0
3095		\\0:
3096		\\ ldaxrh w0, [x2]
3097		\\ cmp w0, w16
3098		\\ bne 1f
3099		\\ stxrh w17, w1, [x2]
3100		\\ cbnz w17, 0b
3101		\\1:
3102		: [ret] "={w0}" (-> u16),
3103		: [expected] "{w0}" (expected),
3104		[desired] "{w1}" (desired),
3105		[ptr] "{x2}" (ptr),
3106		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3107		: "w15", "w16", "w17", "memory"
3108		);
3109		}
3110		fn cas2AcqDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3111		@setRuntimeSafety(false);
3112		__init_aarch64_have_lse_atomics();
3113
3114		return asm volatile (
3115		\\ cbz w16, 8f
3116		\\ .inst 0x08a07c41 + 0x40000000 + 0x400000
3117		\\
3118		\\ cbz wzr, 1f
3119		\\8:
3120		\\ uxth w16, w0
3121		\\0:
3122		\\ ldaxrh w0, [x2]
3123		\\ cmp w0, w16
3124		\\ bne 1f
3125		\\ stxrh w17, w1, [x2]
3126		\\ cbnz w17, 0b
3127		\\1:
3128		: [ret] "={w0}" (-> u16),
3129		: [expected] "{w0}" (expected),
3130		[desired] "{w1}" (desired),
3131		[ptr] "{x2}" (ptr),
3132		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3133		: "w15", "w16", "w17", "memory"
3134		);
3135		}
3136		fn cas2AcqNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3137		@setRuntimeSafety(false);
3138		__init_aarch64_have_lse_atomics();
3139
3140		return asm volatile (
3141		\\ cbz w16, 8f
3142		\\ casah w0, w1, [x2]
3143		\\
3144		\\ cbz wzr, 1f
3145		\\8:
3146		\\ uxth w16, w0
3147		\\0:
3148		\\ ldaxrh w0, [x2]
3149		\\ cmp w0, w16
3150		\\ bne 1f
3151		\\ stxrh w17, w1, [x2]
3152		\\ cbnz w17, 0b
3153		\\1:
3154		: [ret] "={w0}" (-> u16),
3155		: [expected] "{w0}" (expected),
3156		[desired] "{w1}" (desired),
3157		[ptr] "{x2}" (ptr),
3158		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3159		: "w15", "w16", "w17", "memory"
3160		);
3161		}
3162		fn cas2AcqNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3163		@setRuntimeSafety(false);
3164		__init_aarch64_have_lse_atomics();
3165
3166		return asm volatile (
3167		\\ cbz w16, 8f
3168		\\ .inst 0x08a07c41 + 0x40000000 + 0x400000
3169		\\
3170		\\ cbz wzr, 1f
3171		\\8:
3172		\\ uxth w16, w0
3173		\\0:
3174		\\ ldaxrh w0, [x2]
3175		\\ cmp w0, w16
3176		\\ bne 1f
3177		\\ stxrh w17, w1, [x2]
3178		\\ cbnz w17, 0b
3179		\\1:
3180		: [ret] "={w0}" (-> u16),
3181		: [expected] "{w0}" (expected),
3182		[desired] "{w1}" (desired),
3183		[ptr] "{x2}" (ptr),
3184		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3185		: "w15", "w16", "w17", "memory"
3186		);
3187		}
3188		const __aarch64_cas2_acq = if (is_darwin)
3189		if (has_lse)
3190		cas2AcqDarwinLse
3191		else
3192		cas2AcqDarwinNolse
3193		else if (has_lse)
3194		cas2AcqNondarwinLse
3195		else
3196		cas2AcqNondarwinNolse;
3197		fn swp2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3198		@setRuntimeSafety(false);
3199		__init_aarch64_have_lse_atomics();
3200
3201		return asm volatile (
3202		\\ cbz w16, 8f
3203		\\ swpah w0, w0, [x1]
3204		\\ cbz wzr, 1f
3205		\\8:
3206		\\ mov w16, w0
3207		\\0:
3208		\\ ldaxrh w0, [x1]
3209		\\ stxrh w17, w16, [x1]
3210		\\ cbnz w17, 0b
3211		\\1:
3212		: [ret] "={w0}" (-> u16),
3213		: [val] "{w0}" (val),
3214		[ptr] "{x1}" (ptr),
3215		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3216		: "w15", "w16", "w17", "memory"
3217		);
3218		}
3219		fn swp2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3220		@setRuntimeSafety(false);
3221		__init_aarch64_have_lse_atomics();
3222
3223		return asm volatile (
3224		\\ cbz w16, 8f
3225		\\ .inst 0x38208020 + 0x40000000 + 0x800000
3226		\\ cbz wzr, 1f
3227		\\8:
3228		\\ mov w16, w0
3229		\\0:
3230		\\ ldaxrh w0, [x1]
3231		\\ stxrh w17, w16, [x1]
3232		\\ cbnz w17, 0b
3233		\\1:
3234		: [ret] "={w0}" (-> u16),
3235		: [val] "{w0}" (val),
3236		[ptr] "{x1}" (ptr),
3237		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3238		: "w15", "w16", "w17", "memory"
3239		);
3240		}
3241		fn swp2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3242		@setRuntimeSafety(false);
3243		__init_aarch64_have_lse_atomics();
3244
3245		return asm volatile (
3246		\\ cbz w16, 8f
3247		\\ swpah w0, w0, [x1]
3248		\\ cbz wzr, 1f
3249		\\8:
3250		\\ mov w16, w0
3251		\\0:
3252		\\ ldaxrh w0, [x1]
3253		\\ stxrh w17, w16, [x1]
3254		\\ cbnz w17, 0b
3255		\\1:
3256		: [ret] "={w0}" (-> u16),
3257		: [val] "{w0}" (val),
3258		[ptr] "{x1}" (ptr),
3259		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3260		: "w15", "w16", "w17", "memory"
3261		);
3262		}
3263		fn swp2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3264		@setRuntimeSafety(false);
3265		__init_aarch64_have_lse_atomics();
3266
3267		return asm volatile (
3268		\\ cbz w16, 8f
3269		\\ .inst 0x38208020 + 0x40000000 + 0x800000
3270		\\ cbz wzr, 1f
3271		\\8:
3272		\\ mov w16, w0
3273		\\0:
3274		\\ ldaxrh w0, [x1]
3275		\\ stxrh w17, w16, [x1]
3276		\\ cbnz w17, 0b
3277		\\1:
3278		: [ret] "={w0}" (-> u16),
3279		: [val] "{w0}" (val),
3280		[ptr] "{x1}" (ptr),
3281		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3282		: "w15", "w16", "w17", "memory"
3283		);
3284		}
3285		const __aarch64_swp2_acq = if (is_darwin)
3286		if (has_lse)
3287		swp2AcqDarwinLse
3288		else
3289		swp2AcqDarwinNolse
3290		else if (has_lse)
3291		swp2AcqNondarwinLse
3292		else
3293		swp2AcqNondarwinNolse;
3294		fn ldadd2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3295		@setRuntimeSafety(false);
3296		__init_aarch64_have_lse_atomics();
3297
3298		return asm volatile (
3299		\\ cbz w16, 8f
3300		\\ ldaddah w0, w0, [x1]
3301		\\ cbz wzr, 1f
3302		\\8:
3303		\\ mov w16, w0
3304		\\0:
3305		\\ ldaxrh w0, [x1]
3306		\\ add w17, w0, w16
3307		\\ stxrh w15, w17, [x1]
3308		\\ cbnz w15, 0b
3309		\\1:
3310		: [ret] "={w0}" (-> u16),
3311		: [val] "{w0}" (val),
3312		[ptr] "{x1}" (ptr),
3313		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3314		: "w15", "w16", "w17", "memory"
3315		);
3316		}
3317		fn ldadd2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3318		@setRuntimeSafety(false);
3319		__init_aarch64_have_lse_atomics();
3320
3321		return asm volatile (
3322		\\ cbz w16, 8f
3323		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
3324		\\ cbz wzr, 1f
3325		\\8:
3326		\\ mov w16, w0
3327		\\0:
3328		\\ ldaxrh w0, [x1]
3329		\\ add w17, w0, w16
3330		\\ stxrh w15, w17, [x1]
3331		\\ cbnz w15, 0b
3332		\\1:
3333		: [ret] "={w0}" (-> u16),
3334		: [val] "{w0}" (val),
3335		[ptr] "{x1}" (ptr),
3336		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3337		: "w15", "w16", "w17", "memory"
3338		);
3339		}
3340		fn ldadd2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3341		@setRuntimeSafety(false);
3342		__init_aarch64_have_lse_atomics();
3343
3344		return asm volatile (
3345		\\ cbz w16, 8f
3346		\\ ldaddah w0, w0, [x1]
3347		\\ cbz wzr, 1f
3348		\\8:
3349		\\ mov w16, w0
3350		\\0:
3351		\\ ldaxrh w0, [x1]
3352		\\ add w17, w0, w16
3353		\\ stxrh w15, w17, [x1]
3354		\\ cbnz w15, 0b
3355		\\1:
3356		: [ret] "={w0}" (-> u16),
3357		: [val] "{w0}" (val),
3358		[ptr] "{x1}" (ptr),
3359		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3360		: "w15", "w16", "w17", "memory"
3361		);
3362		}
3363		fn ldadd2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3364		@setRuntimeSafety(false);
3365		__init_aarch64_have_lse_atomics();
3366
3367		return asm volatile (
3368		\\ cbz w16, 8f
3369		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x800000
3370		\\ cbz wzr, 1f
3371		\\8:
3372		\\ mov w16, w0
3373		\\0:
3374		\\ ldaxrh w0, [x1]
3375		\\ add w17, w0, w16
3376		\\ stxrh w15, w17, [x1]
3377		\\ cbnz w15, 0b
3378		\\1:
3379		: [ret] "={w0}" (-> u16),
3380		: [val] "{w0}" (val),
3381		[ptr] "{x1}" (ptr),
3382		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3383		: "w15", "w16", "w17", "memory"
3384		);
3385		}
3386		const __aarch64_ldadd2_acq = if (is_darwin)
3387		if (has_lse)
3388		ldadd2AcqDarwinLse
3389		else
3390		ldadd2AcqDarwinNolse
3391		else if (has_lse)
3392		ldadd2AcqNondarwinLse
3393		else
3394		ldadd2AcqNondarwinNolse;
3395		fn ldclr2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3396		@setRuntimeSafety(false);
3397		__init_aarch64_have_lse_atomics();
3398
3399		return asm volatile (
3400		\\ cbz w16, 8f
3401		\\ ldclrah w0, w0, [x1]
3402		\\ cbz wzr, 1f
3403		\\8:
3404		\\ mov w16, w0
3405		\\0:
3406		\\ ldaxrh w0, [x1]
3407		\\ bic w17, w0, w16
3408		\\ stxrh w15, w17, [x1]
3409		\\ cbnz w15, 0b
3410		\\1:
3411		: [ret] "={w0}" (-> u16),
3412		: [val] "{w0}" (val),
3413		[ptr] "{x1}" (ptr),
3414		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3415		: "w15", "w16", "w17", "memory"
3416		);
3417		}
3418		fn ldclr2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3419		@setRuntimeSafety(false);
3420		__init_aarch64_have_lse_atomics();
3421
3422		return asm volatile (
3423		\\ cbz w16, 8f
3424		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
3425		\\ cbz wzr, 1f
3426		\\8:
3427		\\ mov w16, w0
3428		\\0:
3429		\\ ldaxrh w0, [x1]
3430		\\ bic w17, w0, w16
3431		\\ stxrh w15, w17, [x1]
3432		\\ cbnz w15, 0b
3433		\\1:
3434		: [ret] "={w0}" (-> u16),
3435		: [val] "{w0}" (val),
3436		[ptr] "{x1}" (ptr),
3437		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3438		: "w15", "w16", "w17", "memory"
3439		);
3440		}
3441		fn ldclr2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3442		@setRuntimeSafety(false);
3443		__init_aarch64_have_lse_atomics();
3444
3445		return asm volatile (
3446		\\ cbz w16, 8f
3447		\\ ldclrah w0, w0, [x1]
3448		\\ cbz wzr, 1f
3449		\\8:
3450		\\ mov w16, w0
3451		\\0:
3452		\\ ldaxrh w0, [x1]
3453		\\ bic w17, w0, w16
3454		\\ stxrh w15, w17, [x1]
3455		\\ cbnz w15, 0b
3456		\\1:
3457		: [ret] "={w0}" (-> u16),
3458		: [val] "{w0}" (val),
3459		[ptr] "{x1}" (ptr),
3460		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3461		: "w15", "w16", "w17", "memory"
3462		);
3463		}
3464		fn ldclr2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3465		@setRuntimeSafety(false);
3466		__init_aarch64_have_lse_atomics();
3467
3468		return asm volatile (
3469		\\ cbz w16, 8f
3470		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x800000
3471		\\ cbz wzr, 1f
3472		\\8:
3473		\\ mov w16, w0
3474		\\0:
3475		\\ ldaxrh w0, [x1]
3476		\\ bic w17, w0, w16
3477		\\ stxrh w15, w17, [x1]
3478		\\ cbnz w15, 0b
3479		\\1:
3480		: [ret] "={w0}" (-> u16),
3481		: [val] "{w0}" (val),
3482		[ptr] "{x1}" (ptr),
3483		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3484		: "w15", "w16", "w17", "memory"
3485		);
3486		}
3487		const __aarch64_ldclr2_acq = if (is_darwin)
3488		if (has_lse)
3489		ldclr2AcqDarwinLse
3490		else
3491		ldclr2AcqDarwinNolse
3492		else if (has_lse)
3493		ldclr2AcqNondarwinLse
3494		else
3495		ldclr2AcqNondarwinNolse;
3496		fn ldeor2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3497		@setRuntimeSafety(false);
3498		__init_aarch64_have_lse_atomics();
3499
3500		return asm volatile (
3501		\\ cbz w16, 8f
3502		\\ ldeorah w0, w0, [x1]
3503		\\ cbz wzr, 1f
3504		\\8:
3505		\\ mov w16, w0
3506		\\0:
3507		\\ ldaxrh w0, [x1]
3508		\\ eor w17, w0, w16
3509		\\ stxrh w15, w17, [x1]
3510		\\ cbnz w15, 0b
3511		\\1:
3512		: [ret] "={w0}" (-> u16),
3513		: [val] "{w0}" (val),
3514		[ptr] "{x1}" (ptr),
3515		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3516		: "w15", "w16", "w17", "memory"
3517		);
3518		}
3519		fn ldeor2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3520		@setRuntimeSafety(false);
3521		__init_aarch64_have_lse_atomics();
3522
3523		return asm volatile (
3524		\\ cbz w16, 8f
3525		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
3526		\\ cbz wzr, 1f
3527		\\8:
3528		\\ mov w16, w0
3529		\\0:
3530		\\ ldaxrh w0, [x1]
3531		\\ eor w17, w0, w16
3532		\\ stxrh w15, w17, [x1]
3533		\\ cbnz w15, 0b
3534		\\1:
3535		: [ret] "={w0}" (-> u16),
3536		: [val] "{w0}" (val),
3537		[ptr] "{x1}" (ptr),
3538		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3539		: "w15", "w16", "w17", "memory"
3540		);
3541		}
3542		fn ldeor2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3543		@setRuntimeSafety(false);
3544		__init_aarch64_have_lse_atomics();
3545
3546		return asm volatile (
3547		\\ cbz w16, 8f
3548		\\ ldeorah w0, w0, [x1]
3549		\\ cbz wzr, 1f
3550		\\8:
3551		\\ mov w16, w0
3552		\\0:
3553		\\ ldaxrh w0, [x1]
3554		\\ eor w17, w0, w16
3555		\\ stxrh w15, w17, [x1]
3556		\\ cbnz w15, 0b
3557		\\1:
3558		: [ret] "={w0}" (-> u16),
3559		: [val] "{w0}" (val),
3560		[ptr] "{x1}" (ptr),
3561		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3562		: "w15", "w16", "w17", "memory"
3563		);
3564		}
3565		fn ldeor2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3566		@setRuntimeSafety(false);
3567		__init_aarch64_have_lse_atomics();
3568
3569		return asm volatile (
3570		\\ cbz w16, 8f
3571		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x800000
3572		\\ cbz wzr, 1f
3573		\\8:
3574		\\ mov w16, w0
3575		\\0:
3576		\\ ldaxrh w0, [x1]
3577		\\ eor w17, w0, w16
3578		\\ stxrh w15, w17, [x1]
3579		\\ cbnz w15, 0b
3580		\\1:
3581		: [ret] "={w0}" (-> u16),
3582		: [val] "{w0}" (val),
3583		[ptr] "{x1}" (ptr),
3584		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3585		: "w15", "w16", "w17", "memory"
3586		);
3587		}
3588		const __aarch64_ldeor2_acq = if (is_darwin)
3589		if (has_lse)
3590		ldeor2AcqDarwinLse
3591		else
3592		ldeor2AcqDarwinNolse
3593		else if (has_lse)
3594		ldeor2AcqNondarwinLse
3595		else
3596		ldeor2AcqNondarwinNolse;
3597		fn ldset2AcqDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3598		@setRuntimeSafety(false);
3599		__init_aarch64_have_lse_atomics();
3600
3601		return asm volatile (
3602		\\ cbz w16, 8f
3603		\\ ldsetah w0, w0, [x1]
3604		\\ cbz wzr, 1f
3605		\\8:
3606		\\ mov w16, w0
3607		\\0:
3608		\\ ldaxrh w0, [x1]
3609		\\ orr w17, w0, w16
3610		\\ stxrh w15, w17, [x1]
3611		\\ cbnz w15, 0b
3612		\\1:
3613		: [ret] "={w0}" (-> u16),
3614		: [val] "{w0}" (val),
3615		[ptr] "{x1}" (ptr),
3616		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3617		: "w15", "w16", "w17", "memory"
3618		);
3619		}
3620		fn ldset2AcqDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3621		@setRuntimeSafety(false);
3622		__init_aarch64_have_lse_atomics();
3623
3624		return asm volatile (
3625		\\ cbz w16, 8f
3626		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
3627		\\ cbz wzr, 1f
3628		\\8:
3629		\\ mov w16, w0
3630		\\0:
3631		\\ ldaxrh w0, [x1]
3632		\\ orr w17, w0, w16
3633		\\ stxrh w15, w17, [x1]
3634		\\ cbnz w15, 0b
3635		\\1:
3636		: [ret] "={w0}" (-> u16),
3637		: [val] "{w0}" (val),
3638		[ptr] "{x1}" (ptr),
3639		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3640		: "w15", "w16", "w17", "memory"
3641		);
3642		}
3643		fn ldset2AcqNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3644		@setRuntimeSafety(false);
3645		__init_aarch64_have_lse_atomics();
3646
3647		return asm volatile (
3648		\\ cbz w16, 8f
3649		\\ ldsetah w0, w0, [x1]
3650		\\ cbz wzr, 1f
3651		\\8:
3652		\\ mov w16, w0
3653		\\0:
3654		\\ ldaxrh w0, [x1]
3655		\\ orr w17, w0, w16
3656		\\ stxrh w15, w17, [x1]
3657		\\ cbnz w15, 0b
3658		\\1:
3659		: [ret] "={w0}" (-> u16),
3660		: [val] "{w0}" (val),
3661		[ptr] "{x1}" (ptr),
3662		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3663		: "w15", "w16", "w17", "memory"
3664		);
3665		}
3666		fn ldset2AcqNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3667		@setRuntimeSafety(false);
3668		__init_aarch64_have_lse_atomics();
3669
3670		return asm volatile (
3671		\\ cbz w16, 8f
3672		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x800000
3673		\\ cbz wzr, 1f
3674		\\8:
3675		\\ mov w16, w0
3676		\\0:
3677		\\ ldaxrh w0, [x1]
3678		\\ orr w17, w0, w16
3679		\\ stxrh w15, w17, [x1]
3680		\\ cbnz w15, 0b
3681		\\1:
3682		: [ret] "={w0}" (-> u16),
3683		: [val] "{w0}" (val),
3684		[ptr] "{x1}" (ptr),
3685		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3686		: "w15", "w16", "w17", "memory"
3687		);
3688		}
3689		const __aarch64_ldset2_acq = if (is_darwin)
3690		if (has_lse)
3691		ldset2AcqDarwinLse
3692		else
3693		ldset2AcqDarwinNolse
3694		else if (has_lse)
3695		ldset2AcqNondarwinLse
3696		else
3697		ldset2AcqNondarwinNolse;
3698		fn cas2RelDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3699		@setRuntimeSafety(false);
3700		__init_aarch64_have_lse_atomics();
3701
3702		return asm volatile (
3703		\\ cbz w16, 8f
3704		\\ caslh w0, w1, [x2]
3705		\\
3706		\\ cbz wzr, 1f
3707		\\8:
3708		\\ uxth w16, w0
3709		\\0:
3710		\\ ldxrh w0, [x2]
3711		\\ cmp w0, w16
3712		\\ bne 1f
3713		\\ stlxrh w17, w1, [x2]
3714		\\ cbnz w17, 0b
3715		\\1:
3716		: [ret] "={w0}" (-> u16),
3717		: [expected] "{w0}" (expected),
3718		[desired] "{w1}" (desired),
3719		[ptr] "{x2}" (ptr),
3720		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3721		: "w15", "w16", "w17", "memory"
3722		);
3723		}
3724		fn cas2RelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3725		@setRuntimeSafety(false);
3726		__init_aarch64_have_lse_atomics();
3727
3728		return asm volatile (
3729		\\ cbz w16, 8f
3730		\\ .inst 0x08a07c41 + 0x40000000 + 0x008000
3731		\\
3732		\\ cbz wzr, 1f
3733		\\8:
3734		\\ uxth w16, w0
3735		\\0:
3736		\\ ldxrh w0, [x2]
3737		\\ cmp w0, w16
3738		\\ bne 1f
3739		\\ stlxrh w17, w1, [x2]
3740		\\ cbnz w17, 0b
3741		\\1:
3742		: [ret] "={w0}" (-> u16),
3743		: [expected] "{w0}" (expected),
3744		[desired] "{w1}" (desired),
3745		[ptr] "{x2}" (ptr),
3746		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3747		: "w15", "w16", "w17", "memory"
3748		);
3749		}
3750		fn cas2RelNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3751		@setRuntimeSafety(false);
3752		__init_aarch64_have_lse_atomics();
3753
3754		return asm volatile (
3755		\\ cbz w16, 8f
3756		\\ caslh w0, w1, [x2]
3757		\\
3758		\\ cbz wzr, 1f
3759		\\8:
3760		\\ uxth w16, w0
3761		\\0:
3762		\\ ldxrh w0, [x2]
3763		\\ cmp w0, w16
3764		\\ bne 1f
3765		\\ stlxrh w17, w1, [x2]
3766		\\ cbnz w17, 0b
3767		\\1:
3768		: [ret] "={w0}" (-> u16),
3769		: [expected] "{w0}" (expected),
3770		[desired] "{w1}" (desired),
3771		[ptr] "{x2}" (ptr),
3772		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3773		: "w15", "w16", "w17", "memory"
3774		);
3775		}
3776		fn cas2RelNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
3777		@setRuntimeSafety(false);
3778		__init_aarch64_have_lse_atomics();
3779
3780		return asm volatile (
3781		\\ cbz w16, 8f
3782		\\ .inst 0x08a07c41 + 0x40000000 + 0x008000
3783		\\
3784		\\ cbz wzr, 1f
3785		\\8:
3786		\\ uxth w16, w0
3787		\\0:
3788		\\ ldxrh w0, [x2]
3789		\\ cmp w0, w16
3790		\\ bne 1f
3791		\\ stlxrh w17, w1, [x2]
3792		\\ cbnz w17, 0b
3793		\\1:
3794		: [ret] "={w0}" (-> u16),
3795		: [expected] "{w0}" (expected),
3796		[desired] "{w1}" (desired),
3797		[ptr] "{x2}" (ptr),
3798		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3799		: "w15", "w16", "w17", "memory"
3800		);
3801		}
3802		const __aarch64_cas2_rel = if (is_darwin)
3803		if (has_lse)
3804		cas2RelDarwinLse
3805		else
3806		cas2RelDarwinNolse
3807		else if (has_lse)
3808		cas2RelNondarwinLse
3809		else
3810		cas2RelNondarwinNolse;
3811		fn swp2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3812		@setRuntimeSafety(false);
3813		__init_aarch64_have_lse_atomics();
3814
3815		return asm volatile (
3816		\\ cbz w16, 8f
3817		\\ swplh w0, w0, [x1]
3818		\\ cbz wzr, 1f
3819		\\8:
3820		\\ mov w16, w0
3821		\\0:
3822		\\ ldxrh w0, [x1]
3823		\\ stlxrh w17, w16, [x1]
3824		\\ cbnz w17, 0b
3825		\\1:
3826		: [ret] "={w0}" (-> u16),
3827		: [val] "{w0}" (val),
3828		[ptr] "{x1}" (ptr),
3829		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3830		: "w15", "w16", "w17", "memory"
3831		);
3832		}
3833		fn swp2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3834		@setRuntimeSafety(false);
3835		__init_aarch64_have_lse_atomics();
3836
3837		return asm volatile (
3838		\\ cbz w16, 8f
3839		\\ .inst 0x38208020 + 0x40000000 + 0x400000
3840		\\ cbz wzr, 1f
3841		\\8:
3842		\\ mov w16, w0
3843		\\0:
3844		\\ ldxrh w0, [x1]
3845		\\ stlxrh w17, w16, [x1]
3846		\\ cbnz w17, 0b
3847		\\1:
3848		: [ret] "={w0}" (-> u16),
3849		: [val] "{w0}" (val),
3850		[ptr] "{x1}" (ptr),
3851		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3852		: "w15", "w16", "w17", "memory"
3853		);
3854		}
3855		fn swp2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3856		@setRuntimeSafety(false);
3857		__init_aarch64_have_lse_atomics();
3858
3859		return asm volatile (
3860		\\ cbz w16, 8f
3861		\\ swplh w0, w0, [x1]
3862		\\ cbz wzr, 1f
3863		\\8:
3864		\\ mov w16, w0
3865		\\0:
3866		\\ ldxrh w0, [x1]
3867		\\ stlxrh w17, w16, [x1]
3868		\\ cbnz w17, 0b
3869		\\1:
3870		: [ret] "={w0}" (-> u16),
3871		: [val] "{w0}" (val),
3872		[ptr] "{x1}" (ptr),
3873		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3874		: "w15", "w16", "w17", "memory"
3875		);
3876		}
3877		fn swp2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3878		@setRuntimeSafety(false);
3879		__init_aarch64_have_lse_atomics();
3880
3881		return asm volatile (
3882		\\ cbz w16, 8f
3883		\\ .inst 0x38208020 + 0x40000000 + 0x400000
3884		\\ cbz wzr, 1f
3885		\\8:
3886		\\ mov w16, w0
3887		\\0:
3888		\\ ldxrh w0, [x1]
3889		\\ stlxrh w17, w16, [x1]
3890		\\ cbnz w17, 0b
3891		\\1:
3892		: [ret] "={w0}" (-> u16),
3893		: [val] "{w0}" (val),
3894		[ptr] "{x1}" (ptr),
3895		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3896		: "w15", "w16", "w17", "memory"
3897		);
3898		}
3899		const __aarch64_swp2_rel = if (is_darwin)
3900		if (has_lse)
3901		swp2RelDarwinLse
3902		else
3903		swp2RelDarwinNolse
3904		else if (has_lse)
3905		swp2RelNondarwinLse
3906		else
3907		swp2RelNondarwinNolse;
3908		fn ldadd2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3909		@setRuntimeSafety(false);
3910		__init_aarch64_have_lse_atomics();
3911
3912		return asm volatile (
3913		\\ cbz w16, 8f
3914		\\ ldaddlh w0, w0, [x1]
3915		\\ cbz wzr, 1f
3916		\\8:
3917		\\ mov w16, w0
3918		\\0:
3919		\\ ldxrh w0, [x1]
3920		\\ add w17, w0, w16
3921		\\ stlxrh w15, w17, [x1]
3922		\\ cbnz w15, 0b
3923		\\1:
3924		: [ret] "={w0}" (-> u16),
3925		: [val] "{w0}" (val),
3926		[ptr] "{x1}" (ptr),
3927		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3928		: "w15", "w16", "w17", "memory"
3929		);
3930		}
3931		fn ldadd2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3932		@setRuntimeSafety(false);
3933		__init_aarch64_have_lse_atomics();
3934
3935		return asm volatile (
3936		\\ cbz w16, 8f
3937		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
3938		\\ cbz wzr, 1f
3939		\\8:
3940		\\ mov w16, w0
3941		\\0:
3942		\\ ldxrh w0, [x1]
3943		\\ add w17, w0, w16
3944		\\ stlxrh w15, w17, [x1]
3945		\\ cbnz w15, 0b
3946		\\1:
3947		: [ret] "={w0}" (-> u16),
3948		: [val] "{w0}" (val),
3949		[ptr] "{x1}" (ptr),
3950		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3951		: "w15", "w16", "w17", "memory"
3952		);
3953		}
3954		fn ldadd2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
3955		@setRuntimeSafety(false);
3956		__init_aarch64_have_lse_atomics();
3957
3958		return asm volatile (
3959		\\ cbz w16, 8f
3960		\\ ldaddlh w0, w0, [x1]
3961		\\ cbz wzr, 1f
3962		\\8:
3963		\\ mov w16, w0
3964		\\0:
3965		\\ ldxrh w0, [x1]
3966		\\ add w17, w0, w16
3967		\\ stlxrh w15, w17, [x1]
3968		\\ cbnz w15, 0b
3969		\\1:
3970		: [ret] "={w0}" (-> u16),
3971		: [val] "{w0}" (val),
3972		[ptr] "{x1}" (ptr),
3973		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3974		: "w15", "w16", "w17", "memory"
3975		);
3976		}
3977		fn ldadd2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
3978		@setRuntimeSafety(false);
3979		__init_aarch64_have_lse_atomics();
3980
3981		return asm volatile (
3982		\\ cbz w16, 8f
3983		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0x400000
3984		\\ cbz wzr, 1f
3985		\\8:
3986		\\ mov w16, w0
3987		\\0:
3988		\\ ldxrh w0, [x1]
3989		\\ add w17, w0, w16
3990		\\ stlxrh w15, w17, [x1]
3991		\\ cbnz w15, 0b
3992		\\1:
3993		: [ret] "={w0}" (-> u16),
3994		: [val] "{w0}" (val),
3995		[ptr] "{x1}" (ptr),
3996		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
3997		: "w15", "w16", "w17", "memory"
3998		);
3999		}
4000		const __aarch64_ldadd2_rel = if (is_darwin)
4001		if (has_lse)
4002		ldadd2RelDarwinLse
4003		else
4004		ldadd2RelDarwinNolse
4005		else if (has_lse)
4006		ldadd2RelNondarwinLse
4007		else
4008		ldadd2RelNondarwinNolse;
4009		fn ldclr2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4010		@setRuntimeSafety(false);
4011		__init_aarch64_have_lse_atomics();
4012
4013		return asm volatile (
4014		\\ cbz w16, 8f
4015		\\ ldclrlh w0, w0, [x1]
4016		\\ cbz wzr, 1f
4017		\\8:
4018		\\ mov w16, w0
4019		\\0:
4020		\\ ldxrh w0, [x1]
4021		\\ bic w17, w0, w16
4022		\\ stlxrh w15, w17, [x1]
4023		\\ cbnz w15, 0b
4024		\\1:
4025		: [ret] "={w0}" (-> u16),
4026		: [val] "{w0}" (val),
4027		[ptr] "{x1}" (ptr),
4028		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4029		: "w15", "w16", "w17", "memory"
4030		);
4031		}
4032		fn ldclr2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4033		@setRuntimeSafety(false);
4034		__init_aarch64_have_lse_atomics();
4035
4036		return asm volatile (
4037		\\ cbz w16, 8f
4038		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
4039		\\ cbz wzr, 1f
4040		\\8:
4041		\\ mov w16, w0
4042		\\0:
4043		\\ ldxrh w0, [x1]
4044		\\ bic w17, w0, w16
4045		\\ stlxrh w15, w17, [x1]
4046		\\ cbnz w15, 0b
4047		\\1:
4048		: [ret] "={w0}" (-> u16),
4049		: [val] "{w0}" (val),
4050		[ptr] "{x1}" (ptr),
4051		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4052		: "w15", "w16", "w17", "memory"
4053		);
4054		}
4055		fn ldclr2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4056		@setRuntimeSafety(false);
4057		__init_aarch64_have_lse_atomics();
4058
4059		return asm volatile (
4060		\\ cbz w16, 8f
4061		\\ ldclrlh w0, w0, [x1]
4062		\\ cbz wzr, 1f
4063		\\8:
4064		\\ mov w16, w0
4065		\\0:
4066		\\ ldxrh w0, [x1]
4067		\\ bic w17, w0, w16
4068		\\ stlxrh w15, w17, [x1]
4069		\\ cbnz w15, 0b
4070		\\1:
4071		: [ret] "={w0}" (-> u16),
4072		: [val] "{w0}" (val),
4073		[ptr] "{x1}" (ptr),
4074		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4075		: "w15", "w16", "w17", "memory"
4076		);
4077		}
4078		fn ldclr2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4079		@setRuntimeSafety(false);
4080		__init_aarch64_have_lse_atomics();
4081
4082		return asm volatile (
4083		\\ cbz w16, 8f
4084		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0x400000
4085		\\ cbz wzr, 1f
4086		\\8:
4087		\\ mov w16, w0
4088		\\0:
4089		\\ ldxrh w0, [x1]
4090		\\ bic w17, w0, w16
4091		\\ stlxrh w15, w17, [x1]
4092		\\ cbnz w15, 0b
4093		\\1:
4094		: [ret] "={w0}" (-> u16),
4095		: [val] "{w0}" (val),
4096		[ptr] "{x1}" (ptr),
4097		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4098		: "w15", "w16", "w17", "memory"
4099		);
4100		}
4101		const __aarch64_ldclr2_rel = if (is_darwin)
4102		if (has_lse)
4103		ldclr2RelDarwinLse
4104		else
4105		ldclr2RelDarwinNolse
4106		else if (has_lse)
4107		ldclr2RelNondarwinLse
4108		else
4109		ldclr2RelNondarwinNolse;
4110		fn ldeor2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4111		@setRuntimeSafety(false);
4112		__init_aarch64_have_lse_atomics();
4113
4114		return asm volatile (
4115		\\ cbz w16, 8f
4116		\\ ldeorlh w0, w0, [x1]
4117		\\ cbz wzr, 1f
4118		\\8:
4119		\\ mov w16, w0
4120		\\0:
4121		\\ ldxrh w0, [x1]
4122		\\ eor w17, w0, w16
4123		\\ stlxrh w15, w17, [x1]
4124		\\ cbnz w15, 0b
4125		\\1:
4126		: [ret] "={w0}" (-> u16),
4127		: [val] "{w0}" (val),
4128		[ptr] "{x1}" (ptr),
4129		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4130		: "w15", "w16", "w17", "memory"
4131		);
4132		}
4133		fn ldeor2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4134		@setRuntimeSafety(false);
4135		__init_aarch64_have_lse_atomics();
4136
4137		return asm volatile (
4138		\\ cbz w16, 8f
4139		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
4140		\\ cbz wzr, 1f
4141		\\8:
4142		\\ mov w16, w0
4143		\\0:
4144		\\ ldxrh w0, [x1]
4145		\\ eor w17, w0, w16
4146		\\ stlxrh w15, w17, [x1]
4147		\\ cbnz w15, 0b
4148		\\1:
4149		: [ret] "={w0}" (-> u16),
4150		: [val] "{w0}" (val),
4151		[ptr] "{x1}" (ptr),
4152		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4153		: "w15", "w16", "w17", "memory"
4154		);
4155		}
4156		fn ldeor2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4157		@setRuntimeSafety(false);
4158		__init_aarch64_have_lse_atomics();
4159
4160		return asm volatile (
4161		\\ cbz w16, 8f
4162		\\ ldeorlh w0, w0, [x1]
4163		\\ cbz wzr, 1f
4164		\\8:
4165		\\ mov w16, w0
4166		\\0:
4167		\\ ldxrh w0, [x1]
4168		\\ eor w17, w0, w16
4169		\\ stlxrh w15, w17, [x1]
4170		\\ cbnz w15, 0b
4171		\\1:
4172		: [ret] "={w0}" (-> u16),
4173		: [val] "{w0}" (val),
4174		[ptr] "{x1}" (ptr),
4175		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4176		: "w15", "w16", "w17", "memory"
4177		);
4178		}
4179		fn ldeor2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4180		@setRuntimeSafety(false);
4181		__init_aarch64_have_lse_atomics();
4182
4183		return asm volatile (
4184		\\ cbz w16, 8f
4185		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0x400000
4186		\\ cbz wzr, 1f
4187		\\8:
4188		\\ mov w16, w0
4189		\\0:
4190		\\ ldxrh w0, [x1]
4191		\\ eor w17, w0, w16
4192		\\ stlxrh w15, w17, [x1]
4193		\\ cbnz w15, 0b
4194		\\1:
4195		: [ret] "={w0}" (-> u16),
4196		: [val] "{w0}" (val),
4197		[ptr] "{x1}" (ptr),
4198		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4199		: "w15", "w16", "w17", "memory"
4200		);
4201		}
4202		const __aarch64_ldeor2_rel = if (is_darwin)
4203		if (has_lse)
4204		ldeor2RelDarwinLse
4205		else
4206		ldeor2RelDarwinNolse
4207		else if (has_lse)
4208		ldeor2RelNondarwinLse
4209		else
4210		ldeor2RelNondarwinNolse;
4211		fn ldset2RelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4212		@setRuntimeSafety(false);
4213		__init_aarch64_have_lse_atomics();
4214
4215		return asm volatile (
4216		\\ cbz w16, 8f
4217		\\ ldsetlh w0, w0, [x1]
4218		\\ cbz wzr, 1f
4219		\\8:
4220		\\ mov w16, w0
4221		\\0:
4222		\\ ldxrh w0, [x1]
4223		\\ orr w17, w0, w16
4224		\\ stlxrh w15, w17, [x1]
4225		\\ cbnz w15, 0b
4226		\\1:
4227		: [ret] "={w0}" (-> u16),
4228		: [val] "{w0}" (val),
4229		[ptr] "{x1}" (ptr),
4230		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4231		: "w15", "w16", "w17", "memory"
4232		);
4233		}
4234		fn ldset2RelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4235		@setRuntimeSafety(false);
4236		__init_aarch64_have_lse_atomics();
4237
4238		return asm volatile (
4239		\\ cbz w16, 8f
4240		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
4241		\\ cbz wzr, 1f
4242		\\8:
4243		\\ mov w16, w0
4244		\\0:
4245		\\ ldxrh w0, [x1]
4246		\\ orr w17, w0, w16
4247		\\ stlxrh w15, w17, [x1]
4248		\\ cbnz w15, 0b
4249		\\1:
4250		: [ret] "={w0}" (-> u16),
4251		: [val] "{w0}" (val),
4252		[ptr] "{x1}" (ptr),
4253		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4254		: "w15", "w16", "w17", "memory"
4255		);
4256		}
4257		fn ldset2RelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4258		@setRuntimeSafety(false);
4259		__init_aarch64_have_lse_atomics();
4260
4261		return asm volatile (
4262		\\ cbz w16, 8f
4263		\\ ldsetlh w0, w0, [x1]
4264		\\ cbz wzr, 1f
4265		\\8:
4266		\\ mov w16, w0
4267		\\0:
4268		\\ ldxrh w0, [x1]
4269		\\ orr w17, w0, w16
4270		\\ stlxrh w15, w17, [x1]
4271		\\ cbnz w15, 0b
4272		\\1:
4273		: [ret] "={w0}" (-> u16),
4274		: [val] "{w0}" (val),
4275		[ptr] "{x1}" (ptr),
4276		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4277		: "w15", "w16", "w17", "memory"
4278		);
4279		}
4280		fn ldset2RelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4281		@setRuntimeSafety(false);
4282		__init_aarch64_have_lse_atomics();
4283
4284		return asm volatile (
4285		\\ cbz w16, 8f
4286		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0x400000
4287		\\ cbz wzr, 1f
4288		\\8:
4289		\\ mov w16, w0
4290		\\0:
4291		\\ ldxrh w0, [x1]
4292		\\ orr w17, w0, w16
4293		\\ stlxrh w15, w17, [x1]
4294		\\ cbnz w15, 0b
4295		\\1:
4296		: [ret] "={w0}" (-> u16),
4297		: [val] "{w0}" (val),
4298		[ptr] "{x1}" (ptr),
4299		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4300		: "w15", "w16", "w17", "memory"
4301		);
4302		}
4303		const __aarch64_ldset2_rel = if (is_darwin)
4304		if (has_lse)
4305		ldset2RelDarwinLse
4306		else
4307		ldset2RelDarwinNolse
4308		else if (has_lse)
4309		ldset2RelNondarwinLse
4310		else
4311		ldset2RelNondarwinNolse;
4312		fn cas2AcqRelDarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4313		@setRuntimeSafety(false);
4314		__init_aarch64_have_lse_atomics();
4315
4316		return asm volatile (
4317		\\ cbz w16, 8f
4318		\\ casalh w0, w1, [x2]
4319		\\
4320		\\ cbz wzr, 1f
4321		\\8:
4322		\\ uxth w16, w0
4323		\\0:
4324		\\ ldaxrh w0, [x2]
4325		\\ cmp w0, w16
4326		\\ bne 1f
4327		\\ stlxrh w17, w1, [x2]
4328		\\ cbnz w17, 0b
4329		\\1:
4330		: [ret] "={w0}" (-> u16),
4331		: [expected] "{w0}" (expected),
4332		[desired] "{w1}" (desired),
4333		[ptr] "{x2}" (ptr),
4334		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4335		: "w15", "w16", "w17", "memory"
4336		);
4337		}
4338		fn cas2AcqRelDarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4339		@setRuntimeSafety(false);
4340		__init_aarch64_have_lse_atomics();
4341
4342		return asm volatile (
4343		\\ cbz w16, 8f
4344		\\ .inst 0x08a07c41 + 0x40000000 + 0x408000
4345		\\
4346		\\ cbz wzr, 1f
4347		\\8:
4348		\\ uxth w16, w0
4349		\\0:
4350		\\ ldaxrh w0, [x2]
4351		\\ cmp w0, w16
4352		\\ bne 1f
4353		\\ stlxrh w17, w1, [x2]
4354		\\ cbnz w17, 0b
4355		\\1:
4356		: [ret] "={w0}" (-> u16),
4357		: [expected] "{w0}" (expected),
4358		[desired] "{w1}" (desired),
4359		[ptr] "{x2}" (ptr),
4360		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4361		: "w15", "w16", "w17", "memory"
4362		);
4363		}
4364		fn cas2AcqRelNondarwinLse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4365		@setRuntimeSafety(false);
4366		__init_aarch64_have_lse_atomics();
4367
4368		return asm volatile (
4369		\\ cbz w16, 8f
4370		\\ casalh w0, w1, [x2]
4371		\\
4372		\\ cbz wzr, 1f
4373		\\8:
4374		\\ uxth w16, w0
4375		\\0:
4376		\\ ldaxrh w0, [x2]
4377		\\ cmp w0, w16
4378		\\ bne 1f
4379		\\ stlxrh w17, w1, [x2]
4380		\\ cbnz w17, 0b
4381		\\1:
4382		: [ret] "={w0}" (-> u16),
4383		: [expected] "{w0}" (expected),
4384		[desired] "{w1}" (desired),
4385		[ptr] "{x2}" (ptr),
4386		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4387		: "w15", "w16", "w17", "memory"
4388		);
4389		}
4390		fn cas2AcqRelNondarwinNolse(expected: u16, desired: u16, ptr: *u16) callconv(.C) u16 {
4391		@setRuntimeSafety(false);
4392		__init_aarch64_have_lse_atomics();
4393
4394		return asm volatile (
4395		\\ cbz w16, 8f
4396		\\ .inst 0x08a07c41 + 0x40000000 + 0x408000
4397		\\
4398		\\ cbz wzr, 1f
4399		\\8:
4400		\\ uxth w16, w0
4401		\\0:
4402		\\ ldaxrh w0, [x2]
4403		\\ cmp w0, w16
4404		\\ bne 1f
4405		\\ stlxrh w17, w1, [x2]
4406		\\ cbnz w17, 0b
4407		\\1:
4408		: [ret] "={w0}" (-> u16),
4409		: [expected] "{w0}" (expected),
4410		[desired] "{w1}" (desired),
4411		[ptr] "{x2}" (ptr),
4412		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4413		: "w15", "w16", "w17", "memory"
4414		);
4415		}
4416		const __aarch64_cas2_acq_rel = if (is_darwin)
4417		if (has_lse)
4418		cas2AcqRelDarwinLse
4419		else
4420		cas2AcqRelDarwinNolse
4421		else if (has_lse)
4422		cas2AcqRelNondarwinLse
4423		else
4424		cas2AcqRelNondarwinNolse;
4425		fn swp2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4426		@setRuntimeSafety(false);
4427		__init_aarch64_have_lse_atomics();
4428
4429		return asm volatile (
4430		\\ cbz w16, 8f
4431		\\ swpalh w0, w0, [x1]
4432		\\ cbz wzr, 1f
4433		\\8:
4434		\\ mov w16, w0
4435		\\0:
4436		\\ ldaxrh w0, [x1]
4437		\\ stlxrh w17, w16, [x1]
4438		\\ cbnz w17, 0b
4439		\\1:
4440		: [ret] "={w0}" (-> u16),
4441		: [val] "{w0}" (val),
4442		[ptr] "{x1}" (ptr),
4443		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4444		: "w15", "w16", "w17", "memory"
4445		);
4446		}
4447		fn swp2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4448		@setRuntimeSafety(false);
4449		__init_aarch64_have_lse_atomics();
4450
4451		return asm volatile (
4452		\\ cbz w16, 8f
4453		\\ .inst 0x38208020 + 0x40000000 + 0xc00000
4454		\\ cbz wzr, 1f
4455		\\8:
4456		\\ mov w16, w0
4457		\\0:
4458		\\ ldaxrh w0, [x1]
4459		\\ stlxrh w17, w16, [x1]
4460		\\ cbnz w17, 0b
4461		\\1:
4462		: [ret] "={w0}" (-> u16),
4463		: [val] "{w0}" (val),
4464		[ptr] "{x1}" (ptr),
4465		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4466		: "w15", "w16", "w17", "memory"
4467		);
4468		}
4469		fn swp2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4470		@setRuntimeSafety(false);
4471		__init_aarch64_have_lse_atomics();
4472
4473		return asm volatile (
4474		\\ cbz w16, 8f
4475		\\ swpalh w0, w0, [x1]
4476		\\ cbz wzr, 1f
4477		\\8:
4478		\\ mov w16, w0
4479		\\0:
4480		\\ ldaxrh w0, [x1]
4481		\\ stlxrh w17, w16, [x1]
4482		\\ cbnz w17, 0b
4483		\\1:
4484		: [ret] "={w0}" (-> u16),
4485		: [val] "{w0}" (val),
4486		[ptr] "{x1}" (ptr),
4487		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4488		: "w15", "w16", "w17", "memory"
4489		);
4490		}
4491		fn swp2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4492		@setRuntimeSafety(false);
4493		__init_aarch64_have_lse_atomics();
4494
4495		return asm volatile (
4496		\\ cbz w16, 8f
4497		\\ .inst 0x38208020 + 0x40000000 + 0xc00000
4498		\\ cbz wzr, 1f
4499		\\8:
4500		\\ mov w16, w0
4501		\\0:
4502		\\ ldaxrh w0, [x1]
4503		\\ stlxrh w17, w16, [x1]
4504		\\ cbnz w17, 0b
4505		\\1:
4506		: [ret] "={w0}" (-> u16),
4507		: [val] "{w0}" (val),
4508		[ptr] "{x1}" (ptr),
4509		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4510		: "w15", "w16", "w17", "memory"
4511		);
4512		}
4513		const __aarch64_swp2_acq_rel = if (is_darwin)
4514		if (has_lse)
4515		swp2AcqRelDarwinLse
4516		else
4517		swp2AcqRelDarwinNolse
4518		else if (has_lse)
4519		swp2AcqRelNondarwinLse
4520		else
4521		swp2AcqRelNondarwinNolse;
4522		fn ldadd2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4523		@setRuntimeSafety(false);
4524		__init_aarch64_have_lse_atomics();
4525
4526		return asm volatile (
4527		\\ cbz w16, 8f
4528		\\ ldaddalh w0, w0, [x1]
4529		\\ cbz wzr, 1f
4530		\\8:
4531		\\ mov w16, w0
4532		\\0:
4533		\\ ldaxrh w0, [x1]
4534		\\ add w17, w0, w16
4535		\\ stlxrh w15, w17, [x1]
4536		\\ cbnz w15, 0b
4537		\\1:
4538		: [ret] "={w0}" (-> u16),
4539		: [val] "{w0}" (val),
4540		[ptr] "{x1}" (ptr),
4541		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4542		: "w15", "w16", "w17", "memory"
4543		);
4544		}
4545		fn ldadd2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4546		@setRuntimeSafety(false);
4547		__init_aarch64_have_lse_atomics();
4548
4549		return asm volatile (
4550		\\ cbz w16, 8f
4551		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
4552		\\ cbz wzr, 1f
4553		\\8:
4554		\\ mov w16, w0
4555		\\0:
4556		\\ ldaxrh w0, [x1]
4557		\\ add w17, w0, w16
4558		\\ stlxrh w15, w17, [x1]
4559		\\ cbnz w15, 0b
4560		\\1:
4561		: [ret] "={w0}" (-> u16),
4562		: [val] "{w0}" (val),
4563		[ptr] "{x1}" (ptr),
4564		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4565		: "w15", "w16", "w17", "memory"
4566		);
4567		}
4568		fn ldadd2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4569		@setRuntimeSafety(false);
4570		__init_aarch64_have_lse_atomics();
4571
4572		return asm volatile (
4573		\\ cbz w16, 8f
4574		\\ ldaddalh w0, w0, [x1]
4575		\\ cbz wzr, 1f
4576		\\8:
4577		\\ mov w16, w0
4578		\\0:
4579		\\ ldaxrh w0, [x1]
4580		\\ add w17, w0, w16
4581		\\ stlxrh w15, w17, [x1]
4582		\\ cbnz w15, 0b
4583		\\1:
4584		: [ret] "={w0}" (-> u16),
4585		: [val] "{w0}" (val),
4586		[ptr] "{x1}" (ptr),
4587		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4588		: "w15", "w16", "w17", "memory"
4589		);
4590		}
4591		fn ldadd2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4592		@setRuntimeSafety(false);
4593		__init_aarch64_have_lse_atomics();
4594
4595		return asm volatile (
4596		\\ cbz w16, 8f
4597		\\ .inst 0x38200020 + 0x0000 + 0x40000000 + 0xc00000
4598		\\ cbz wzr, 1f
4599		\\8:
4600		\\ mov w16, w0
4601		\\0:
4602		\\ ldaxrh w0, [x1]
4603		\\ add w17, w0, w16
4604		\\ stlxrh w15, w17, [x1]
4605		\\ cbnz w15, 0b
4606		\\1:
4607		: [ret] "={w0}" (-> u16),
4608		: [val] "{w0}" (val),
4609		[ptr] "{x1}" (ptr),
4610		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4611		: "w15", "w16", "w17", "memory"
4612		);
4613		}
4614		const __aarch64_ldadd2_acq_rel = if (is_darwin)
4615		if (has_lse)
4616		ldadd2AcqRelDarwinLse
4617		else
4618		ldadd2AcqRelDarwinNolse
4619		else if (has_lse)
4620		ldadd2AcqRelNondarwinLse
4621		else
4622		ldadd2AcqRelNondarwinNolse;
4623		fn ldclr2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4624		@setRuntimeSafety(false);
4625		__init_aarch64_have_lse_atomics();
4626
4627		return asm volatile (
4628		\\ cbz w16, 8f
4629		\\ ldclralh w0, w0, [x1]
4630		\\ cbz wzr, 1f
4631		\\8:
4632		\\ mov w16, w0
4633		\\0:
4634		\\ ldaxrh w0, [x1]
4635		\\ bic w17, w0, w16
4636		\\ stlxrh w15, w17, [x1]
4637		\\ cbnz w15, 0b
4638		\\1:
4639		: [ret] "={w0}" (-> u16),
4640		: [val] "{w0}" (val),
4641		[ptr] "{x1}" (ptr),
4642		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4643		: "w15", "w16", "w17", "memory"
4644		);
4645		}
4646		fn ldclr2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4647		@setRuntimeSafety(false);
4648		__init_aarch64_have_lse_atomics();
4649
4650		return asm volatile (
4651		\\ cbz w16, 8f
4652		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
4653		\\ cbz wzr, 1f
4654		\\8:
4655		\\ mov w16, w0
4656		\\0:
4657		\\ ldaxrh w0, [x1]
4658		\\ bic w17, w0, w16
4659		\\ stlxrh w15, w17, [x1]
4660		\\ cbnz w15, 0b
4661		\\1:
4662		: [ret] "={w0}" (-> u16),
4663		: [val] "{w0}" (val),
4664		[ptr] "{x1}" (ptr),
4665		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4666		: "w15", "w16", "w17", "memory"
4667		);
4668		}
4669		fn ldclr2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4670		@setRuntimeSafety(false);
4671		__init_aarch64_have_lse_atomics();
4672
4673		return asm volatile (
4674		\\ cbz w16, 8f
4675		\\ ldclralh w0, w0, [x1]
4676		\\ cbz wzr, 1f
4677		\\8:
4678		\\ mov w16, w0
4679		\\0:
4680		\\ ldaxrh w0, [x1]
4681		\\ bic w17, w0, w16
4682		\\ stlxrh w15, w17, [x1]
4683		\\ cbnz w15, 0b
4684		\\1:
4685		: [ret] "={w0}" (-> u16),
4686		: [val] "{w0}" (val),
4687		[ptr] "{x1}" (ptr),
4688		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4689		: "w15", "w16", "w17", "memory"
4690		);
4691		}
4692		fn ldclr2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4693		@setRuntimeSafety(false);
4694		__init_aarch64_have_lse_atomics();
4695
4696		return asm volatile (
4697		\\ cbz w16, 8f
4698		\\ .inst 0x38200020 + 0x1000 + 0x40000000 + 0xc00000
4699		\\ cbz wzr, 1f
4700		\\8:
4701		\\ mov w16, w0
4702		\\0:
4703		\\ ldaxrh w0, [x1]
4704		\\ bic w17, w0, w16
4705		\\ stlxrh w15, w17, [x1]
4706		\\ cbnz w15, 0b
4707		\\1:
4708		: [ret] "={w0}" (-> u16),
4709		: [val] "{w0}" (val),
4710		[ptr] "{x1}" (ptr),
4711		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4712		: "w15", "w16", "w17", "memory"
4713		);
4714		}
4715		const __aarch64_ldclr2_acq_rel = if (is_darwin)
4716		if (has_lse)
4717		ldclr2AcqRelDarwinLse
4718		else
4719		ldclr2AcqRelDarwinNolse
4720		else if (has_lse)
4721		ldclr2AcqRelNondarwinLse
4722		else
4723		ldclr2AcqRelNondarwinNolse;
4724		fn ldeor2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4725		@setRuntimeSafety(false);
4726		__init_aarch64_have_lse_atomics();
4727
4728		return asm volatile (
4729		\\ cbz w16, 8f
4730		\\ ldeoralh w0, w0, [x1]
4731		\\ cbz wzr, 1f
4732		\\8:
4733		\\ mov w16, w0
4734		\\0:
4735		\\ ldaxrh w0, [x1]
4736		\\ eor w17, w0, w16
4737		\\ stlxrh w15, w17, [x1]
4738		\\ cbnz w15, 0b
4739		\\1:
4740		: [ret] "={w0}" (-> u16),
4741		: [val] "{w0}" (val),
4742		[ptr] "{x1}" (ptr),
4743		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4744		: "w15", "w16", "w17", "memory"
4745		);
4746		}
4747		fn ldeor2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4748		@setRuntimeSafety(false);
4749		__init_aarch64_have_lse_atomics();
4750
4751		return asm volatile (
4752		\\ cbz w16, 8f
4753		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
4754		\\ cbz wzr, 1f
4755		\\8:
4756		\\ mov w16, w0
4757		\\0:
4758		\\ ldaxrh w0, [x1]
4759		\\ eor w17, w0, w16
4760		\\ stlxrh w15, w17, [x1]
4761		\\ cbnz w15, 0b
4762		\\1:
4763		: [ret] "={w0}" (-> u16),
4764		: [val] "{w0}" (val),
4765		[ptr] "{x1}" (ptr),
4766		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4767		: "w15", "w16", "w17", "memory"
4768		);
4769		}
4770		fn ldeor2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4771		@setRuntimeSafety(false);
4772		__init_aarch64_have_lse_atomics();
4773
4774		return asm volatile (
4775		\\ cbz w16, 8f
4776		\\ ldeoralh w0, w0, [x1]
4777		\\ cbz wzr, 1f
4778		\\8:
4779		\\ mov w16, w0
4780		\\0:
4781		\\ ldaxrh w0, [x1]
4782		\\ eor w17, w0, w16
4783		\\ stlxrh w15, w17, [x1]
4784		\\ cbnz w15, 0b
4785		\\1:
4786		: [ret] "={w0}" (-> u16),
4787		: [val] "{w0}" (val),
4788		[ptr] "{x1}" (ptr),
4789		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4790		: "w15", "w16", "w17", "memory"
4791		);
4792		}
4793		fn ldeor2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4794		@setRuntimeSafety(false);
4795		__init_aarch64_have_lse_atomics();
4796
4797		return asm volatile (
4798		\\ cbz w16, 8f
4799		\\ .inst 0x38200020 + 0x2000 + 0x40000000 + 0xc00000
4800		\\ cbz wzr, 1f
4801		\\8:
4802		\\ mov w16, w0
4803		\\0:
4804		\\ ldaxrh w0, [x1]
4805		\\ eor w17, w0, w16
4806		\\ stlxrh w15, w17, [x1]
4807		\\ cbnz w15, 0b
4808		\\1:
4809		: [ret] "={w0}" (-> u16),
4810		: [val] "{w0}" (val),
4811		[ptr] "{x1}" (ptr),
4812		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4813		: "w15", "w16", "w17", "memory"
4814		);
4815		}
4816		const __aarch64_ldeor2_acq_rel = if (is_darwin)
4817		if (has_lse)
4818		ldeor2AcqRelDarwinLse
4819		else
4820		ldeor2AcqRelDarwinNolse
4821		else if (has_lse)
4822		ldeor2AcqRelNondarwinLse
4823		else
4824		ldeor2AcqRelNondarwinNolse;
4825		fn ldset2AcqRelDarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4826		@setRuntimeSafety(false);
4827		__init_aarch64_have_lse_atomics();
4828
4829		return asm volatile (
4830		\\ cbz w16, 8f
4831		\\ ldsetalh w0, w0, [x1]
4832		\\ cbz wzr, 1f
4833		\\8:
4834		\\ mov w16, w0
4835		\\0:
4836		\\ ldaxrh w0, [x1]
4837		\\ orr w17, w0, w16
4838		\\ stlxrh w15, w17, [x1]
4839		\\ cbnz w15, 0b
4840		\\1:
4841		: [ret] "={w0}" (-> u16),
4842		: [val] "{w0}" (val),
4843		[ptr] "{x1}" (ptr),
4844		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4845		: "w15", "w16", "w17", "memory"
4846		);
4847		}
4848		fn ldset2AcqRelDarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4849		@setRuntimeSafety(false);
4850		__init_aarch64_have_lse_atomics();
4851
4852		return asm volatile (
4853		\\ cbz w16, 8f
4854		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
4855		\\ cbz wzr, 1f
4856		\\8:
4857		\\ mov w16, w0
4858		\\0:
4859		\\ ldaxrh w0, [x1]
4860		\\ orr w17, w0, w16
4861		\\ stlxrh w15, w17, [x1]
4862		\\ cbnz w15, 0b
4863		\\1:
4864		: [ret] "={w0}" (-> u16),
4865		: [val] "{w0}" (val),
4866		[ptr] "{x1}" (ptr),
4867		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4868		: "w15", "w16", "w17", "memory"
4869		);
4870		}
4871		fn ldset2AcqRelNondarwinLse(val: u16, ptr: *u16) callconv(.C) u16 {
4872		@setRuntimeSafety(false);
4873		__init_aarch64_have_lse_atomics();
4874
4875		return asm volatile (
4876		\\ cbz w16, 8f
4877		\\ ldsetalh w0, w0, [x1]
4878		\\ cbz wzr, 1f
4879		\\8:
4880		\\ mov w16, w0
4881		\\0:
4882		\\ ldaxrh w0, [x1]
4883		\\ orr w17, w0, w16
4884		\\ stlxrh w15, w17, [x1]
4885		\\ cbnz w15, 0b
4886		\\1:
4887		: [ret] "={w0}" (-> u16),
4888		: [val] "{w0}" (val),
4889		[ptr] "{x1}" (ptr),
4890		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4891		: "w15", "w16", "w17", "memory"
4892		);
4893		}
4894		fn ldset2AcqRelNondarwinNolse(val: u16, ptr: *u16) callconv(.C) u16 {
4895		@setRuntimeSafety(false);
4896		__init_aarch64_have_lse_atomics();
4897
4898		return asm volatile (
4899		\\ cbz w16, 8f
4900		\\ .inst 0x38200020 + 0x3000 + 0x40000000 + 0xc00000
4901		\\ cbz wzr, 1f
4902		\\8:
4903		\\ mov w16, w0
4904		\\0:
4905		\\ ldaxrh w0, [x1]
4906		\\ orr w17, w0, w16
4907		\\ stlxrh w15, w17, [x1]
4908		\\ cbnz w15, 0b
4909		\\1:
4910		: [ret] "={w0}" (-> u16),
4911		: [val] "{w0}" (val),
4912		[ptr] "{x1}" (ptr),
4913		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4914		: "w15", "w16", "w17", "memory"
4915		);
4916		}
4917		const __aarch64_ldset2_acq_rel = if (is_darwin)
4918		if (has_lse)
4919		ldset2AcqRelDarwinLse
4920		else
4921		ldset2AcqRelDarwinNolse
4922		else if (has_lse)
4923		ldset2AcqRelNondarwinLse
4924		else
4925		ldset2AcqRelNondarwinNolse;
4926		fn cas4RelaxDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
4927		@setRuntimeSafety(false);
4928		__init_aarch64_have_lse_atomics();
4929
4930		return asm volatile (
4931		\\ cbz w16, 8f
4932		\\ cas w0, w1, [x2]
4933		\\
4934		\\ cbz wzr, 1f
4935		\\8:
4936		\\ mov w16, w0
4937		\\0:
4938		\\ ldxr w0, [x2]
4939		\\ cmp w0, w16
4940		\\ bne 1f
4941		\\ stxr w17, w1, [x2]
4942		\\ cbnz w17, 0b
4943		\\1:
4944		: [ret] "={w0}" (-> u32),
4945		: [expected] "{w0}" (expected),
4946		[desired] "{w1}" (desired),
4947		[ptr] "{x2}" (ptr),
4948		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4949		: "w15", "w16", "w17", "memory"
4950		);
4951		}
4952		fn cas4RelaxDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
4953		@setRuntimeSafety(false);
4954		__init_aarch64_have_lse_atomics();
4955
4956		return asm volatile (
4957		\\ cbz w16, 8f
4958		\\ .inst 0x08a07c41 + 0x80000000 + 0x000000
4959		\\
4960		\\ cbz wzr, 1f
4961		\\8:
4962		\\ mov w16, w0
4963		\\0:
4964		\\ ldxr w0, [x2]
4965		\\ cmp w0, w16
4966		\\ bne 1f
4967		\\ stxr w17, w1, [x2]
4968		\\ cbnz w17, 0b
4969		\\1:
4970		: [ret] "={w0}" (-> u32),
4971		: [expected] "{w0}" (expected),
4972		[desired] "{w1}" (desired),
4973		[ptr] "{x2}" (ptr),
4974		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
4975		: "w15", "w16", "w17", "memory"
4976		);
4977		}
4978		fn cas4RelaxNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
4979		@setRuntimeSafety(false);
4980		__init_aarch64_have_lse_atomics();
4981
4982		return asm volatile (
4983		\\ cbz w16, 8f
4984		\\ cas w0, w1, [x2]
4985		\\
4986		\\ cbz wzr, 1f
4987		\\8:
4988		\\ mov w16, w0
4989		\\0:
4990		\\ ldxr w0, [x2]
4991		\\ cmp w0, w16
4992		\\ bne 1f
4993		\\ stxr w17, w1, [x2]
4994		\\ cbnz w17, 0b
4995		\\1:
4996		: [ret] "={w0}" (-> u32),
4997		: [expected] "{w0}" (expected),
4998		[desired] "{w1}" (desired),
4999		[ptr] "{x2}" (ptr),
5000		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5001		: "w15", "w16", "w17", "memory"
5002		);
5003		}
5004		fn cas4RelaxNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5005		@setRuntimeSafety(false);
5006		__init_aarch64_have_lse_atomics();
5007
5008		return asm volatile (
5009		\\ cbz w16, 8f
5010		\\ .inst 0x08a07c41 + 0x80000000 + 0x000000
5011		\\
5012		\\ cbz wzr, 1f
5013		\\8:
5014		\\ mov w16, w0
5015		\\0:
5016		\\ ldxr w0, [x2]
5017		\\ cmp w0, w16
5018		\\ bne 1f
5019		\\ stxr w17, w1, [x2]
5020		\\ cbnz w17, 0b
5021		\\1:
5022		: [ret] "={w0}" (-> u32),
5023		: [expected] "{w0}" (expected),
5024		[desired] "{w1}" (desired),
5025		[ptr] "{x2}" (ptr),
5026		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5027		: "w15", "w16", "w17", "memory"
5028		);
5029		}
5030		const __aarch64_cas4_relax = if (is_darwin)
5031		if (has_lse)
5032		cas4RelaxDarwinLse
5033		else
5034		cas4RelaxDarwinNolse
5035		else if (has_lse)
5036		cas4RelaxNondarwinLse
5037		else
5038		cas4RelaxNondarwinNolse;
5039		fn swp4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5040		@setRuntimeSafety(false);
5041		__init_aarch64_have_lse_atomics();
5042
5043		return asm volatile (
5044		\\ cbz w16, 8f
5045		\\ swp w0, w0, [x1]
5046		\\ cbz wzr, 1f
5047		\\8:
5048		\\ mov w16, w0
5049		\\0:
5050		\\ ldxr w0, [x1]
5051		\\ stxr w17, w16, [x1]
5052		\\ cbnz w17, 0b
5053		\\1:
5054		: [ret] "={w0}" (-> u32),
5055		: [val] "{w0}" (val),
5056		[ptr] "{x1}" (ptr),
5057		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5058		: "w15", "w16", "w17", "memory"
5059		);
5060		}
5061		fn swp4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5062		@setRuntimeSafety(false);
5063		__init_aarch64_have_lse_atomics();
5064
5065		return asm volatile (
5066		\\ cbz w16, 8f
5067		\\ .inst 0x38208020 + 0x80000000 + 0x000000
5068		\\ cbz wzr, 1f
5069		\\8:
5070		\\ mov w16, w0
5071		\\0:
5072		\\ ldxr w0, [x1]
5073		\\ stxr w17, w16, [x1]
5074		\\ cbnz w17, 0b
5075		\\1:
5076		: [ret] "={w0}" (-> u32),
5077		: [val] "{w0}" (val),
5078		[ptr] "{x1}" (ptr),
5079		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5080		: "w15", "w16", "w17", "memory"
5081		);
5082		}
5083		fn swp4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5084		@setRuntimeSafety(false);
5085		__init_aarch64_have_lse_atomics();
5086
5087		return asm volatile (
5088		\\ cbz w16, 8f
5089		\\ swp w0, w0, [x1]
5090		\\ cbz wzr, 1f
5091		\\8:
5092		\\ mov w16, w0
5093		\\0:
5094		\\ ldxr w0, [x1]
5095		\\ stxr w17, w16, [x1]
5096		\\ cbnz w17, 0b
5097		\\1:
5098		: [ret] "={w0}" (-> u32),
5099		: [val] "{w0}" (val),
5100		[ptr] "{x1}" (ptr),
5101		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5102		: "w15", "w16", "w17", "memory"
5103		);
5104		}
5105		fn swp4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5106		@setRuntimeSafety(false);
5107		__init_aarch64_have_lse_atomics();
5108
5109		return asm volatile (
5110		\\ cbz w16, 8f
5111		\\ .inst 0x38208020 + 0x80000000 + 0x000000
5112		\\ cbz wzr, 1f
5113		\\8:
5114		\\ mov w16, w0
5115		\\0:
5116		\\ ldxr w0, [x1]
5117		\\ stxr w17, w16, [x1]
5118		\\ cbnz w17, 0b
5119		\\1:
5120		: [ret] "={w0}" (-> u32),
5121		: [val] "{w0}" (val),
5122		[ptr] "{x1}" (ptr),
5123		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5124		: "w15", "w16", "w17", "memory"
5125		);
5126		}
5127		const __aarch64_swp4_relax = if (is_darwin)
5128		if (has_lse)
5129		swp4RelaxDarwinLse
5130		else
5131		swp4RelaxDarwinNolse
5132		else if (has_lse)
5133		swp4RelaxNondarwinLse
5134		else
5135		swp4RelaxNondarwinNolse;
5136		fn ldadd4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5137		@setRuntimeSafety(false);
5138		__init_aarch64_have_lse_atomics();
5139
5140		return asm volatile (
5141		\\ cbz w16, 8f
5142		\\ ldadd w0, w0, [x1]
5143		\\ cbz wzr, 1f
5144		\\8:
5145		\\ mov w16, w0
5146		\\0:
5147		\\ ldxr w0, [x1]
5148		\\ add w17, w0, w16
5149		\\ stxr w15, w17, [x1]
5150		\\ cbnz w15, 0b
5151		\\1:
5152		: [ret] "={w0}" (-> u32),
5153		: [val] "{w0}" (val),
5154		[ptr] "{x1}" (ptr),
5155		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5156		: "w15", "w16", "w17", "memory"
5157		);
5158		}
5159		fn ldadd4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5160		@setRuntimeSafety(false);
5161		__init_aarch64_have_lse_atomics();
5162
5163		return asm volatile (
5164		\\ cbz w16, 8f
5165		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
5166		\\ cbz wzr, 1f
5167		\\8:
5168		\\ mov w16, w0
5169		\\0:
5170		\\ ldxr w0, [x1]
5171		\\ add w17, w0, w16
5172		\\ stxr w15, w17, [x1]
5173		\\ cbnz w15, 0b
5174		\\1:
5175		: [ret] "={w0}" (-> u32),
5176		: [val] "{w0}" (val),
5177		[ptr] "{x1}" (ptr),
5178		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5179		: "w15", "w16", "w17", "memory"
5180		);
5181		}
5182		fn ldadd4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5183		@setRuntimeSafety(false);
5184		__init_aarch64_have_lse_atomics();
5185
5186		return asm volatile (
5187		\\ cbz w16, 8f
5188		\\ ldadd w0, w0, [x1]
5189		\\ cbz wzr, 1f
5190		\\8:
5191		\\ mov w16, w0
5192		\\0:
5193		\\ ldxr w0, [x1]
5194		\\ add w17, w0, w16
5195		\\ stxr w15, w17, [x1]
5196		\\ cbnz w15, 0b
5197		\\1:
5198		: [ret] "={w0}" (-> u32),
5199		: [val] "{w0}" (val),
5200		[ptr] "{x1}" (ptr),
5201		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5202		: "w15", "w16", "w17", "memory"
5203		);
5204		}
5205		fn ldadd4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5206		@setRuntimeSafety(false);
5207		__init_aarch64_have_lse_atomics();
5208
5209		return asm volatile (
5210		\\ cbz w16, 8f
5211		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x000000
5212		\\ cbz wzr, 1f
5213		\\8:
5214		\\ mov w16, w0
5215		\\0:
5216		\\ ldxr w0, [x1]
5217		\\ add w17, w0, w16
5218		\\ stxr w15, w17, [x1]
5219		\\ cbnz w15, 0b
5220		\\1:
5221		: [ret] "={w0}" (-> u32),
5222		: [val] "{w0}" (val),
5223		[ptr] "{x1}" (ptr),
5224		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5225		: "w15", "w16", "w17", "memory"
5226		);
5227		}
5228		const __aarch64_ldadd4_relax = if (is_darwin)
5229		if (has_lse)
5230		ldadd4RelaxDarwinLse
5231		else
5232		ldadd4RelaxDarwinNolse
5233		else if (has_lse)
5234		ldadd4RelaxNondarwinLse
5235		else
5236		ldadd4RelaxNondarwinNolse;
5237		fn ldclr4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5238		@setRuntimeSafety(false);
5239		__init_aarch64_have_lse_atomics();
5240
5241		return asm volatile (
5242		\\ cbz w16, 8f
5243		\\ ldclr w0, w0, [x1]
5244		\\ cbz wzr, 1f
5245		\\8:
5246		\\ mov w16, w0
5247		\\0:
5248		\\ ldxr w0, [x1]
5249		\\ bic w17, w0, w16
5250		\\ stxr w15, w17, [x1]
5251		\\ cbnz w15, 0b
5252		\\1:
5253		: [ret] "={w0}" (-> u32),
5254		: [val] "{w0}" (val),
5255		[ptr] "{x1}" (ptr),
5256		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5257		: "w15", "w16", "w17", "memory"
5258		);
5259		}
5260		fn ldclr4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5261		@setRuntimeSafety(false);
5262		__init_aarch64_have_lse_atomics();
5263
5264		return asm volatile (
5265		\\ cbz w16, 8f
5266		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
5267		\\ cbz wzr, 1f
5268		\\8:
5269		\\ mov w16, w0
5270		\\0:
5271		\\ ldxr w0, [x1]
5272		\\ bic w17, w0, w16
5273		\\ stxr w15, w17, [x1]
5274		\\ cbnz w15, 0b
5275		\\1:
5276		: [ret] "={w0}" (-> u32),
5277		: [val] "{w0}" (val),
5278		[ptr] "{x1}" (ptr),
5279		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5280		: "w15", "w16", "w17", "memory"
5281		);
5282		}
5283		fn ldclr4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5284		@setRuntimeSafety(false);
5285		__init_aarch64_have_lse_atomics();
5286
5287		return asm volatile (
5288		\\ cbz w16, 8f
5289		\\ ldclr w0, w0, [x1]
5290		\\ cbz wzr, 1f
5291		\\8:
5292		\\ mov w16, w0
5293		\\0:
5294		\\ ldxr w0, [x1]
5295		\\ bic w17, w0, w16
5296		\\ stxr w15, w17, [x1]
5297		\\ cbnz w15, 0b
5298		\\1:
5299		: [ret] "={w0}" (-> u32),
5300		: [val] "{w0}" (val),
5301		[ptr] "{x1}" (ptr),
5302		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5303		: "w15", "w16", "w17", "memory"
5304		);
5305		}
5306		fn ldclr4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5307		@setRuntimeSafety(false);
5308		__init_aarch64_have_lse_atomics();
5309
5310		return asm volatile (
5311		\\ cbz w16, 8f
5312		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x000000
5313		\\ cbz wzr, 1f
5314		\\8:
5315		\\ mov w16, w0
5316		\\0:
5317		\\ ldxr w0, [x1]
5318		\\ bic w17, w0, w16
5319		\\ stxr w15, w17, [x1]
5320		\\ cbnz w15, 0b
5321		\\1:
5322		: [ret] "={w0}" (-> u32),
5323		: [val] "{w0}" (val),
5324		[ptr] "{x1}" (ptr),
5325		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5326		: "w15", "w16", "w17", "memory"
5327		);
5328		}
5329		const __aarch64_ldclr4_relax = if (is_darwin)
5330		if (has_lse)
5331		ldclr4RelaxDarwinLse
5332		else
5333		ldclr4RelaxDarwinNolse
5334		else if (has_lse)
5335		ldclr4RelaxNondarwinLse
5336		else
5337		ldclr4RelaxNondarwinNolse;
5338		fn ldeor4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5339		@setRuntimeSafety(false);
5340		__init_aarch64_have_lse_atomics();
5341
5342		return asm volatile (
5343		\\ cbz w16, 8f
5344		\\ ldeor w0, w0, [x1]
5345		\\ cbz wzr, 1f
5346		\\8:
5347		\\ mov w16, w0
5348		\\0:
5349		\\ ldxr w0, [x1]
5350		\\ eor w17, w0, w16
5351		\\ stxr w15, w17, [x1]
5352		\\ cbnz w15, 0b
5353		\\1:
5354		: [ret] "={w0}" (-> u32),
5355		: [val] "{w0}" (val),
5356		[ptr] "{x1}" (ptr),
5357		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5358		: "w15", "w16", "w17", "memory"
5359		);
5360		}
5361		fn ldeor4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5362		@setRuntimeSafety(false);
5363		__init_aarch64_have_lse_atomics();
5364
5365		return asm volatile (
5366		\\ cbz w16, 8f
5367		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
5368		\\ cbz wzr, 1f
5369		\\8:
5370		\\ mov w16, w0
5371		\\0:
5372		\\ ldxr w0, [x1]
5373		\\ eor w17, w0, w16
5374		\\ stxr w15, w17, [x1]
5375		\\ cbnz w15, 0b
5376		\\1:
5377		: [ret] "={w0}" (-> u32),
5378		: [val] "{w0}" (val),
5379		[ptr] "{x1}" (ptr),
5380		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5381		: "w15", "w16", "w17", "memory"
5382		);
5383		}
5384		fn ldeor4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5385		@setRuntimeSafety(false);
5386		__init_aarch64_have_lse_atomics();
5387
5388		return asm volatile (
5389		\\ cbz w16, 8f
5390		\\ ldeor w0, w0, [x1]
5391		\\ cbz wzr, 1f
5392		\\8:
5393		\\ mov w16, w0
5394		\\0:
5395		\\ ldxr w0, [x1]
5396		\\ eor w17, w0, w16
5397		\\ stxr w15, w17, [x1]
5398		\\ cbnz w15, 0b
5399		\\1:
5400		: [ret] "={w0}" (-> u32),
5401		: [val] "{w0}" (val),
5402		[ptr] "{x1}" (ptr),
5403		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5404		: "w15", "w16", "w17", "memory"
5405		);
5406		}
5407		fn ldeor4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5408		@setRuntimeSafety(false);
5409		__init_aarch64_have_lse_atomics();
5410
5411		return asm volatile (
5412		\\ cbz w16, 8f
5413		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x000000
5414		\\ cbz wzr, 1f
5415		\\8:
5416		\\ mov w16, w0
5417		\\0:
5418		\\ ldxr w0, [x1]
5419		\\ eor w17, w0, w16
5420		\\ stxr w15, w17, [x1]
5421		\\ cbnz w15, 0b
5422		\\1:
5423		: [ret] "={w0}" (-> u32),
5424		: [val] "{w0}" (val),
5425		[ptr] "{x1}" (ptr),
5426		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5427		: "w15", "w16", "w17", "memory"
5428		);
5429		}
5430		const __aarch64_ldeor4_relax = if (is_darwin)
5431		if (has_lse)
5432		ldeor4RelaxDarwinLse
5433		else
5434		ldeor4RelaxDarwinNolse
5435		else if (has_lse)
5436		ldeor4RelaxNondarwinLse
5437		else
5438		ldeor4RelaxNondarwinNolse;
5439		fn ldset4RelaxDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5440		@setRuntimeSafety(false);
5441		__init_aarch64_have_lse_atomics();
5442
5443		return asm volatile (
5444		\\ cbz w16, 8f
5445		\\ ldset w0, w0, [x1]
5446		\\ cbz wzr, 1f
5447		\\8:
5448		\\ mov w16, w0
5449		\\0:
5450		\\ ldxr w0, [x1]
5451		\\ orr w17, w0, w16
5452		\\ stxr w15, w17, [x1]
5453		\\ cbnz w15, 0b
5454		\\1:
5455		: [ret] "={w0}" (-> u32),
5456		: [val] "{w0}" (val),
5457		[ptr] "{x1}" (ptr),
5458		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5459		: "w15", "w16", "w17", "memory"
5460		);
5461		}
5462		fn ldset4RelaxDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5463		@setRuntimeSafety(false);
5464		__init_aarch64_have_lse_atomics();
5465
5466		return asm volatile (
5467		\\ cbz w16, 8f
5468		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
5469		\\ cbz wzr, 1f
5470		\\8:
5471		\\ mov w16, w0
5472		\\0:
5473		\\ ldxr w0, [x1]
5474		\\ orr w17, w0, w16
5475		\\ stxr w15, w17, [x1]
5476		\\ cbnz w15, 0b
5477		\\1:
5478		: [ret] "={w0}" (-> u32),
5479		: [val] "{w0}" (val),
5480		[ptr] "{x1}" (ptr),
5481		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5482		: "w15", "w16", "w17", "memory"
5483		);
5484		}
5485		fn ldset4RelaxNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5486		@setRuntimeSafety(false);
5487		__init_aarch64_have_lse_atomics();
5488
5489		return asm volatile (
5490		\\ cbz w16, 8f
5491		\\ ldset w0, w0, [x1]
5492		\\ cbz wzr, 1f
5493		\\8:
5494		\\ mov w16, w0
5495		\\0:
5496		\\ ldxr w0, [x1]
5497		\\ orr w17, w0, w16
5498		\\ stxr w15, w17, [x1]
5499		\\ cbnz w15, 0b
5500		\\1:
5501		: [ret] "={w0}" (-> u32),
5502		: [val] "{w0}" (val),
5503		[ptr] "{x1}" (ptr),
5504		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5505		: "w15", "w16", "w17", "memory"
5506		);
5507		}
5508		fn ldset4RelaxNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5509		@setRuntimeSafety(false);
5510		__init_aarch64_have_lse_atomics();
5511
5512		return asm volatile (
5513		\\ cbz w16, 8f
5514		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x000000
5515		\\ cbz wzr, 1f
5516		\\8:
5517		\\ mov w16, w0
5518		\\0:
5519		\\ ldxr w0, [x1]
5520		\\ orr w17, w0, w16
5521		\\ stxr w15, w17, [x1]
5522		\\ cbnz w15, 0b
5523		\\1:
5524		: [ret] "={w0}" (-> u32),
5525		: [val] "{w0}" (val),
5526		[ptr] "{x1}" (ptr),
5527		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5528		: "w15", "w16", "w17", "memory"
5529		);
5530		}
5531		const __aarch64_ldset4_relax = if (is_darwin)
5532		if (has_lse)
5533		ldset4RelaxDarwinLse
5534		else
5535		ldset4RelaxDarwinNolse
5536		else if (has_lse)
5537		ldset4RelaxNondarwinLse
5538		else
5539		ldset4RelaxNondarwinNolse;
5540		fn cas4AcqDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5541		@setRuntimeSafety(false);
5542		__init_aarch64_have_lse_atomics();
5543
5544		return asm volatile (
5545		\\ cbz w16, 8f
5546		\\ casa w0, w1, [x2]
5547		\\
5548		\\ cbz wzr, 1f
5549		\\8:
5550		\\ mov w16, w0
5551		\\0:
5552		\\ ldaxr w0, [x2]
5553		\\ cmp w0, w16
5554		\\ bne 1f
5555		\\ stxr w17, w1, [x2]
5556		\\ cbnz w17, 0b
5557		\\1:
5558		: [ret] "={w0}" (-> u32),
5559		: [expected] "{w0}" (expected),
5560		[desired] "{w1}" (desired),
5561		[ptr] "{x2}" (ptr),
5562		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5563		: "w15", "w16", "w17", "memory"
5564		);
5565		}
5566		fn cas4AcqDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5567		@setRuntimeSafety(false);
5568		__init_aarch64_have_lse_atomics();
5569
5570		return asm volatile (
5571		\\ cbz w16, 8f
5572		\\ .inst 0x08a07c41 + 0x80000000 + 0x400000
5573		\\
5574		\\ cbz wzr, 1f
5575		\\8:
5576		\\ mov w16, w0
5577		\\0:
5578		\\ ldaxr w0, [x2]
5579		\\ cmp w0, w16
5580		\\ bne 1f
5581		\\ stxr w17, w1, [x2]
5582		\\ cbnz w17, 0b
5583		\\1:
5584		: [ret] "={w0}" (-> u32),
5585		: [expected] "{w0}" (expected),
5586		[desired] "{w1}" (desired),
5587		[ptr] "{x2}" (ptr),
5588		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5589		: "w15", "w16", "w17", "memory"
5590		);
5591		}
5592		fn cas4AcqNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5593		@setRuntimeSafety(false);
5594		__init_aarch64_have_lse_atomics();
5595
5596		return asm volatile (
5597		\\ cbz w16, 8f
5598		\\ casa w0, w1, [x2]
5599		\\
5600		\\ cbz wzr, 1f
5601		\\8:
5602		\\ mov w16, w0
5603		\\0:
5604		\\ ldaxr w0, [x2]
5605		\\ cmp w0, w16
5606		\\ bne 1f
5607		\\ stxr w17, w1, [x2]
5608		\\ cbnz w17, 0b
5609		\\1:
5610		: [ret] "={w0}" (-> u32),
5611		: [expected] "{w0}" (expected),
5612		[desired] "{w1}" (desired),
5613		[ptr] "{x2}" (ptr),
5614		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5615		: "w15", "w16", "w17", "memory"
5616		);
5617		}
5618		fn cas4AcqNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
5619		@setRuntimeSafety(false);
5620		__init_aarch64_have_lse_atomics();
5621
5622		return asm volatile (
5623		\\ cbz w16, 8f
5624		\\ .inst 0x08a07c41 + 0x80000000 + 0x400000
5625		\\
5626		\\ cbz wzr, 1f
5627		\\8:
5628		\\ mov w16, w0
5629		\\0:
5630		\\ ldaxr w0, [x2]
5631		\\ cmp w0, w16
5632		\\ bne 1f
5633		\\ stxr w17, w1, [x2]
5634		\\ cbnz w17, 0b
5635		\\1:
5636		: [ret] "={w0}" (-> u32),
5637		: [expected] "{w0}" (expected),
5638		[desired] "{w1}" (desired),
5639		[ptr] "{x2}" (ptr),
5640		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5641		: "w15", "w16", "w17", "memory"
5642		);
5643		}
5644		const __aarch64_cas4_acq = if (is_darwin)
5645		if (has_lse)
5646		cas4AcqDarwinLse
5647		else
5648		cas4AcqDarwinNolse
5649		else if (has_lse)
5650		cas4AcqNondarwinLse
5651		else
5652		cas4AcqNondarwinNolse;
5653		fn swp4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5654		@setRuntimeSafety(false);
5655		__init_aarch64_have_lse_atomics();
5656
5657		return asm volatile (
5658		\\ cbz w16, 8f
5659		\\ swpa w0, w0, [x1]
5660		\\ cbz wzr, 1f
5661		\\8:
5662		\\ mov w16, w0
5663		\\0:
5664		\\ ldaxr w0, [x1]
5665		\\ stxr w17, w16, [x1]
5666		\\ cbnz w17, 0b
5667		\\1:
5668		: [ret] "={w0}" (-> u32),
5669		: [val] "{w0}" (val),
5670		[ptr] "{x1}" (ptr),
5671		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5672		: "w15", "w16", "w17", "memory"
5673		);
5674		}
5675		fn swp4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5676		@setRuntimeSafety(false);
5677		__init_aarch64_have_lse_atomics();
5678
5679		return asm volatile (
5680		\\ cbz w16, 8f
5681		\\ .inst 0x38208020 + 0x80000000 + 0x800000
5682		\\ cbz wzr, 1f
5683		\\8:
5684		\\ mov w16, w0
5685		\\0:
5686		\\ ldaxr w0, [x1]
5687		\\ stxr w17, w16, [x1]
5688		\\ cbnz w17, 0b
5689		\\1:
5690		: [ret] "={w0}" (-> u32),
5691		: [val] "{w0}" (val),
5692		[ptr] "{x1}" (ptr),
5693		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5694		: "w15", "w16", "w17", "memory"
5695		);
5696		}
5697		fn swp4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5698		@setRuntimeSafety(false);
5699		__init_aarch64_have_lse_atomics();
5700
5701		return asm volatile (
5702		\\ cbz w16, 8f
5703		\\ swpa w0, w0, [x1]
5704		\\ cbz wzr, 1f
5705		\\8:
5706		\\ mov w16, w0
5707		\\0:
5708		\\ ldaxr w0, [x1]
5709		\\ stxr w17, w16, [x1]
5710		\\ cbnz w17, 0b
5711		\\1:
5712		: [ret] "={w0}" (-> u32),
5713		: [val] "{w0}" (val),
5714		[ptr] "{x1}" (ptr),
5715		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5716		: "w15", "w16", "w17", "memory"
5717		);
5718		}
5719		fn swp4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5720		@setRuntimeSafety(false);
5721		__init_aarch64_have_lse_atomics();
5722
5723		return asm volatile (
5724		\\ cbz w16, 8f
5725		\\ .inst 0x38208020 + 0x80000000 + 0x800000
5726		\\ cbz wzr, 1f
5727		\\8:
5728		\\ mov w16, w0
5729		\\0:
5730		\\ ldaxr w0, [x1]
5731		\\ stxr w17, w16, [x1]
5732		\\ cbnz w17, 0b
5733		\\1:
5734		: [ret] "={w0}" (-> u32),
5735		: [val] "{w0}" (val),
5736		[ptr] "{x1}" (ptr),
5737		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5738		: "w15", "w16", "w17", "memory"
5739		);
5740		}
5741		const __aarch64_swp4_acq = if (is_darwin)
5742		if (has_lse)
5743		swp4AcqDarwinLse
5744		else
5745		swp4AcqDarwinNolse
5746		else if (has_lse)
5747		swp4AcqNondarwinLse
5748		else
5749		swp4AcqNondarwinNolse;
5750		fn ldadd4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5751		@setRuntimeSafety(false);
5752		__init_aarch64_have_lse_atomics();
5753
5754		return asm volatile (
5755		\\ cbz w16, 8f
5756		\\ ldadda w0, w0, [x1]
5757		\\ cbz wzr, 1f
5758		\\8:
5759		\\ mov w16, w0
5760		\\0:
5761		\\ ldaxr w0, [x1]
5762		\\ add w17, w0, w16
5763		\\ stxr w15, w17, [x1]
5764		\\ cbnz w15, 0b
5765		\\1:
5766		: [ret] "={w0}" (-> u32),
5767		: [val] "{w0}" (val),
5768		[ptr] "{x1}" (ptr),
5769		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5770		: "w15", "w16", "w17", "memory"
5771		);
5772		}
5773		fn ldadd4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5774		@setRuntimeSafety(false);
5775		__init_aarch64_have_lse_atomics();
5776
5777		return asm volatile (
5778		\\ cbz w16, 8f
5779		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
5780		\\ cbz wzr, 1f
5781		\\8:
5782		\\ mov w16, w0
5783		\\0:
5784		\\ ldaxr w0, [x1]
5785		\\ add w17, w0, w16
5786		\\ stxr w15, w17, [x1]
5787		\\ cbnz w15, 0b
5788		\\1:
5789		: [ret] "={w0}" (-> u32),
5790		: [val] "{w0}" (val),
5791		[ptr] "{x1}" (ptr),
5792		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5793		: "w15", "w16", "w17", "memory"
5794		);
5795		}
5796		fn ldadd4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5797		@setRuntimeSafety(false);
5798		__init_aarch64_have_lse_atomics();
5799
5800		return asm volatile (
5801		\\ cbz w16, 8f
5802		\\ ldadda w0, w0, [x1]
5803		\\ cbz wzr, 1f
5804		\\8:
5805		\\ mov w16, w0
5806		\\0:
5807		\\ ldaxr w0, [x1]
5808		\\ add w17, w0, w16
5809		\\ stxr w15, w17, [x1]
5810		\\ cbnz w15, 0b
5811		\\1:
5812		: [ret] "={w0}" (-> u32),
5813		: [val] "{w0}" (val),
5814		[ptr] "{x1}" (ptr),
5815		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5816		: "w15", "w16", "w17", "memory"
5817		);
5818		}
5819		fn ldadd4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5820		@setRuntimeSafety(false);
5821		__init_aarch64_have_lse_atomics();
5822
5823		return asm volatile (
5824		\\ cbz w16, 8f
5825		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x800000
5826		\\ cbz wzr, 1f
5827		\\8:
5828		\\ mov w16, w0
5829		\\0:
5830		\\ ldaxr w0, [x1]
5831		\\ add w17, w0, w16
5832		\\ stxr w15, w17, [x1]
5833		\\ cbnz w15, 0b
5834		\\1:
5835		: [ret] "={w0}" (-> u32),
5836		: [val] "{w0}" (val),
5837		[ptr] "{x1}" (ptr),
5838		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5839		: "w15", "w16", "w17", "memory"
5840		);
5841		}
5842		const __aarch64_ldadd4_acq = if (is_darwin)
5843		if (has_lse)
5844		ldadd4AcqDarwinLse
5845		else
5846		ldadd4AcqDarwinNolse
5847		else if (has_lse)
5848		ldadd4AcqNondarwinLse
5849		else
5850		ldadd4AcqNondarwinNolse;
5851		fn ldclr4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5852		@setRuntimeSafety(false);
5853		__init_aarch64_have_lse_atomics();
5854
5855		return asm volatile (
5856		\\ cbz w16, 8f
5857		\\ ldclra w0, w0, [x1]
5858		\\ cbz wzr, 1f
5859		\\8:
5860		\\ mov w16, w0
5861		\\0:
5862		\\ ldaxr w0, [x1]
5863		\\ bic w17, w0, w16
5864		\\ stxr w15, w17, [x1]
5865		\\ cbnz w15, 0b
5866		\\1:
5867		: [ret] "={w0}" (-> u32),
5868		: [val] "{w0}" (val),
5869		[ptr] "{x1}" (ptr),
5870		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5871		: "w15", "w16", "w17", "memory"
5872		);
5873		}
5874		fn ldclr4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5875		@setRuntimeSafety(false);
5876		__init_aarch64_have_lse_atomics();
5877
5878		return asm volatile (
5879		\\ cbz w16, 8f
5880		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
5881		\\ cbz wzr, 1f
5882		\\8:
5883		\\ mov w16, w0
5884		\\0:
5885		\\ ldaxr w0, [x1]
5886		\\ bic w17, w0, w16
5887		\\ stxr w15, w17, [x1]
5888		\\ cbnz w15, 0b
5889		\\1:
5890		: [ret] "={w0}" (-> u32),
5891		: [val] "{w0}" (val),
5892		[ptr] "{x1}" (ptr),
5893		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5894		: "w15", "w16", "w17", "memory"
5895		);
5896		}
5897		fn ldclr4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5898		@setRuntimeSafety(false);
5899		__init_aarch64_have_lse_atomics();
5900
5901		return asm volatile (
5902		\\ cbz w16, 8f
5903		\\ ldclra w0, w0, [x1]
5904		\\ cbz wzr, 1f
5905		\\8:
5906		\\ mov w16, w0
5907		\\0:
5908		\\ ldaxr w0, [x1]
5909		\\ bic w17, w0, w16
5910		\\ stxr w15, w17, [x1]
5911		\\ cbnz w15, 0b
5912		\\1:
5913		: [ret] "={w0}" (-> u32),
5914		: [val] "{w0}" (val),
5915		[ptr] "{x1}" (ptr),
5916		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5917		: "w15", "w16", "w17", "memory"
5918		);
5919		}
5920		fn ldclr4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5921		@setRuntimeSafety(false);
5922		__init_aarch64_have_lse_atomics();
5923
5924		return asm volatile (
5925		\\ cbz w16, 8f
5926		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x800000
5927		\\ cbz wzr, 1f
5928		\\8:
5929		\\ mov w16, w0
5930		\\0:
5931		\\ ldaxr w0, [x1]
5932		\\ bic w17, w0, w16
5933		\\ stxr w15, w17, [x1]
5934		\\ cbnz w15, 0b
5935		\\1:
5936		: [ret] "={w0}" (-> u32),
5937		: [val] "{w0}" (val),
5938		[ptr] "{x1}" (ptr),
5939		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5940		: "w15", "w16", "w17", "memory"
5941		);
5942		}
5943		const __aarch64_ldclr4_acq = if (is_darwin)
5944		if (has_lse)
5945		ldclr4AcqDarwinLse
5946		else
5947		ldclr4AcqDarwinNolse
5948		else if (has_lse)
5949		ldclr4AcqNondarwinLse
5950		else
5951		ldclr4AcqNondarwinNolse;
5952		fn ldeor4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5953		@setRuntimeSafety(false);
5954		__init_aarch64_have_lse_atomics();
5955
5956		return asm volatile (
5957		\\ cbz w16, 8f
5958		\\ ldeora w0, w0, [x1]
5959		\\ cbz wzr, 1f
5960		\\8:
5961		\\ mov w16, w0
5962		\\0:
5963		\\ ldaxr w0, [x1]
5964		\\ eor w17, w0, w16
5965		\\ stxr w15, w17, [x1]
5966		\\ cbnz w15, 0b
5967		\\1:
5968		: [ret] "={w0}" (-> u32),
5969		: [val] "{w0}" (val),
5970		[ptr] "{x1}" (ptr),
5971		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5972		: "w15", "w16", "w17", "memory"
5973		);
5974		}
5975		fn ldeor4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
5976		@setRuntimeSafety(false);
5977		__init_aarch64_have_lse_atomics();
5978
5979		return asm volatile (
5980		\\ cbz w16, 8f
5981		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
5982		\\ cbz wzr, 1f
5983		\\8:
5984		\\ mov w16, w0
5985		\\0:
5986		\\ ldaxr w0, [x1]
5987		\\ eor w17, w0, w16
5988		\\ stxr w15, w17, [x1]
5989		\\ cbnz w15, 0b
5990		\\1:
5991		: [ret] "={w0}" (-> u32),
5992		: [val] "{w0}" (val),
5993		[ptr] "{x1}" (ptr),
5994		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
5995		: "w15", "w16", "w17", "memory"
5996		);
5997		}
5998		fn ldeor4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
5999		@setRuntimeSafety(false);
6000		__init_aarch64_have_lse_atomics();
6001
6002		return asm volatile (
6003		\\ cbz w16, 8f
6004		\\ ldeora w0, w0, [x1]
6005		\\ cbz wzr, 1f
6006		\\8:
6007		\\ mov w16, w0
6008		\\0:
6009		\\ ldaxr w0, [x1]
6010		\\ eor w17, w0, w16
6011		\\ stxr w15, w17, [x1]
6012		\\ cbnz w15, 0b
6013		\\1:
6014		: [ret] "={w0}" (-> u32),
6015		: [val] "{w0}" (val),
6016		[ptr] "{x1}" (ptr),
6017		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6018		: "w15", "w16", "w17", "memory"
6019		);
6020		}
6021		fn ldeor4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6022		@setRuntimeSafety(false);
6023		__init_aarch64_have_lse_atomics();
6024
6025		return asm volatile (
6026		\\ cbz w16, 8f
6027		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x800000
6028		\\ cbz wzr, 1f
6029		\\8:
6030		\\ mov w16, w0
6031		\\0:
6032		\\ ldaxr w0, [x1]
6033		\\ eor w17, w0, w16
6034		\\ stxr w15, w17, [x1]
6035		\\ cbnz w15, 0b
6036		\\1:
6037		: [ret] "={w0}" (-> u32),
6038		: [val] "{w0}" (val),
6039		[ptr] "{x1}" (ptr),
6040		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6041		: "w15", "w16", "w17", "memory"
6042		);
6043		}
6044		const __aarch64_ldeor4_acq = if (is_darwin)
6045		if (has_lse)
6046		ldeor4AcqDarwinLse
6047		else
6048		ldeor4AcqDarwinNolse
6049		else if (has_lse)
6050		ldeor4AcqNondarwinLse
6051		else
6052		ldeor4AcqNondarwinNolse;
6053		fn ldset4AcqDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6054		@setRuntimeSafety(false);
6055		__init_aarch64_have_lse_atomics();
6056
6057		return asm volatile (
6058		\\ cbz w16, 8f
6059		\\ ldseta w0, w0, [x1]
6060		\\ cbz wzr, 1f
6061		\\8:
6062		\\ mov w16, w0
6063		\\0:
6064		\\ ldaxr w0, [x1]
6065		\\ orr w17, w0, w16
6066		\\ stxr w15, w17, [x1]
6067		\\ cbnz w15, 0b
6068		\\1:
6069		: [ret] "={w0}" (-> u32),
6070		: [val] "{w0}" (val),
6071		[ptr] "{x1}" (ptr),
6072		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6073		: "w15", "w16", "w17", "memory"
6074		);
6075		}
6076		fn ldset4AcqDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6077		@setRuntimeSafety(false);
6078		__init_aarch64_have_lse_atomics();
6079
6080		return asm volatile (
6081		\\ cbz w16, 8f
6082		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
6083		\\ cbz wzr, 1f
6084		\\8:
6085		\\ mov w16, w0
6086		\\0:
6087		\\ ldaxr w0, [x1]
6088		\\ orr w17, w0, w16
6089		\\ stxr w15, w17, [x1]
6090		\\ cbnz w15, 0b
6091		\\1:
6092		: [ret] "={w0}" (-> u32),
6093		: [val] "{w0}" (val),
6094		[ptr] "{x1}" (ptr),
6095		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6096		: "w15", "w16", "w17", "memory"
6097		);
6098		}
6099		fn ldset4AcqNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6100		@setRuntimeSafety(false);
6101		__init_aarch64_have_lse_atomics();
6102
6103		return asm volatile (
6104		\\ cbz w16, 8f
6105		\\ ldseta w0, w0, [x1]
6106		\\ cbz wzr, 1f
6107		\\8:
6108		\\ mov w16, w0
6109		\\0:
6110		\\ ldaxr w0, [x1]
6111		\\ orr w17, w0, w16
6112		\\ stxr w15, w17, [x1]
6113		\\ cbnz w15, 0b
6114		\\1:
6115		: [ret] "={w0}" (-> u32),
6116		: [val] "{w0}" (val),
6117		[ptr] "{x1}" (ptr),
6118		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6119		: "w15", "w16", "w17", "memory"
6120		);
6121		}
6122		fn ldset4AcqNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6123		@setRuntimeSafety(false);
6124		__init_aarch64_have_lse_atomics();
6125
6126		return asm volatile (
6127		\\ cbz w16, 8f
6128		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x800000
6129		\\ cbz wzr, 1f
6130		\\8:
6131		\\ mov w16, w0
6132		\\0:
6133		\\ ldaxr w0, [x1]
6134		\\ orr w17, w0, w16
6135		\\ stxr w15, w17, [x1]
6136		\\ cbnz w15, 0b
6137		\\1:
6138		: [ret] "={w0}" (-> u32),
6139		: [val] "{w0}" (val),
6140		[ptr] "{x1}" (ptr),
6141		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6142		: "w15", "w16", "w17", "memory"
6143		);
6144		}
6145		const __aarch64_ldset4_acq = if (is_darwin)
6146		if (has_lse)
6147		ldset4AcqDarwinLse
6148		else
6149		ldset4AcqDarwinNolse
6150		else if (has_lse)
6151		ldset4AcqNondarwinLse
6152		else
6153		ldset4AcqNondarwinNolse;
6154		fn cas4RelDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6155		@setRuntimeSafety(false);
6156		__init_aarch64_have_lse_atomics();
6157
6158		return asm volatile (
6159		\\ cbz w16, 8f
6160		\\ casl w0, w1, [x2]
6161		\\
6162		\\ cbz wzr, 1f
6163		\\8:
6164		\\ mov w16, w0
6165		\\0:
6166		\\ ldxr w0, [x2]
6167		\\ cmp w0, w16
6168		\\ bne 1f
6169		\\ stlxr w17, w1, [x2]
6170		\\ cbnz w17, 0b
6171		\\1:
6172		: [ret] "={w0}" (-> u32),
6173		: [expected] "{w0}" (expected),
6174		[desired] "{w1}" (desired),
6175		[ptr] "{x2}" (ptr),
6176		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6177		: "w15", "w16", "w17", "memory"
6178		);
6179		}
6180		fn cas4RelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6181		@setRuntimeSafety(false);
6182		__init_aarch64_have_lse_atomics();
6183
6184		return asm volatile (
6185		\\ cbz w16, 8f
6186		\\ .inst 0x08a07c41 + 0x80000000 + 0x008000
6187		\\
6188		\\ cbz wzr, 1f
6189		\\8:
6190		\\ mov w16, w0
6191		\\0:
6192		\\ ldxr w0, [x2]
6193		\\ cmp w0, w16
6194		\\ bne 1f
6195		\\ stlxr w17, w1, [x2]
6196		\\ cbnz w17, 0b
6197		\\1:
6198		: [ret] "={w0}" (-> u32),
6199		: [expected] "{w0}" (expected),
6200		[desired] "{w1}" (desired),
6201		[ptr] "{x2}" (ptr),
6202		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6203		: "w15", "w16", "w17", "memory"
6204		);
6205		}
6206		fn cas4RelNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6207		@setRuntimeSafety(false);
6208		__init_aarch64_have_lse_atomics();
6209
6210		return asm volatile (
6211		\\ cbz w16, 8f
6212		\\ casl w0, w1, [x2]
6213		\\
6214		\\ cbz wzr, 1f
6215		\\8:
6216		\\ mov w16, w0
6217		\\0:
6218		\\ ldxr w0, [x2]
6219		\\ cmp w0, w16
6220		\\ bne 1f
6221		\\ stlxr w17, w1, [x2]
6222		\\ cbnz w17, 0b
6223		\\1:
6224		: [ret] "={w0}" (-> u32),
6225		: [expected] "{w0}" (expected),
6226		[desired] "{w1}" (desired),
6227		[ptr] "{x2}" (ptr),
6228		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6229		: "w15", "w16", "w17", "memory"
6230		);
6231		}
6232		fn cas4RelNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6233		@setRuntimeSafety(false);
6234		__init_aarch64_have_lse_atomics();
6235
6236		return asm volatile (
6237		\\ cbz w16, 8f
6238		\\ .inst 0x08a07c41 + 0x80000000 + 0x008000
6239		\\
6240		\\ cbz wzr, 1f
6241		\\8:
6242		\\ mov w16, w0
6243		\\0:
6244		\\ ldxr w0, [x2]
6245		\\ cmp w0, w16
6246		\\ bne 1f
6247		\\ stlxr w17, w1, [x2]
6248		\\ cbnz w17, 0b
6249		\\1:
6250		: [ret] "={w0}" (-> u32),
6251		: [expected] "{w0}" (expected),
6252		[desired] "{w1}" (desired),
6253		[ptr] "{x2}" (ptr),
6254		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6255		: "w15", "w16", "w17", "memory"
6256		);
6257		}
6258		const __aarch64_cas4_rel = if (is_darwin)
6259		if (has_lse)
6260		cas4RelDarwinLse
6261		else
6262		cas4RelDarwinNolse
6263		else if (has_lse)
6264		cas4RelNondarwinLse
6265		else
6266		cas4RelNondarwinNolse;
6267		fn swp4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6268		@setRuntimeSafety(false);
6269		__init_aarch64_have_lse_atomics();
6270
6271		return asm volatile (
6272		\\ cbz w16, 8f
6273		\\ swpl w0, w0, [x1]
6274		\\ cbz wzr, 1f
6275		\\8:
6276		\\ mov w16, w0
6277		\\0:
6278		\\ ldxr w0, [x1]
6279		\\ stlxr w17, w16, [x1]
6280		\\ cbnz w17, 0b
6281		\\1:
6282		: [ret] "={w0}" (-> u32),
6283		: [val] "{w0}" (val),
6284		[ptr] "{x1}" (ptr),
6285		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6286		: "w15", "w16", "w17", "memory"
6287		);
6288		}
6289		fn swp4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6290		@setRuntimeSafety(false);
6291		__init_aarch64_have_lse_atomics();
6292
6293		return asm volatile (
6294		\\ cbz w16, 8f
6295		\\ .inst 0x38208020 + 0x80000000 + 0x400000
6296		\\ cbz wzr, 1f
6297		\\8:
6298		\\ mov w16, w0
6299		\\0:
6300		\\ ldxr w0, [x1]
6301		\\ stlxr w17, w16, [x1]
6302		\\ cbnz w17, 0b
6303		\\1:
6304		: [ret] "={w0}" (-> u32),
6305		: [val] "{w0}" (val),
6306		[ptr] "{x1}" (ptr),
6307		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6308		: "w15", "w16", "w17", "memory"
6309		);
6310		}
6311		fn swp4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6312		@setRuntimeSafety(false);
6313		__init_aarch64_have_lse_atomics();
6314
6315		return asm volatile (
6316		\\ cbz w16, 8f
6317		\\ swpl w0, w0, [x1]
6318		\\ cbz wzr, 1f
6319		\\8:
6320		\\ mov w16, w0
6321		\\0:
6322		\\ ldxr w0, [x1]
6323		\\ stlxr w17, w16, [x1]
6324		\\ cbnz w17, 0b
6325		\\1:
6326		: [ret] "={w0}" (-> u32),
6327		: [val] "{w0}" (val),
6328		[ptr] "{x1}" (ptr),
6329		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6330		: "w15", "w16", "w17", "memory"
6331		);
6332		}
6333		fn swp4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6334		@setRuntimeSafety(false);
6335		__init_aarch64_have_lse_atomics();
6336
6337		return asm volatile (
6338		\\ cbz w16, 8f
6339		\\ .inst 0x38208020 + 0x80000000 + 0x400000
6340		\\ cbz wzr, 1f
6341		\\8:
6342		\\ mov w16, w0
6343		\\0:
6344		\\ ldxr w0, [x1]
6345		\\ stlxr w17, w16, [x1]
6346		\\ cbnz w17, 0b
6347		\\1:
6348		: [ret] "={w0}" (-> u32),
6349		: [val] "{w0}" (val),
6350		[ptr] "{x1}" (ptr),
6351		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6352		: "w15", "w16", "w17", "memory"
6353		);
6354		}
6355		const __aarch64_swp4_rel = if (is_darwin)
6356		if (has_lse)
6357		swp4RelDarwinLse
6358		else
6359		swp4RelDarwinNolse
6360		else if (has_lse)
6361		swp4RelNondarwinLse
6362		else
6363		swp4RelNondarwinNolse;
6364		fn ldadd4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6365		@setRuntimeSafety(false);
6366		__init_aarch64_have_lse_atomics();
6367
6368		return asm volatile (
6369		\\ cbz w16, 8f
6370		\\ ldaddl w0, w0, [x1]
6371		\\ cbz wzr, 1f
6372		\\8:
6373		\\ mov w16, w0
6374		\\0:
6375		\\ ldxr w0, [x1]
6376		\\ add w17, w0, w16
6377		\\ stlxr w15, w17, [x1]
6378		\\ cbnz w15, 0b
6379		\\1:
6380		: [ret] "={w0}" (-> u32),
6381		: [val] "{w0}" (val),
6382		[ptr] "{x1}" (ptr),
6383		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6384		: "w15", "w16", "w17", "memory"
6385		);
6386		}
6387		fn ldadd4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6388		@setRuntimeSafety(false);
6389		__init_aarch64_have_lse_atomics();
6390
6391		return asm volatile (
6392		\\ cbz w16, 8f
6393		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
6394		\\ cbz wzr, 1f
6395		\\8:
6396		\\ mov w16, w0
6397		\\0:
6398		\\ ldxr w0, [x1]
6399		\\ add w17, w0, w16
6400		\\ stlxr w15, w17, [x1]
6401		\\ cbnz w15, 0b
6402		\\1:
6403		: [ret] "={w0}" (-> u32),
6404		: [val] "{w0}" (val),
6405		[ptr] "{x1}" (ptr),
6406		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6407		: "w15", "w16", "w17", "memory"
6408		);
6409		}
6410		fn ldadd4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6411		@setRuntimeSafety(false);
6412		__init_aarch64_have_lse_atomics();
6413
6414		return asm volatile (
6415		\\ cbz w16, 8f
6416		\\ ldaddl w0, w0, [x1]
6417		\\ cbz wzr, 1f
6418		\\8:
6419		\\ mov w16, w0
6420		\\0:
6421		\\ ldxr w0, [x1]
6422		\\ add w17, w0, w16
6423		\\ stlxr w15, w17, [x1]
6424		\\ cbnz w15, 0b
6425		\\1:
6426		: [ret] "={w0}" (-> u32),
6427		: [val] "{w0}" (val),
6428		[ptr] "{x1}" (ptr),
6429		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6430		: "w15", "w16", "w17", "memory"
6431		);
6432		}
6433		fn ldadd4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6434		@setRuntimeSafety(false);
6435		__init_aarch64_have_lse_atomics();
6436
6437		return asm volatile (
6438		\\ cbz w16, 8f
6439		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0x400000
6440		\\ cbz wzr, 1f
6441		\\8:
6442		\\ mov w16, w0
6443		\\0:
6444		\\ ldxr w0, [x1]
6445		\\ add w17, w0, w16
6446		\\ stlxr w15, w17, [x1]
6447		\\ cbnz w15, 0b
6448		\\1:
6449		: [ret] "={w0}" (-> u32),
6450		: [val] "{w0}" (val),
6451		[ptr] "{x1}" (ptr),
6452		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6453		: "w15", "w16", "w17", "memory"
6454		);
6455		}
6456		const __aarch64_ldadd4_rel = if (is_darwin)
6457		if (has_lse)
6458		ldadd4RelDarwinLse
6459		else
6460		ldadd4RelDarwinNolse
6461		else if (has_lse)
6462		ldadd4RelNondarwinLse
6463		else
6464		ldadd4RelNondarwinNolse;
6465		fn ldclr4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6466		@setRuntimeSafety(false);
6467		__init_aarch64_have_lse_atomics();
6468
6469		return asm volatile (
6470		\\ cbz w16, 8f
6471		\\ ldclrl w0, w0, [x1]
6472		\\ cbz wzr, 1f
6473		\\8:
6474		\\ mov w16, w0
6475		\\0:
6476		\\ ldxr w0, [x1]
6477		\\ bic w17, w0, w16
6478		\\ stlxr w15, w17, [x1]
6479		\\ cbnz w15, 0b
6480		\\1:
6481		: [ret] "={w0}" (-> u32),
6482		: [val] "{w0}" (val),
6483		[ptr] "{x1}" (ptr),
6484		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6485		: "w15", "w16", "w17", "memory"
6486		);
6487		}
6488		fn ldclr4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6489		@setRuntimeSafety(false);
6490		__init_aarch64_have_lse_atomics();
6491
6492		return asm volatile (
6493		\\ cbz w16, 8f
6494		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
6495		\\ cbz wzr, 1f
6496		\\8:
6497		\\ mov w16, w0
6498		\\0:
6499		\\ ldxr w0, [x1]
6500		\\ bic w17, w0, w16
6501		\\ stlxr w15, w17, [x1]
6502		\\ cbnz w15, 0b
6503		\\1:
6504		: [ret] "={w0}" (-> u32),
6505		: [val] "{w0}" (val),
6506		[ptr] "{x1}" (ptr),
6507		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6508		: "w15", "w16", "w17", "memory"
6509		);
6510		}
6511		fn ldclr4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6512		@setRuntimeSafety(false);
6513		__init_aarch64_have_lse_atomics();
6514
6515		return asm volatile (
6516		\\ cbz w16, 8f
6517		\\ ldclrl w0, w0, [x1]
6518		\\ cbz wzr, 1f
6519		\\8:
6520		\\ mov w16, w0
6521		\\0:
6522		\\ ldxr w0, [x1]
6523		\\ bic w17, w0, w16
6524		\\ stlxr w15, w17, [x1]
6525		\\ cbnz w15, 0b
6526		\\1:
6527		: [ret] "={w0}" (-> u32),
6528		: [val] "{w0}" (val),
6529		[ptr] "{x1}" (ptr),
6530		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6531		: "w15", "w16", "w17", "memory"
6532		);
6533		}
6534		fn ldclr4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6535		@setRuntimeSafety(false);
6536		__init_aarch64_have_lse_atomics();
6537
6538		return asm volatile (
6539		\\ cbz w16, 8f
6540		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0x400000
6541		\\ cbz wzr, 1f
6542		\\8:
6543		\\ mov w16, w0
6544		\\0:
6545		\\ ldxr w0, [x1]
6546		\\ bic w17, w0, w16
6547		\\ stlxr w15, w17, [x1]
6548		\\ cbnz w15, 0b
6549		\\1:
6550		: [ret] "={w0}" (-> u32),
6551		: [val] "{w0}" (val),
6552		[ptr] "{x1}" (ptr),
6553		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6554		: "w15", "w16", "w17", "memory"
6555		);
6556		}
6557		const __aarch64_ldclr4_rel = if (is_darwin)
6558		if (has_lse)
6559		ldclr4RelDarwinLse
6560		else
6561		ldclr4RelDarwinNolse
6562		else if (has_lse)
6563		ldclr4RelNondarwinLse
6564		else
6565		ldclr4RelNondarwinNolse;
6566		fn ldeor4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6567		@setRuntimeSafety(false);
6568		__init_aarch64_have_lse_atomics();
6569
6570		return asm volatile (
6571		\\ cbz w16, 8f
6572		\\ ldeorl w0, w0, [x1]
6573		\\ cbz wzr, 1f
6574		\\8:
6575		\\ mov w16, w0
6576		\\0:
6577		\\ ldxr w0, [x1]
6578		\\ eor w17, w0, w16
6579		\\ stlxr w15, w17, [x1]
6580		\\ cbnz w15, 0b
6581		\\1:
6582		: [ret] "={w0}" (-> u32),
6583		: [val] "{w0}" (val),
6584		[ptr] "{x1}" (ptr),
6585		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6586		: "w15", "w16", "w17", "memory"
6587		);
6588		}
6589		fn ldeor4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6590		@setRuntimeSafety(false);
6591		__init_aarch64_have_lse_atomics();
6592
6593		return asm volatile (
6594		\\ cbz w16, 8f
6595		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
6596		\\ cbz wzr, 1f
6597		\\8:
6598		\\ mov w16, w0
6599		\\0:
6600		\\ ldxr w0, [x1]
6601		\\ eor w17, w0, w16
6602		\\ stlxr w15, w17, [x1]
6603		\\ cbnz w15, 0b
6604		\\1:
6605		: [ret] "={w0}" (-> u32),
6606		: [val] "{w0}" (val),
6607		[ptr] "{x1}" (ptr),
6608		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6609		: "w15", "w16", "w17", "memory"
6610		);
6611		}
6612		fn ldeor4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6613		@setRuntimeSafety(false);
6614		__init_aarch64_have_lse_atomics();
6615
6616		return asm volatile (
6617		\\ cbz w16, 8f
6618		\\ ldeorl w0, w0, [x1]
6619		\\ cbz wzr, 1f
6620		\\8:
6621		\\ mov w16, w0
6622		\\0:
6623		\\ ldxr w0, [x1]
6624		\\ eor w17, w0, w16
6625		\\ stlxr w15, w17, [x1]
6626		\\ cbnz w15, 0b
6627		\\1:
6628		: [ret] "={w0}" (-> u32),
6629		: [val] "{w0}" (val),
6630		[ptr] "{x1}" (ptr),
6631		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6632		: "w15", "w16", "w17", "memory"
6633		);
6634		}
6635		fn ldeor4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6636		@setRuntimeSafety(false);
6637		__init_aarch64_have_lse_atomics();
6638
6639		return asm volatile (
6640		\\ cbz w16, 8f
6641		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0x400000
6642		\\ cbz wzr, 1f
6643		\\8:
6644		\\ mov w16, w0
6645		\\0:
6646		\\ ldxr w0, [x1]
6647		\\ eor w17, w0, w16
6648		\\ stlxr w15, w17, [x1]
6649		\\ cbnz w15, 0b
6650		\\1:
6651		: [ret] "={w0}" (-> u32),
6652		: [val] "{w0}" (val),
6653		[ptr] "{x1}" (ptr),
6654		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6655		: "w15", "w16", "w17", "memory"
6656		);
6657		}
6658		const __aarch64_ldeor4_rel = if (is_darwin)
6659		if (has_lse)
6660		ldeor4RelDarwinLse
6661		else
6662		ldeor4RelDarwinNolse
6663		else if (has_lse)
6664		ldeor4RelNondarwinLse
6665		else
6666		ldeor4RelNondarwinNolse;
6667		fn ldset4RelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6668		@setRuntimeSafety(false);
6669		__init_aarch64_have_lse_atomics();
6670
6671		return asm volatile (
6672		\\ cbz w16, 8f
6673		\\ ldsetl w0, w0, [x1]
6674		\\ cbz wzr, 1f
6675		\\8:
6676		\\ mov w16, w0
6677		\\0:
6678		\\ ldxr w0, [x1]
6679		\\ orr w17, w0, w16
6680		\\ stlxr w15, w17, [x1]
6681		\\ cbnz w15, 0b
6682		\\1:
6683		: [ret] "={w0}" (-> u32),
6684		: [val] "{w0}" (val),
6685		[ptr] "{x1}" (ptr),
6686		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6687		: "w15", "w16", "w17", "memory"
6688		);
6689		}
6690		fn ldset4RelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6691		@setRuntimeSafety(false);
6692		__init_aarch64_have_lse_atomics();
6693
6694		return asm volatile (
6695		\\ cbz w16, 8f
6696		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
6697		\\ cbz wzr, 1f
6698		\\8:
6699		\\ mov w16, w0
6700		\\0:
6701		\\ ldxr w0, [x1]
6702		\\ orr w17, w0, w16
6703		\\ stlxr w15, w17, [x1]
6704		\\ cbnz w15, 0b
6705		\\1:
6706		: [ret] "={w0}" (-> u32),
6707		: [val] "{w0}" (val),
6708		[ptr] "{x1}" (ptr),
6709		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6710		: "w15", "w16", "w17", "memory"
6711		);
6712		}
6713		fn ldset4RelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6714		@setRuntimeSafety(false);
6715		__init_aarch64_have_lse_atomics();
6716
6717		return asm volatile (
6718		\\ cbz w16, 8f
6719		\\ ldsetl w0, w0, [x1]
6720		\\ cbz wzr, 1f
6721		\\8:
6722		\\ mov w16, w0
6723		\\0:
6724		\\ ldxr w0, [x1]
6725		\\ orr w17, w0, w16
6726		\\ stlxr w15, w17, [x1]
6727		\\ cbnz w15, 0b
6728		\\1:
6729		: [ret] "={w0}" (-> u32),
6730		: [val] "{w0}" (val),
6731		[ptr] "{x1}" (ptr),
6732		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6733		: "w15", "w16", "w17", "memory"
6734		);
6735		}
6736		fn ldset4RelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6737		@setRuntimeSafety(false);
6738		__init_aarch64_have_lse_atomics();
6739
6740		return asm volatile (
6741		\\ cbz w16, 8f
6742		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0x400000
6743		\\ cbz wzr, 1f
6744		\\8:
6745		\\ mov w16, w0
6746		\\0:
6747		\\ ldxr w0, [x1]
6748		\\ orr w17, w0, w16
6749		\\ stlxr w15, w17, [x1]
6750		\\ cbnz w15, 0b
6751		\\1:
6752		: [ret] "={w0}" (-> u32),
6753		: [val] "{w0}" (val),
6754		[ptr] "{x1}" (ptr),
6755		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6756		: "w15", "w16", "w17", "memory"
6757		);
6758		}
6759		const __aarch64_ldset4_rel = if (is_darwin)
6760		if (has_lse)
6761		ldset4RelDarwinLse
6762		else
6763		ldset4RelDarwinNolse
6764		else if (has_lse)
6765		ldset4RelNondarwinLse
6766		else
6767		ldset4RelNondarwinNolse;
6768		fn cas4AcqRelDarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6769		@setRuntimeSafety(false);
6770		__init_aarch64_have_lse_atomics();
6771
6772		return asm volatile (
6773		\\ cbz w16, 8f
6774		\\ casal w0, w1, [x2]
6775		\\
6776		\\ cbz wzr, 1f
6777		\\8:
6778		\\ mov w16, w0
6779		\\0:
6780		\\ ldaxr w0, [x2]
6781		\\ cmp w0, w16
6782		\\ bne 1f
6783		\\ stlxr w17, w1, [x2]
6784		\\ cbnz w17, 0b
6785		\\1:
6786		: [ret] "={w0}" (-> u32),
6787		: [expected] "{w0}" (expected),
6788		[desired] "{w1}" (desired),
6789		[ptr] "{x2}" (ptr),
6790		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6791		: "w15", "w16", "w17", "memory"
6792		);
6793		}
6794		fn cas4AcqRelDarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6795		@setRuntimeSafety(false);
6796		__init_aarch64_have_lse_atomics();
6797
6798		return asm volatile (
6799		\\ cbz w16, 8f
6800		\\ .inst 0x08a07c41 + 0x80000000 + 0x408000
6801		\\
6802		\\ cbz wzr, 1f
6803		\\8:
6804		\\ mov w16, w0
6805		\\0:
6806		\\ ldaxr w0, [x2]
6807		\\ cmp w0, w16
6808		\\ bne 1f
6809		\\ stlxr w17, w1, [x2]
6810		\\ cbnz w17, 0b
6811		\\1:
6812		: [ret] "={w0}" (-> u32),
6813		: [expected] "{w0}" (expected),
6814		[desired] "{w1}" (desired),
6815		[ptr] "{x2}" (ptr),
6816		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6817		: "w15", "w16", "w17", "memory"
6818		);
6819		}
6820		fn cas4AcqRelNondarwinLse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6821		@setRuntimeSafety(false);
6822		__init_aarch64_have_lse_atomics();
6823
6824		return asm volatile (
6825		\\ cbz w16, 8f
6826		\\ casal w0, w1, [x2]
6827		\\
6828		\\ cbz wzr, 1f
6829		\\8:
6830		\\ mov w16, w0
6831		\\0:
6832		\\ ldaxr w0, [x2]
6833		\\ cmp w0, w16
6834		\\ bne 1f
6835		\\ stlxr w17, w1, [x2]
6836		\\ cbnz w17, 0b
6837		\\1:
6838		: [ret] "={w0}" (-> u32),
6839		: [expected] "{w0}" (expected),
6840		[desired] "{w1}" (desired),
6841		[ptr] "{x2}" (ptr),
6842		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6843		: "w15", "w16", "w17", "memory"
6844		);
6845		}
6846		fn cas4AcqRelNondarwinNolse(expected: u32, desired: u32, ptr: *u32) callconv(.C) u32 {
6847		@setRuntimeSafety(false);
6848		__init_aarch64_have_lse_atomics();
6849
6850		return asm volatile (
6851		\\ cbz w16, 8f
6852		\\ .inst 0x08a07c41 + 0x80000000 + 0x408000
6853		\\
6854		\\ cbz wzr, 1f
6855		\\8:
6856		\\ mov w16, w0
6857		\\0:
6858		\\ ldaxr w0, [x2]
6859		\\ cmp w0, w16
6860		\\ bne 1f
6861		\\ stlxr w17, w1, [x2]
6862		\\ cbnz w17, 0b
6863		\\1:
6864		: [ret] "={w0}" (-> u32),
6865		: [expected] "{w0}" (expected),
6866		[desired] "{w1}" (desired),
6867		[ptr] "{x2}" (ptr),
6868		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6869		: "w15", "w16", "w17", "memory"
6870		);
6871		}
6872		const __aarch64_cas4_acq_rel = if (is_darwin)
6873		if (has_lse)
6874		cas4AcqRelDarwinLse
6875		else
6876		cas4AcqRelDarwinNolse
6877		else if (has_lse)
6878		cas4AcqRelNondarwinLse
6879		else
6880		cas4AcqRelNondarwinNolse;
6881		fn swp4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6882		@setRuntimeSafety(false);
6883		__init_aarch64_have_lse_atomics();
6884
6885		return asm volatile (
6886		\\ cbz w16, 8f
6887		\\ swpal w0, w0, [x1]
6888		\\ cbz wzr, 1f
6889		\\8:
6890		\\ mov w16, w0
6891		\\0:
6892		\\ ldaxr w0, [x1]
6893		\\ stlxr w17, w16, [x1]
6894		\\ cbnz w17, 0b
6895		\\1:
6896		: [ret] "={w0}" (-> u32),
6897		: [val] "{w0}" (val),
6898		[ptr] "{x1}" (ptr),
6899		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6900		: "w15", "w16", "w17", "memory"
6901		);
6902		}
6903		fn swp4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6904		@setRuntimeSafety(false);
6905		__init_aarch64_have_lse_atomics();
6906
6907		return asm volatile (
6908		\\ cbz w16, 8f
6909		\\ .inst 0x38208020 + 0x80000000 + 0xc00000
6910		\\ cbz wzr, 1f
6911		\\8:
6912		\\ mov w16, w0
6913		\\0:
6914		\\ ldaxr w0, [x1]
6915		\\ stlxr w17, w16, [x1]
6916		\\ cbnz w17, 0b
6917		\\1:
6918		: [ret] "={w0}" (-> u32),
6919		: [val] "{w0}" (val),
6920		[ptr] "{x1}" (ptr),
6921		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6922		: "w15", "w16", "w17", "memory"
6923		);
6924		}
6925		fn swp4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6926		@setRuntimeSafety(false);
6927		__init_aarch64_have_lse_atomics();
6928
6929		return asm volatile (
6930		\\ cbz w16, 8f
6931		\\ swpal w0, w0, [x1]
6932		\\ cbz wzr, 1f
6933		\\8:
6934		\\ mov w16, w0
6935		\\0:
6936		\\ ldaxr w0, [x1]
6937		\\ stlxr w17, w16, [x1]
6938		\\ cbnz w17, 0b
6939		\\1:
6940		: [ret] "={w0}" (-> u32),
6941		: [val] "{w0}" (val),
6942		[ptr] "{x1}" (ptr),
6943		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6944		: "w15", "w16", "w17", "memory"
6945		);
6946		}
6947		fn swp4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
6948		@setRuntimeSafety(false);
6949		__init_aarch64_have_lse_atomics();
6950
6951		return asm volatile (
6952		\\ cbz w16, 8f
6953		\\ .inst 0x38208020 + 0x80000000 + 0xc00000
6954		\\ cbz wzr, 1f
6955		\\8:
6956		\\ mov w16, w0
6957		\\0:
6958		\\ ldaxr w0, [x1]
6959		\\ stlxr w17, w16, [x1]
6960		\\ cbnz w17, 0b
6961		\\1:
6962		: [ret] "={w0}" (-> u32),
6963		: [val] "{w0}" (val),
6964		[ptr] "{x1}" (ptr),
6965		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6966		: "w15", "w16", "w17", "memory"
6967		);
6968		}
6969		const __aarch64_swp4_acq_rel = if (is_darwin)
6970		if (has_lse)
6971		swp4AcqRelDarwinLse
6972		else
6973		swp4AcqRelDarwinNolse
6974		else if (has_lse)
6975		swp4AcqRelNondarwinLse
6976		else
6977		swp4AcqRelNondarwinNolse;
6978		fn ldadd4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
6979		@setRuntimeSafety(false);
6980		__init_aarch64_have_lse_atomics();
6981
6982		return asm volatile (
6983		\\ cbz w16, 8f
6984		\\ ldaddal w0, w0, [x1]
6985		\\ cbz wzr, 1f
6986		\\8:
6987		\\ mov w16, w0
6988		\\0:
6989		\\ ldaxr w0, [x1]
6990		\\ add w17, w0, w16
6991		\\ stlxr w15, w17, [x1]
6992		\\ cbnz w15, 0b
6993		\\1:
6994		: [ret] "={w0}" (-> u32),
6995		: [val] "{w0}" (val),
6996		[ptr] "{x1}" (ptr),
6997		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
6998		: "w15", "w16", "w17", "memory"
6999		);
7000		}
7001		fn ldadd4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7002		@setRuntimeSafety(false);
7003		__init_aarch64_have_lse_atomics();
7004
7005		return asm volatile (
7006		\\ cbz w16, 8f
7007		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
7008		\\ cbz wzr, 1f
7009		\\8:
7010		\\ mov w16, w0
7011		\\0:
7012		\\ ldaxr w0, [x1]
7013		\\ add w17, w0, w16
7014		\\ stlxr w15, w17, [x1]
7015		\\ cbnz w15, 0b
7016		\\1:
7017		: [ret] "={w0}" (-> u32),
7018		: [val] "{w0}" (val),
7019		[ptr] "{x1}" (ptr),
7020		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7021		: "w15", "w16", "w17", "memory"
7022		);
7023		}
7024		fn ldadd4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7025		@setRuntimeSafety(false);
7026		__init_aarch64_have_lse_atomics();
7027
7028		return asm volatile (
7029		\\ cbz w16, 8f
7030		\\ ldaddal w0, w0, [x1]
7031		\\ cbz wzr, 1f
7032		\\8:
7033		\\ mov w16, w0
7034		\\0:
7035		\\ ldaxr w0, [x1]
7036		\\ add w17, w0, w16
7037		\\ stlxr w15, w17, [x1]
7038		\\ cbnz w15, 0b
7039		\\1:
7040		: [ret] "={w0}" (-> u32),
7041		: [val] "{w0}" (val),
7042		[ptr] "{x1}" (ptr),
7043		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7044		: "w15", "w16", "w17", "memory"
7045		);
7046		}
7047		fn ldadd4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7048		@setRuntimeSafety(false);
7049		__init_aarch64_have_lse_atomics();
7050
7051		return asm volatile (
7052		\\ cbz w16, 8f
7053		\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
7054		\\ cbz wzr, 1f
7055		\\8:
7056		\\ mov w16, w0
7057		\\0:
7058		\\ ldaxr w0, [x1]
7059		\\ add w17, w0, w16
7060		\\ stlxr w15, w17, [x1]
7061		\\ cbnz w15, 0b
7062		\\1:
7063		: [ret] "={w0}" (-> u32),
7064		: [val] "{w0}" (val),
7065		[ptr] "{x1}" (ptr),
7066		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7067		: "w15", "w16", "w17", "memory"
7068		);
7069		}
7070		const __aarch64_ldadd4_acq_rel = if (is_darwin)
7071		if (has_lse)
7072		ldadd4AcqRelDarwinLse
7073		else
7074		ldadd4AcqRelDarwinNolse
7075		else if (has_lse)
7076		ldadd4AcqRelNondarwinLse
7077		else
7078		ldadd4AcqRelNondarwinNolse;
7079		fn ldclr4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7080		@setRuntimeSafety(false);
7081		__init_aarch64_have_lse_atomics();
7082
7083		return asm volatile (
7084		\\ cbz w16, 8f
7085		\\ ldclral w0, w0, [x1]
7086		\\ cbz wzr, 1f
7087		\\8:
7088		\\ mov w16, w0
7089		\\0:
7090		\\ ldaxr w0, [x1]
7091		\\ bic w17, w0, w16
7092		\\ stlxr w15, w17, [x1]
7093		\\ cbnz w15, 0b
7094		\\1:
7095		: [ret] "={w0}" (-> u32),
7096		: [val] "{w0}" (val),
7097		[ptr] "{x1}" (ptr),
7098		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7099		: "w15", "w16", "w17", "memory"
7100		);
7101		}
7102		fn ldclr4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7103		@setRuntimeSafety(false);
7104		__init_aarch64_have_lse_atomics();
7105
7106		return asm volatile (
7107		\\ cbz w16, 8f
7108		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
7109		\\ cbz wzr, 1f
7110		\\8:
7111		\\ mov w16, w0
7112		\\0:
7113		\\ ldaxr w0, [x1]
7114		\\ bic w17, w0, w16
7115		\\ stlxr w15, w17, [x1]
7116		\\ cbnz w15, 0b
7117		\\1:
7118		: [ret] "={w0}" (-> u32),
7119		: [val] "{w0}" (val),
7120		[ptr] "{x1}" (ptr),
7121		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7122		: "w15", "w16", "w17", "memory"
7123		);
7124		}
7125		fn ldclr4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7126		@setRuntimeSafety(false);
7127		__init_aarch64_have_lse_atomics();
7128
7129		return asm volatile (
7130		\\ cbz w16, 8f
7131		\\ ldclral w0, w0, [x1]
7132		\\ cbz wzr, 1f
7133		\\8:
7134		\\ mov w16, w0
7135		\\0:
7136		\\ ldaxr w0, [x1]
7137		\\ bic w17, w0, w16
7138		\\ stlxr w15, w17, [x1]
7139		\\ cbnz w15, 0b
7140		\\1:
7141		: [ret] "={w0}" (-> u32),
7142		: [val] "{w0}" (val),
7143		[ptr] "{x1}" (ptr),
7144		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7145		: "w15", "w16", "w17", "memory"
7146		);
7147		}
7148		fn ldclr4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7149		@setRuntimeSafety(false);
7150		__init_aarch64_have_lse_atomics();
7151
7152		return asm volatile (
7153		\\ cbz w16, 8f
7154		\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
7155		\\ cbz wzr, 1f
7156		\\8:
7157		\\ mov w16, w0
7158		\\0:
7159		\\ ldaxr w0, [x1]
7160		\\ bic w17, w0, w16
7161		\\ stlxr w15, w17, [x1]
7162		\\ cbnz w15, 0b
7163		\\1:
7164		: [ret] "={w0}" (-> u32),
7165		: [val] "{w0}" (val),
7166		[ptr] "{x1}" (ptr),
7167		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7168		: "w15", "w16", "w17", "memory"
7169		);
7170		}
7171		const __aarch64_ldclr4_acq_rel = if (is_darwin)
7172		if (has_lse)
7173		ldclr4AcqRelDarwinLse
7174		else
7175		ldclr4AcqRelDarwinNolse
7176		else if (has_lse)
7177		ldclr4AcqRelNondarwinLse
7178		else
7179		ldclr4AcqRelNondarwinNolse;
7180		fn ldeor4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7181		@setRuntimeSafety(false);
7182		__init_aarch64_have_lse_atomics();
7183
7184		return asm volatile (
7185		\\ cbz w16, 8f
7186		\\ ldeoral w0, w0, [x1]
7187		\\ cbz wzr, 1f
7188		\\8:
7189		\\ mov w16, w0
7190		\\0:
7191		\\ ldaxr w0, [x1]
7192		\\ eor w17, w0, w16
7193		\\ stlxr w15, w17, [x1]
7194		\\ cbnz w15, 0b
7195		\\1:
7196		: [ret] "={w0}" (-> u32),
7197		: [val] "{w0}" (val),
7198		[ptr] "{x1}" (ptr),
7199		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7200		: "w15", "w16", "w17", "memory"
7201		);
7202		}
7203		fn ldeor4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7204		@setRuntimeSafety(false);
7205		__init_aarch64_have_lse_atomics();
7206
7207		return asm volatile (
7208		\\ cbz w16, 8f
7209		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
7210		\\ cbz wzr, 1f
7211		\\8:
7212		\\ mov w16, w0
7213		\\0:
7214		\\ ldaxr w0, [x1]
7215		\\ eor w17, w0, w16
7216		\\ stlxr w15, w17, [x1]
7217		\\ cbnz w15, 0b
7218		\\1:
7219		: [ret] "={w0}" (-> u32),
7220		: [val] "{w0}" (val),
7221		[ptr] "{x1}" (ptr),
7222		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7223		: "w15", "w16", "w17", "memory"
7224		);
7225		}
7226		fn ldeor4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7227		@setRuntimeSafety(false);
7228		__init_aarch64_have_lse_atomics();
7229
7230		return asm volatile (
7231		\\ cbz w16, 8f
7232		\\ ldeoral w0, w0, [x1]
7233		\\ cbz wzr, 1f
7234		\\8:
7235		\\ mov w16, w0
7236		\\0:
7237		\\ ldaxr w0, [x1]
7238		\\ eor w17, w0, w16
7239		\\ stlxr w15, w17, [x1]
7240		\\ cbnz w15, 0b
7241		\\1:
7242		: [ret] "={w0}" (-> u32),
7243		: [val] "{w0}" (val),
7244		[ptr] "{x1}" (ptr),
7245		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7246		: "w15", "w16", "w17", "memory"
7247		);
7248		}
7249		fn ldeor4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7250		@setRuntimeSafety(false);
7251		__init_aarch64_have_lse_atomics();
7252
7253		return asm volatile (
7254		\\ cbz w16, 8f
7255		\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
7256		\\ cbz wzr, 1f
7257		\\8:
7258		\\ mov w16, w0
7259		\\0:
7260		\\ ldaxr w0, [x1]
7261		\\ eor w17, w0, w16
7262		\\ stlxr w15, w17, [x1]
7263		\\ cbnz w15, 0b
7264		\\1:
7265		: [ret] "={w0}" (-> u32),
7266		: [val] "{w0}" (val),
7267		[ptr] "{x1}" (ptr),
7268		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7269		: "w15", "w16", "w17", "memory"
7270		);
7271		}
7272		const __aarch64_ldeor4_acq_rel = if (is_darwin)
7273		if (has_lse)
7274		ldeor4AcqRelDarwinLse
7275		else
7276		ldeor4AcqRelDarwinNolse
7277		else if (has_lse)
7278		ldeor4AcqRelNondarwinLse
7279		else
7280		ldeor4AcqRelNondarwinNolse;
7281		fn ldset4AcqRelDarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7282		@setRuntimeSafety(false);
7283		__init_aarch64_have_lse_atomics();
7284
7285		return asm volatile (
7286		\\ cbz w16, 8f
7287		\\ ldsetal w0, w0, [x1]
7288		\\ cbz wzr, 1f
7289		\\8:
7290		\\ mov w16, w0
7291		\\0:
7292		\\ ldaxr w0, [x1]
7293		\\ orr w17, w0, w16
7294		\\ stlxr w15, w17, [x1]
7295		\\ cbnz w15, 0b
7296		\\1:
7297		: [ret] "={w0}" (-> u32),
7298		: [val] "{w0}" (val),
7299		[ptr] "{x1}" (ptr),
7300		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7301		: "w15", "w16", "w17", "memory"
7302		);
7303		}
7304		fn ldset4AcqRelDarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7305		@setRuntimeSafety(false);
7306		__init_aarch64_have_lse_atomics();
7307
7308		return asm volatile (
7309		\\ cbz w16, 8f
7310		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
7311		\\ cbz wzr, 1f
7312		\\8:
7313		\\ mov w16, w0
7314		\\0:
7315		\\ ldaxr w0, [x1]
7316		\\ orr w17, w0, w16
7317		\\ stlxr w15, w17, [x1]
7318		\\ cbnz w15, 0b
7319		\\1:
7320		: [ret] "={w0}" (-> u32),
7321		: [val] "{w0}" (val),
7322		[ptr] "{x1}" (ptr),
7323		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7324		: "w15", "w16", "w17", "memory"
7325		);
7326		}
7327		fn ldset4AcqRelNondarwinLse(val: u32, ptr: *u32) callconv(.C) u32 {
7328		@setRuntimeSafety(false);
7329		__init_aarch64_have_lse_atomics();
7330
7331		return asm volatile (
7332		\\ cbz w16, 8f
7333		\\ ldsetal w0, w0, [x1]
7334		\\ cbz wzr, 1f
7335		\\8:
7336		\\ mov w16, w0
7337		\\0:
7338		\\ ldaxr w0, [x1]
7339		\\ orr w17, w0, w16
7340		\\ stlxr w15, w17, [x1]
7341		\\ cbnz w15, 0b
7342		\\1:
7343		: [ret] "={w0}" (-> u32),
7344		: [val] "{w0}" (val),
7345		[ptr] "{x1}" (ptr),
7346		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7347		: "w15", "w16", "w17", "memory"
7348		);
7349		}
7350		fn ldset4AcqRelNondarwinNolse(val: u32, ptr: *u32) callconv(.C) u32 {
7351		@setRuntimeSafety(false);
7352		__init_aarch64_have_lse_atomics();
7353
7354		return asm volatile (
7355		\\ cbz w16, 8f
7356		\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
7357		\\ cbz wzr, 1f
7358		\\8:
7359		\\ mov w16, w0
7360		\\0:
7361		\\ ldaxr w0, [x1]
7362		\\ orr w17, w0, w16
7363		\\ stlxr w15, w17, [x1]
7364		\\ cbnz w15, 0b
7365		\\1:
7366		: [ret] "={w0}" (-> u32),
7367		: [val] "{w0}" (val),
7368		[ptr] "{x1}" (ptr),
7369		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7370		: "w15", "w16", "w17", "memory"
7371		);
7372		}
7373		const __aarch64_ldset4_acq_rel = if (is_darwin)
7374		if (has_lse)
7375		ldset4AcqRelDarwinLse
7376		else
7377		ldset4AcqRelDarwinNolse
7378		else if (has_lse)
7379		ldset4AcqRelNondarwinLse
7380		else
7381		ldset4AcqRelNondarwinNolse;
7382		fn cas8RelaxDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7383		@setRuntimeSafety(false);
7384		__init_aarch64_have_lse_atomics();
7385
7386		return asm volatile (
7387		\\ cbz w16, 8f
7388		\\ cas x0, x1, [x2]
7389		\\
7390		\\ cbz wzr, 1f
7391		\\8:
7392		\\ mov x16, x0
7393		\\0:
7394		\\ ldxr x0, [x2]
7395		\\ cmp x0, x16
7396		\\ bne 1f
7397		\\ stxr w17, x1, [x2]
7398		\\ cbnz w17, 0b
7399		\\1:
7400		: [ret] "={x0}" (-> u64),
7401		: [expected] "{x0}" (expected),
7402		[desired] "{x1}" (desired),
7403		[ptr] "{x2}" (ptr),
7404		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7405		: "w15", "w16", "w17", "memory"
7406		);
7407		}
7408		fn cas8RelaxDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7409		@setRuntimeSafety(false);
7410		__init_aarch64_have_lse_atomics();
7411
7412		return asm volatile (
7413		\\ cbz w16, 8f
7414		\\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
7415		\\
7416		\\ cbz wzr, 1f
7417		\\8:
7418		\\ mov x16, x0
7419		\\0:
7420		\\ ldxr x0, [x2]
7421		\\ cmp x0, x16
7422		\\ bne 1f
7423		\\ stxr w17, x1, [x2]
7424		\\ cbnz w17, 0b
7425		\\1:
7426		: [ret] "={x0}" (-> u64),
7427		: [expected] "{x0}" (expected),
7428		[desired] "{x1}" (desired),
7429		[ptr] "{x2}" (ptr),
7430		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7431		: "w15", "w16", "w17", "memory"
7432		);
7433		}
7434		fn cas8RelaxNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7435		@setRuntimeSafety(false);
7436		__init_aarch64_have_lse_atomics();
7437
7438		return asm volatile (
7439		\\ cbz w16, 8f
7440		\\ cas x0, x1, [x2]
7441		\\
7442		\\ cbz wzr, 1f
7443		\\8:
7444		\\ mov x16, x0
7445		\\0:
7446		\\ ldxr x0, [x2]
7447		\\ cmp x0, x16
7448		\\ bne 1f
7449		\\ stxr w17, x1, [x2]
7450		\\ cbnz w17, 0b
7451		\\1:
7452		: [ret] "={x0}" (-> u64),
7453		: [expected] "{x0}" (expected),
7454		[desired] "{x1}" (desired),
7455		[ptr] "{x2}" (ptr),
7456		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7457		: "w15", "w16", "w17", "memory"
7458		);
7459		}
7460		fn cas8RelaxNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7461		@setRuntimeSafety(false);
7462		__init_aarch64_have_lse_atomics();
7463
7464		return asm volatile (
7465		\\ cbz w16, 8f
7466		\\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
7467		\\
7468		\\ cbz wzr, 1f
7469		\\8:
7470		\\ mov x16, x0
7471		\\0:
7472		\\ ldxr x0, [x2]
7473		\\ cmp x0, x16
7474		\\ bne 1f
7475		\\ stxr w17, x1, [x2]
7476		\\ cbnz w17, 0b
7477		\\1:
7478		: [ret] "={x0}" (-> u64),
7479		: [expected] "{x0}" (expected),
7480		[desired] "{x1}" (desired),
7481		[ptr] "{x2}" (ptr),
7482		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7483		: "w15", "w16", "w17", "memory"
7484		);
7485		}
7486		const __aarch64_cas8_relax = if (is_darwin)
7487		if (has_lse)
7488		cas8RelaxDarwinLse
7489		else
7490		cas8RelaxDarwinNolse
7491		else if (has_lse)
7492		cas8RelaxNondarwinLse
7493		else
7494		cas8RelaxNondarwinNolse;
7495		fn swp8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7496		@setRuntimeSafety(false);
7497		__init_aarch64_have_lse_atomics();
7498
7499		return asm volatile (
7500		\\ cbz w16, 8f
7501		\\ swp x0, x0, [x1]
7502		\\ cbz wzr, 1f
7503		\\8:
7504		\\ mov x16, x0
7505		\\0:
7506		\\ ldxr x0, [x1]
7507		\\ stxr w17, x16, [x1]
7508		\\ cbnz w17, 0b
7509		\\1:
7510		: [ret] "={x0}" (-> u64),
7511		: [val] "{x0}" (val),
7512		[ptr] "{x1}" (ptr),
7513		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7514		: "w15", "w16", "w17", "memory"
7515		);
7516		}
7517		fn swp8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7518		@setRuntimeSafety(false);
7519		__init_aarch64_have_lse_atomics();
7520
7521		return asm volatile (
7522		\\ cbz w16, 8f
7523		\\ .inst 0x38208020 + 0xc0000000 + 0x000000
7524		\\ cbz wzr, 1f
7525		\\8:
7526		\\ mov x16, x0
7527		\\0:
7528		\\ ldxr x0, [x1]
7529		\\ stxr w17, x16, [x1]
7530		\\ cbnz w17, 0b
7531		\\1:
7532		: [ret] "={x0}" (-> u64),
7533		: [val] "{x0}" (val),
7534		[ptr] "{x1}" (ptr),
7535		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7536		: "w15", "w16", "w17", "memory"
7537		);
7538		}
7539		fn swp8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7540		@setRuntimeSafety(false);
7541		__init_aarch64_have_lse_atomics();
7542
7543		return asm volatile (
7544		\\ cbz w16, 8f
7545		\\ swp x0, x0, [x1]
7546		\\ cbz wzr, 1f
7547		\\8:
7548		\\ mov x16, x0
7549		\\0:
7550		\\ ldxr x0, [x1]
7551		\\ stxr w17, x16, [x1]
7552		\\ cbnz w17, 0b
7553		\\1:
7554		: [ret] "={x0}" (-> u64),
7555		: [val] "{x0}" (val),
7556		[ptr] "{x1}" (ptr),
7557		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7558		: "w15", "w16", "w17", "memory"
7559		);
7560		}
7561		fn swp8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7562		@setRuntimeSafety(false);
7563		__init_aarch64_have_lse_atomics();
7564
7565		return asm volatile (
7566		\\ cbz w16, 8f
7567		\\ .inst 0x38208020 + 0xc0000000 + 0x000000
7568		\\ cbz wzr, 1f
7569		\\8:
7570		\\ mov x16, x0
7571		\\0:
7572		\\ ldxr x0, [x1]
7573		\\ stxr w17, x16, [x1]
7574		\\ cbnz w17, 0b
7575		\\1:
7576		: [ret] "={x0}" (-> u64),
7577		: [val] "{x0}" (val),
7578		[ptr] "{x1}" (ptr),
7579		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7580		: "w15", "w16", "w17", "memory"
7581		);
7582		}
7583		const __aarch64_swp8_relax = if (is_darwin)
7584		if (has_lse)
7585		swp8RelaxDarwinLse
7586		else
7587		swp8RelaxDarwinNolse
7588		else if (has_lse)
7589		swp8RelaxNondarwinLse
7590		else
7591		swp8RelaxNondarwinNolse;
7592		fn ldadd8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7593		@setRuntimeSafety(false);
7594		__init_aarch64_have_lse_atomics();
7595
7596		return asm volatile (
7597		\\ cbz w16, 8f
7598		\\ ldadd x0, x0, [x1]
7599		\\ cbz wzr, 1f
7600		\\8:
7601		\\ mov x16, x0
7602		\\0:
7603		\\ ldxr x0, [x1]
7604		\\ add x17, x0, x16
7605		\\ stxr w15, x17, [x1]
7606		\\ cbnz w15, 0b
7607		\\1:
7608		: [ret] "={x0}" (-> u64),
7609		: [val] "{x0}" (val),
7610		[ptr] "{x1}" (ptr),
7611		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7612		: "w15", "w16", "w17", "memory"
7613		);
7614		}
7615		fn ldadd8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7616		@setRuntimeSafety(false);
7617		__init_aarch64_have_lse_atomics();
7618
7619		return asm volatile (
7620		\\ cbz w16, 8f
7621		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
7622		\\ cbz wzr, 1f
7623		\\8:
7624		\\ mov x16, x0
7625		\\0:
7626		\\ ldxr x0, [x1]
7627		\\ add x17, x0, x16
7628		\\ stxr w15, x17, [x1]
7629		\\ cbnz w15, 0b
7630		\\1:
7631		: [ret] "={x0}" (-> u64),
7632		: [val] "{x0}" (val),
7633		[ptr] "{x1}" (ptr),
7634		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7635		: "w15", "w16", "w17", "memory"
7636		);
7637		}
7638		fn ldadd8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7639		@setRuntimeSafety(false);
7640		__init_aarch64_have_lse_atomics();
7641
7642		return asm volatile (
7643		\\ cbz w16, 8f
7644		\\ ldadd x0, x0, [x1]
7645		\\ cbz wzr, 1f
7646		\\8:
7647		\\ mov x16, x0
7648		\\0:
7649		\\ ldxr x0, [x1]
7650		\\ add x17, x0, x16
7651		\\ stxr w15, x17, [x1]
7652		\\ cbnz w15, 0b
7653		\\1:
7654		: [ret] "={x0}" (-> u64),
7655		: [val] "{x0}" (val),
7656		[ptr] "{x1}" (ptr),
7657		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7658		: "w15", "w16", "w17", "memory"
7659		);
7660		}
7661		fn ldadd8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7662		@setRuntimeSafety(false);
7663		__init_aarch64_have_lse_atomics();
7664
7665		return asm volatile (
7666		\\ cbz w16, 8f
7667		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
7668		\\ cbz wzr, 1f
7669		\\8:
7670		\\ mov x16, x0
7671		\\0:
7672		\\ ldxr x0, [x1]
7673		\\ add x17, x0, x16
7674		\\ stxr w15, x17, [x1]
7675		\\ cbnz w15, 0b
7676		\\1:
7677		: [ret] "={x0}" (-> u64),
7678		: [val] "{x0}" (val),
7679		[ptr] "{x1}" (ptr),
7680		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7681		: "w15", "w16", "w17", "memory"
7682		);
7683		}
7684		const __aarch64_ldadd8_relax = if (is_darwin)
7685		if (has_lse)
7686		ldadd8RelaxDarwinLse
7687		else
7688		ldadd8RelaxDarwinNolse
7689		else if (has_lse)
7690		ldadd8RelaxNondarwinLse
7691		else
7692		ldadd8RelaxNondarwinNolse;
7693		fn ldclr8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7694		@setRuntimeSafety(false);
7695		__init_aarch64_have_lse_atomics();
7696
7697		return asm volatile (
7698		\\ cbz w16, 8f
7699		\\ ldclr x0, x0, [x1]
7700		\\ cbz wzr, 1f
7701		\\8:
7702		\\ mov x16, x0
7703		\\0:
7704		\\ ldxr x0, [x1]
7705		\\ bic x17, x0, x16
7706		\\ stxr w15, x17, [x1]
7707		\\ cbnz w15, 0b
7708		\\1:
7709		: [ret] "={x0}" (-> u64),
7710		: [val] "{x0}" (val),
7711		[ptr] "{x1}" (ptr),
7712		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7713		: "w15", "w16", "w17", "memory"
7714		);
7715		}
7716		fn ldclr8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7717		@setRuntimeSafety(false);
7718		__init_aarch64_have_lse_atomics();
7719
7720		return asm volatile (
7721		\\ cbz w16, 8f
7722		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
7723		\\ cbz wzr, 1f
7724		\\8:
7725		\\ mov x16, x0
7726		\\0:
7727		\\ ldxr x0, [x1]
7728		\\ bic x17, x0, x16
7729		\\ stxr w15, x17, [x1]
7730		\\ cbnz w15, 0b
7731		\\1:
7732		: [ret] "={x0}" (-> u64),
7733		: [val] "{x0}" (val),
7734		[ptr] "{x1}" (ptr),
7735		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7736		: "w15", "w16", "w17", "memory"
7737		);
7738		}
7739		fn ldclr8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7740		@setRuntimeSafety(false);
7741		__init_aarch64_have_lse_atomics();
7742
7743		return asm volatile (
7744		\\ cbz w16, 8f
7745		\\ ldclr x0, x0, [x1]
7746		\\ cbz wzr, 1f
7747		\\8:
7748		\\ mov x16, x0
7749		\\0:
7750		\\ ldxr x0, [x1]
7751		\\ bic x17, x0, x16
7752		\\ stxr w15, x17, [x1]
7753		\\ cbnz w15, 0b
7754		\\1:
7755		: [ret] "={x0}" (-> u64),
7756		: [val] "{x0}" (val),
7757		[ptr] "{x1}" (ptr),
7758		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7759		: "w15", "w16", "w17", "memory"
7760		);
7761		}
7762		fn ldclr8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7763		@setRuntimeSafety(false);
7764		__init_aarch64_have_lse_atomics();
7765
7766		return asm volatile (
7767		\\ cbz w16, 8f
7768		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
7769		\\ cbz wzr, 1f
7770		\\8:
7771		\\ mov x16, x0
7772		\\0:
7773		\\ ldxr x0, [x1]
7774		\\ bic x17, x0, x16
7775		\\ stxr w15, x17, [x1]
7776		\\ cbnz w15, 0b
7777		\\1:
7778		: [ret] "={x0}" (-> u64),
7779		: [val] "{x0}" (val),
7780		[ptr] "{x1}" (ptr),
7781		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7782		: "w15", "w16", "w17", "memory"
7783		);
7784		}
7785		const __aarch64_ldclr8_relax = if (is_darwin)
7786		if (has_lse)
7787		ldclr8RelaxDarwinLse
7788		else
7789		ldclr8RelaxDarwinNolse
7790		else if (has_lse)
7791		ldclr8RelaxNondarwinLse
7792		else
7793		ldclr8RelaxNondarwinNolse;
7794		fn ldeor8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7795		@setRuntimeSafety(false);
7796		__init_aarch64_have_lse_atomics();
7797
7798		return asm volatile (
7799		\\ cbz w16, 8f
7800		\\ ldeor x0, x0, [x1]
7801		\\ cbz wzr, 1f
7802		\\8:
7803		\\ mov x16, x0
7804		\\0:
7805		\\ ldxr x0, [x1]
7806		\\ eor x17, x0, x16
7807		\\ stxr w15, x17, [x1]
7808		\\ cbnz w15, 0b
7809		\\1:
7810		: [ret] "={x0}" (-> u64),
7811		: [val] "{x0}" (val),
7812		[ptr] "{x1}" (ptr),
7813		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7814		: "w15", "w16", "w17", "memory"
7815		);
7816		}
7817		fn ldeor8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7818		@setRuntimeSafety(false);
7819		__init_aarch64_have_lse_atomics();
7820
7821		return asm volatile (
7822		\\ cbz w16, 8f
7823		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
7824		\\ cbz wzr, 1f
7825		\\8:
7826		\\ mov x16, x0
7827		\\0:
7828		\\ ldxr x0, [x1]
7829		\\ eor x17, x0, x16
7830		\\ stxr w15, x17, [x1]
7831		\\ cbnz w15, 0b
7832		\\1:
7833		: [ret] "={x0}" (-> u64),
7834		: [val] "{x0}" (val),
7835		[ptr] "{x1}" (ptr),
7836		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7837		: "w15", "w16", "w17", "memory"
7838		);
7839		}
7840		fn ldeor8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7841		@setRuntimeSafety(false);
7842		__init_aarch64_have_lse_atomics();
7843
7844		return asm volatile (
7845		\\ cbz w16, 8f
7846		\\ ldeor x0, x0, [x1]
7847		\\ cbz wzr, 1f
7848		\\8:
7849		\\ mov x16, x0
7850		\\0:
7851		\\ ldxr x0, [x1]
7852		\\ eor x17, x0, x16
7853		\\ stxr w15, x17, [x1]
7854		\\ cbnz w15, 0b
7855		\\1:
7856		: [ret] "={x0}" (-> u64),
7857		: [val] "{x0}" (val),
7858		[ptr] "{x1}" (ptr),
7859		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7860		: "w15", "w16", "w17", "memory"
7861		);
7862		}
7863		fn ldeor8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7864		@setRuntimeSafety(false);
7865		__init_aarch64_have_lse_atomics();
7866
7867		return asm volatile (
7868		\\ cbz w16, 8f
7869		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
7870		\\ cbz wzr, 1f
7871		\\8:
7872		\\ mov x16, x0
7873		\\0:
7874		\\ ldxr x0, [x1]
7875		\\ eor x17, x0, x16
7876		\\ stxr w15, x17, [x1]
7877		\\ cbnz w15, 0b
7878		\\1:
7879		: [ret] "={x0}" (-> u64),
7880		: [val] "{x0}" (val),
7881		[ptr] "{x1}" (ptr),
7882		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7883		: "w15", "w16", "w17", "memory"
7884		);
7885		}
7886		const __aarch64_ldeor8_relax = if (is_darwin)
7887		if (has_lse)
7888		ldeor8RelaxDarwinLse
7889		else
7890		ldeor8RelaxDarwinNolse
7891		else if (has_lse)
7892		ldeor8RelaxNondarwinLse
7893		else
7894		ldeor8RelaxNondarwinNolse;
7895		fn ldset8RelaxDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7896		@setRuntimeSafety(false);
7897		__init_aarch64_have_lse_atomics();
7898
7899		return asm volatile (
7900		\\ cbz w16, 8f
7901		\\ ldset x0, x0, [x1]
7902		\\ cbz wzr, 1f
7903		\\8:
7904		\\ mov x16, x0
7905		\\0:
7906		\\ ldxr x0, [x1]
7907		\\ orr x17, x0, x16
7908		\\ stxr w15, x17, [x1]
7909		\\ cbnz w15, 0b
7910		\\1:
7911		: [ret] "={x0}" (-> u64),
7912		: [val] "{x0}" (val),
7913		[ptr] "{x1}" (ptr),
7914		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7915		: "w15", "w16", "w17", "memory"
7916		);
7917		}
7918		fn ldset8RelaxDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7919		@setRuntimeSafety(false);
7920		__init_aarch64_have_lse_atomics();
7921
7922		return asm volatile (
7923		\\ cbz w16, 8f
7924		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
7925		\\ cbz wzr, 1f
7926		\\8:
7927		\\ mov x16, x0
7928		\\0:
7929		\\ ldxr x0, [x1]
7930		\\ orr x17, x0, x16
7931		\\ stxr w15, x17, [x1]
7932		\\ cbnz w15, 0b
7933		\\1:
7934		: [ret] "={x0}" (-> u64),
7935		: [val] "{x0}" (val),
7936		[ptr] "{x1}" (ptr),
7937		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7938		: "w15", "w16", "w17", "memory"
7939		);
7940		}
7941		fn ldset8RelaxNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
7942		@setRuntimeSafety(false);
7943		__init_aarch64_have_lse_atomics();
7944
7945		return asm volatile (
7946		\\ cbz w16, 8f
7947		\\ ldset x0, x0, [x1]
7948		\\ cbz wzr, 1f
7949		\\8:
7950		\\ mov x16, x0
7951		\\0:
7952		\\ ldxr x0, [x1]
7953		\\ orr x17, x0, x16
7954		\\ stxr w15, x17, [x1]
7955		\\ cbnz w15, 0b
7956		\\1:
7957		: [ret] "={x0}" (-> u64),
7958		: [val] "{x0}" (val),
7959		[ptr] "{x1}" (ptr),
7960		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7961		: "w15", "w16", "w17", "memory"
7962		);
7963		}
7964		fn ldset8RelaxNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
7965		@setRuntimeSafety(false);
7966		__init_aarch64_have_lse_atomics();
7967
7968		return asm volatile (
7969		\\ cbz w16, 8f
7970		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
7971		\\ cbz wzr, 1f
7972		\\8:
7973		\\ mov x16, x0
7974		\\0:
7975		\\ ldxr x0, [x1]
7976		\\ orr x17, x0, x16
7977		\\ stxr w15, x17, [x1]
7978		\\ cbnz w15, 0b
7979		\\1:
7980		: [ret] "={x0}" (-> u64),
7981		: [val] "{x0}" (val),
7982		[ptr] "{x1}" (ptr),
7983		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
7984		: "w15", "w16", "w17", "memory"
7985		);
7986		}
7987		const __aarch64_ldset8_relax = if (is_darwin)
7988		if (has_lse)
7989		ldset8RelaxDarwinLse
7990		else
7991		ldset8RelaxDarwinNolse
7992		else if (has_lse)
7993		ldset8RelaxNondarwinLse
7994		else
7995		ldset8RelaxNondarwinNolse;
7996		fn cas8AcqDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
7997		@setRuntimeSafety(false);
7998		__init_aarch64_have_lse_atomics();
7999
8000		return asm volatile (
8001		\\ cbz w16, 8f
8002		\\ casa x0, x1, [x2]
8003		\\
8004		\\ cbz wzr, 1f
8005		\\8:
8006		\\ mov x16, x0
8007		\\0:
8008		\\ ldaxr x0, [x2]
8009		\\ cmp x0, x16
8010		\\ bne 1f
8011		\\ stxr w17, x1, [x2]
8012		\\ cbnz w17, 0b
8013		\\1:
8014		: [ret] "={x0}" (-> u64),
8015		: [expected] "{x0}" (expected),
8016		[desired] "{x1}" (desired),
8017		[ptr] "{x2}" (ptr),
8018		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8019		: "w15", "w16", "w17", "memory"
8020		);
8021		}
8022		fn cas8AcqDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8023		@setRuntimeSafety(false);
8024		__init_aarch64_have_lse_atomics();
8025
8026		return asm volatile (
8027		\\ cbz w16, 8f
8028		\\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
8029		\\
8030		\\ cbz wzr, 1f
8031		\\8:
8032		\\ mov x16, x0
8033		\\0:
8034		\\ ldaxr x0, [x2]
8035		\\ cmp x0, x16
8036		\\ bne 1f
8037		\\ stxr w17, x1, [x2]
8038		\\ cbnz w17, 0b
8039		\\1:
8040		: [ret] "={x0}" (-> u64),
8041		: [expected] "{x0}" (expected),
8042		[desired] "{x1}" (desired),
8043		[ptr] "{x2}" (ptr),
8044		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8045		: "w15", "w16", "w17", "memory"
8046		);
8047		}
8048		fn cas8AcqNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8049		@setRuntimeSafety(false);
8050		__init_aarch64_have_lse_atomics();
8051
8052		return asm volatile (
8053		\\ cbz w16, 8f
8054		\\ casa x0, x1, [x2]
8055		\\
8056		\\ cbz wzr, 1f
8057		\\8:
8058		\\ mov x16, x0
8059		\\0:
8060		\\ ldaxr x0, [x2]
8061		\\ cmp x0, x16
8062		\\ bne 1f
8063		\\ stxr w17, x1, [x2]
8064		\\ cbnz w17, 0b
8065		\\1:
8066		: [ret] "={x0}" (-> u64),
8067		: [expected] "{x0}" (expected),
8068		[desired] "{x1}" (desired),
8069		[ptr] "{x2}" (ptr),
8070		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8071		: "w15", "w16", "w17", "memory"
8072		);
8073		}
8074		fn cas8AcqNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8075		@setRuntimeSafety(false);
8076		__init_aarch64_have_lse_atomics();
8077
8078		return asm volatile (
8079		\\ cbz w16, 8f
8080		\\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
8081		\\
8082		\\ cbz wzr, 1f
8083		\\8:
8084		\\ mov x16, x0
8085		\\0:
8086		\\ ldaxr x0, [x2]
8087		\\ cmp x0, x16
8088		\\ bne 1f
8089		\\ stxr w17, x1, [x2]
8090		\\ cbnz w17, 0b
8091		\\1:
8092		: [ret] "={x0}" (-> u64),
8093		: [expected] "{x0}" (expected),
8094		[desired] "{x1}" (desired),
8095		[ptr] "{x2}" (ptr),
8096		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8097		: "w15", "w16", "w17", "memory"
8098		);
8099		}
8100		const __aarch64_cas8_acq = if (is_darwin)
8101		if (has_lse)
8102		cas8AcqDarwinLse
8103		else
8104		cas8AcqDarwinNolse
8105		else if (has_lse)
8106		cas8AcqNondarwinLse
8107		else
8108		cas8AcqNondarwinNolse;
8109		fn swp8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8110		@setRuntimeSafety(false);
8111		__init_aarch64_have_lse_atomics();
8112
8113		return asm volatile (
8114		\\ cbz w16, 8f
8115		\\ swpa x0, x0, [x1]
8116		\\ cbz wzr, 1f
8117		\\8:
8118		\\ mov x16, x0
8119		\\0:
8120		\\ ldaxr x0, [x1]
8121		\\ stxr w17, x16, [x1]
8122		\\ cbnz w17, 0b
8123		\\1:
8124		: [ret] "={x0}" (-> u64),
8125		: [val] "{x0}" (val),
8126		[ptr] "{x1}" (ptr),
8127		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8128		: "w15", "w16", "w17", "memory"
8129		);
8130		}
8131		fn swp8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8132		@setRuntimeSafety(false);
8133		__init_aarch64_have_lse_atomics();
8134
8135		return asm volatile (
8136		\\ cbz w16, 8f
8137		\\ .inst 0x38208020 + 0xc0000000 + 0x800000
8138		\\ cbz wzr, 1f
8139		\\8:
8140		\\ mov x16, x0
8141		\\0:
8142		\\ ldaxr x0, [x1]
8143		\\ stxr w17, x16, [x1]
8144		\\ cbnz w17, 0b
8145		\\1:
8146		: [ret] "={x0}" (-> u64),
8147		: [val] "{x0}" (val),
8148		[ptr] "{x1}" (ptr),
8149		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8150		: "w15", "w16", "w17", "memory"
8151		);
8152		}
8153		fn swp8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8154		@setRuntimeSafety(false);
8155		__init_aarch64_have_lse_atomics();
8156
8157		return asm volatile (
8158		\\ cbz w16, 8f
8159		\\ swpa x0, x0, [x1]
8160		\\ cbz wzr, 1f
8161		\\8:
8162		\\ mov x16, x0
8163		\\0:
8164		\\ ldaxr x0, [x1]
8165		\\ stxr w17, x16, [x1]
8166		\\ cbnz w17, 0b
8167		\\1:
8168		: [ret] "={x0}" (-> u64),
8169		: [val] "{x0}" (val),
8170		[ptr] "{x1}" (ptr),
8171		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8172		: "w15", "w16", "w17", "memory"
8173		);
8174		}
8175		fn swp8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8176		@setRuntimeSafety(false);
8177		__init_aarch64_have_lse_atomics();
8178
8179		return asm volatile (
8180		\\ cbz w16, 8f
8181		\\ .inst 0x38208020 + 0xc0000000 + 0x800000
8182		\\ cbz wzr, 1f
8183		\\8:
8184		\\ mov x16, x0
8185		\\0:
8186		\\ ldaxr x0, [x1]
8187		\\ stxr w17, x16, [x1]
8188		\\ cbnz w17, 0b
8189		\\1:
8190		: [ret] "={x0}" (-> u64),
8191		: [val] "{x0}" (val),
8192		[ptr] "{x1}" (ptr),
8193		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8194		: "w15", "w16", "w17", "memory"
8195		);
8196		}
8197		const __aarch64_swp8_acq = if (is_darwin)
8198		if (has_lse)
8199		swp8AcqDarwinLse
8200		else
8201		swp8AcqDarwinNolse
8202		else if (has_lse)
8203		swp8AcqNondarwinLse
8204		else
8205		swp8AcqNondarwinNolse;
8206		fn ldadd8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8207		@setRuntimeSafety(false);
8208		__init_aarch64_have_lse_atomics();
8209
8210		return asm volatile (
8211		\\ cbz w16, 8f
8212		\\ ldadda x0, x0, [x1]
8213		\\ cbz wzr, 1f
8214		\\8:
8215		\\ mov x16, x0
8216		\\0:
8217		\\ ldaxr x0, [x1]
8218		\\ add x17, x0, x16
8219		\\ stxr w15, x17, [x1]
8220		\\ cbnz w15, 0b
8221		\\1:
8222		: [ret] "={x0}" (-> u64),
8223		: [val] "{x0}" (val),
8224		[ptr] "{x1}" (ptr),
8225		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8226		: "w15", "w16", "w17", "memory"
8227		);
8228		}
8229		fn ldadd8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8230		@setRuntimeSafety(false);
8231		__init_aarch64_have_lse_atomics();
8232
8233		return asm volatile (
8234		\\ cbz w16, 8f
8235		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
8236		\\ cbz wzr, 1f
8237		\\8:
8238		\\ mov x16, x0
8239		\\0:
8240		\\ ldaxr x0, [x1]
8241		\\ add x17, x0, x16
8242		\\ stxr w15, x17, [x1]
8243		\\ cbnz w15, 0b
8244		\\1:
8245		: [ret] "={x0}" (-> u64),
8246		: [val] "{x0}" (val),
8247		[ptr] "{x1}" (ptr),
8248		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8249		: "w15", "w16", "w17", "memory"
8250		);
8251		}
8252		fn ldadd8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8253		@setRuntimeSafety(false);
8254		__init_aarch64_have_lse_atomics();
8255
8256		return asm volatile (
8257		\\ cbz w16, 8f
8258		\\ ldadda x0, x0, [x1]
8259		\\ cbz wzr, 1f
8260		\\8:
8261		\\ mov x16, x0
8262		\\0:
8263		\\ ldaxr x0, [x1]
8264		\\ add x17, x0, x16
8265		\\ stxr w15, x17, [x1]
8266		\\ cbnz w15, 0b
8267		\\1:
8268		: [ret] "={x0}" (-> u64),
8269		: [val] "{x0}" (val),
8270		[ptr] "{x1}" (ptr),
8271		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8272		: "w15", "w16", "w17", "memory"
8273		);
8274		}
8275		fn ldadd8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8276		@setRuntimeSafety(false);
8277		__init_aarch64_have_lse_atomics();
8278
8279		return asm volatile (
8280		\\ cbz w16, 8f
8281		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
8282		\\ cbz wzr, 1f
8283		\\8:
8284		\\ mov x16, x0
8285		\\0:
8286		\\ ldaxr x0, [x1]
8287		\\ add x17, x0, x16
8288		\\ stxr w15, x17, [x1]
8289		\\ cbnz w15, 0b
8290		\\1:
8291		: [ret] "={x0}" (-> u64),
8292		: [val] "{x0}" (val),
8293		[ptr] "{x1}" (ptr),
8294		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8295		: "w15", "w16", "w17", "memory"
8296		);
8297		}
8298		const __aarch64_ldadd8_acq = if (is_darwin)
8299		if (has_lse)
8300		ldadd8AcqDarwinLse
8301		else
8302		ldadd8AcqDarwinNolse
8303		else if (has_lse)
8304		ldadd8AcqNondarwinLse
8305		else
8306		ldadd8AcqNondarwinNolse;
8307		fn ldclr8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8308		@setRuntimeSafety(false);
8309		__init_aarch64_have_lse_atomics();
8310
8311		return asm volatile (
8312		\\ cbz w16, 8f
8313		\\ ldclra x0, x0, [x1]
8314		\\ cbz wzr, 1f
8315		\\8:
8316		\\ mov x16, x0
8317		\\0:
8318		\\ ldaxr x0, [x1]
8319		\\ bic x17, x0, x16
8320		\\ stxr w15, x17, [x1]
8321		\\ cbnz w15, 0b
8322		\\1:
8323		: [ret] "={x0}" (-> u64),
8324		: [val] "{x0}" (val),
8325		[ptr] "{x1}" (ptr),
8326		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8327		: "w15", "w16", "w17", "memory"
8328		);
8329		}
8330		fn ldclr8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8331		@setRuntimeSafety(false);
8332		__init_aarch64_have_lse_atomics();
8333
8334		return asm volatile (
8335		\\ cbz w16, 8f
8336		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
8337		\\ cbz wzr, 1f
8338		\\8:
8339		\\ mov x16, x0
8340		\\0:
8341		\\ ldaxr x0, [x1]
8342		\\ bic x17, x0, x16
8343		\\ stxr w15, x17, [x1]
8344		\\ cbnz w15, 0b
8345		\\1:
8346		: [ret] "={x0}" (-> u64),
8347		: [val] "{x0}" (val),
8348		[ptr] "{x1}" (ptr),
8349		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8350		: "w15", "w16", "w17", "memory"
8351		);
8352		}
8353		fn ldclr8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8354		@setRuntimeSafety(false);
8355		__init_aarch64_have_lse_atomics();
8356
8357		return asm volatile (
8358		\\ cbz w16, 8f
8359		\\ ldclra x0, x0, [x1]
8360		\\ cbz wzr, 1f
8361		\\8:
8362		\\ mov x16, x0
8363		\\0:
8364		\\ ldaxr x0, [x1]
8365		\\ bic x17, x0, x16
8366		\\ stxr w15, x17, [x1]
8367		\\ cbnz w15, 0b
8368		\\1:
8369		: [ret] "={x0}" (-> u64),
8370		: [val] "{x0}" (val),
8371		[ptr] "{x1}" (ptr),
8372		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8373		: "w15", "w16", "w17", "memory"
8374		);
8375		}
8376		fn ldclr8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8377		@setRuntimeSafety(false);
8378		__init_aarch64_have_lse_atomics();
8379
8380		return asm volatile (
8381		\\ cbz w16, 8f
8382		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
8383		\\ cbz wzr, 1f
8384		\\8:
8385		\\ mov x16, x0
8386		\\0:
8387		\\ ldaxr x0, [x1]
8388		\\ bic x17, x0, x16
8389		\\ stxr w15, x17, [x1]
8390		\\ cbnz w15, 0b
8391		\\1:
8392		: [ret] "={x0}" (-> u64),
8393		: [val] "{x0}" (val),
8394		[ptr] "{x1}" (ptr),
8395		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8396		: "w15", "w16", "w17", "memory"
8397		);
8398		}
8399		const __aarch64_ldclr8_acq = if (is_darwin)
8400		if (has_lse)
8401		ldclr8AcqDarwinLse
8402		else
8403		ldclr8AcqDarwinNolse
8404		else if (has_lse)
8405		ldclr8AcqNondarwinLse
8406		else
8407		ldclr8AcqNondarwinNolse;
8408		fn ldeor8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8409		@setRuntimeSafety(false);
8410		__init_aarch64_have_lse_atomics();
8411
8412		return asm volatile (
8413		\\ cbz w16, 8f
8414		\\ ldeora x0, x0, [x1]
8415		\\ cbz wzr, 1f
8416		\\8:
8417		\\ mov x16, x0
8418		\\0:
8419		\\ ldaxr x0, [x1]
8420		\\ eor x17, x0, x16
8421		\\ stxr w15, x17, [x1]
8422		\\ cbnz w15, 0b
8423		\\1:
8424		: [ret] "={x0}" (-> u64),
8425		: [val] "{x0}" (val),
8426		[ptr] "{x1}" (ptr),
8427		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8428		: "w15", "w16", "w17", "memory"
8429		);
8430		}
8431		fn ldeor8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8432		@setRuntimeSafety(false);
8433		__init_aarch64_have_lse_atomics();
8434
8435		return asm volatile (
8436		\\ cbz w16, 8f
8437		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
8438		\\ cbz wzr, 1f
8439		\\8:
8440		\\ mov x16, x0
8441		\\0:
8442		\\ ldaxr x0, [x1]
8443		\\ eor x17, x0, x16
8444		\\ stxr w15, x17, [x1]
8445		\\ cbnz w15, 0b
8446		\\1:
8447		: [ret] "={x0}" (-> u64),
8448		: [val] "{x0}" (val),
8449		[ptr] "{x1}" (ptr),
8450		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8451		: "w15", "w16", "w17", "memory"
8452		);
8453		}
8454		fn ldeor8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8455		@setRuntimeSafety(false);
8456		__init_aarch64_have_lse_atomics();
8457
8458		return asm volatile (
8459		\\ cbz w16, 8f
8460		\\ ldeora x0, x0, [x1]
8461		\\ cbz wzr, 1f
8462		\\8:
8463		\\ mov x16, x0
8464		\\0:
8465		\\ ldaxr x0, [x1]
8466		\\ eor x17, x0, x16
8467		\\ stxr w15, x17, [x1]
8468		\\ cbnz w15, 0b
8469		\\1:
8470		: [ret] "={x0}" (-> u64),
8471		: [val] "{x0}" (val),
8472		[ptr] "{x1}" (ptr),
8473		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8474		: "w15", "w16", "w17", "memory"
8475		);
8476		}
8477		fn ldeor8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8478		@setRuntimeSafety(false);
8479		__init_aarch64_have_lse_atomics();
8480
8481		return asm volatile (
8482		\\ cbz w16, 8f
8483		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
8484		\\ cbz wzr, 1f
8485		\\8:
8486		\\ mov x16, x0
8487		\\0:
8488		\\ ldaxr x0, [x1]
8489		\\ eor x17, x0, x16
8490		\\ stxr w15, x17, [x1]
8491		\\ cbnz w15, 0b
8492		\\1:
8493		: [ret] "={x0}" (-> u64),
8494		: [val] "{x0}" (val),
8495		[ptr] "{x1}" (ptr),
8496		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8497		: "w15", "w16", "w17", "memory"
8498		);
8499		}
8500		const __aarch64_ldeor8_acq = if (is_darwin)
8501		if (has_lse)
8502		ldeor8AcqDarwinLse
8503		else
8504		ldeor8AcqDarwinNolse
8505		else if (has_lse)
8506		ldeor8AcqNondarwinLse
8507		else
8508		ldeor8AcqNondarwinNolse;
8509		fn ldset8AcqDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8510		@setRuntimeSafety(false);
8511		__init_aarch64_have_lse_atomics();
8512
8513		return asm volatile (
8514		\\ cbz w16, 8f
8515		\\ ldseta x0, x0, [x1]
8516		\\ cbz wzr, 1f
8517		\\8:
8518		\\ mov x16, x0
8519		\\0:
8520		\\ ldaxr x0, [x1]
8521		\\ orr x17, x0, x16
8522		\\ stxr w15, x17, [x1]
8523		\\ cbnz w15, 0b
8524		\\1:
8525		: [ret] "={x0}" (-> u64),
8526		: [val] "{x0}" (val),
8527		[ptr] "{x1}" (ptr),
8528		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8529		: "w15", "w16", "w17", "memory"
8530		);
8531		}
8532		fn ldset8AcqDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8533		@setRuntimeSafety(false);
8534		__init_aarch64_have_lse_atomics();
8535
8536		return asm volatile (
8537		\\ cbz w16, 8f
8538		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
8539		\\ cbz wzr, 1f
8540		\\8:
8541		\\ mov x16, x0
8542		\\0:
8543		\\ ldaxr x0, [x1]
8544		\\ orr x17, x0, x16
8545		\\ stxr w15, x17, [x1]
8546		\\ cbnz w15, 0b
8547		\\1:
8548		: [ret] "={x0}" (-> u64),
8549		: [val] "{x0}" (val),
8550		[ptr] "{x1}" (ptr),
8551		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8552		: "w15", "w16", "w17", "memory"
8553		);
8554		}
8555		fn ldset8AcqNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8556		@setRuntimeSafety(false);
8557		__init_aarch64_have_lse_atomics();
8558
8559		return asm volatile (
8560		\\ cbz w16, 8f
8561		\\ ldseta x0, x0, [x1]
8562		\\ cbz wzr, 1f
8563		\\8:
8564		\\ mov x16, x0
8565		\\0:
8566		\\ ldaxr x0, [x1]
8567		\\ orr x17, x0, x16
8568		\\ stxr w15, x17, [x1]
8569		\\ cbnz w15, 0b
8570		\\1:
8571		: [ret] "={x0}" (-> u64),
8572		: [val] "{x0}" (val),
8573		[ptr] "{x1}" (ptr),
8574		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8575		: "w15", "w16", "w17", "memory"
8576		);
8577		}
8578		fn ldset8AcqNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8579		@setRuntimeSafety(false);
8580		__init_aarch64_have_lse_atomics();
8581
8582		return asm volatile (
8583		\\ cbz w16, 8f
8584		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
8585		\\ cbz wzr, 1f
8586		\\8:
8587		\\ mov x16, x0
8588		\\0:
8589		\\ ldaxr x0, [x1]
8590		\\ orr x17, x0, x16
8591		\\ stxr w15, x17, [x1]
8592		\\ cbnz w15, 0b
8593		\\1:
8594		: [ret] "={x0}" (-> u64),
8595		: [val] "{x0}" (val),
8596		[ptr] "{x1}" (ptr),
8597		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8598		: "w15", "w16", "w17", "memory"
8599		);
8600		}
8601		const __aarch64_ldset8_acq = if (is_darwin)
8602		if (has_lse)
8603		ldset8AcqDarwinLse
8604		else
8605		ldset8AcqDarwinNolse
8606		else if (has_lse)
8607		ldset8AcqNondarwinLse
8608		else
8609		ldset8AcqNondarwinNolse;
8610		fn cas8RelDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8611		@setRuntimeSafety(false);
8612		__init_aarch64_have_lse_atomics();
8613
8614		return asm volatile (
8615		\\ cbz w16, 8f
8616		\\ casl x0, x1, [x2]
8617		\\
8618		\\ cbz wzr, 1f
8619		\\8:
8620		\\ mov x16, x0
8621		\\0:
8622		\\ ldxr x0, [x2]
8623		\\ cmp x0, x16
8624		\\ bne 1f
8625		\\ stlxr w17, x1, [x2]
8626		\\ cbnz w17, 0b
8627		\\1:
8628		: [ret] "={x0}" (-> u64),
8629		: [expected] "{x0}" (expected),
8630		[desired] "{x1}" (desired),
8631		[ptr] "{x2}" (ptr),
8632		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8633		: "w15", "w16", "w17", "memory"
8634		);
8635		}
8636		fn cas8RelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8637		@setRuntimeSafety(false);
8638		__init_aarch64_have_lse_atomics();
8639
8640		return asm volatile (
8641		\\ cbz w16, 8f
8642		\\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
8643		\\
8644		\\ cbz wzr, 1f
8645		\\8:
8646		\\ mov x16, x0
8647		\\0:
8648		\\ ldxr x0, [x2]
8649		\\ cmp x0, x16
8650		\\ bne 1f
8651		\\ stlxr w17, x1, [x2]
8652		\\ cbnz w17, 0b
8653		\\1:
8654		: [ret] "={x0}" (-> u64),
8655		: [expected] "{x0}" (expected),
8656		[desired] "{x1}" (desired),
8657		[ptr] "{x2}" (ptr),
8658		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8659		: "w15", "w16", "w17", "memory"
8660		);
8661		}
8662		fn cas8RelNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8663		@setRuntimeSafety(false);
8664		__init_aarch64_have_lse_atomics();
8665
8666		return asm volatile (
8667		\\ cbz w16, 8f
8668		\\ casl x0, x1, [x2]
8669		\\
8670		\\ cbz wzr, 1f
8671		\\8:
8672		\\ mov x16, x0
8673		\\0:
8674		\\ ldxr x0, [x2]
8675		\\ cmp x0, x16
8676		\\ bne 1f
8677		\\ stlxr w17, x1, [x2]
8678		\\ cbnz w17, 0b
8679		\\1:
8680		: [ret] "={x0}" (-> u64),
8681		: [expected] "{x0}" (expected),
8682		[desired] "{x1}" (desired),
8683		[ptr] "{x2}" (ptr),
8684		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8685		: "w15", "w16", "w17", "memory"
8686		);
8687		}
8688		fn cas8RelNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
8689		@setRuntimeSafety(false);
8690		__init_aarch64_have_lse_atomics();
8691
8692		return asm volatile (
8693		\\ cbz w16, 8f
8694		\\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
8695		\\
8696		\\ cbz wzr, 1f
8697		\\8:
8698		\\ mov x16, x0
8699		\\0:
8700		\\ ldxr x0, [x2]
8701		\\ cmp x0, x16
8702		\\ bne 1f
8703		\\ stlxr w17, x1, [x2]
8704		\\ cbnz w17, 0b
8705		\\1:
8706		: [ret] "={x0}" (-> u64),
8707		: [expected] "{x0}" (expected),
8708		[desired] "{x1}" (desired),
8709		[ptr] "{x2}" (ptr),
8710		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8711		: "w15", "w16", "w17", "memory"
8712		);
8713		}
8714		const __aarch64_cas8_rel = if (is_darwin)
8715		if (has_lse)
8716		cas8RelDarwinLse
8717		else
8718		cas8RelDarwinNolse
8719		else if (has_lse)
8720		cas8RelNondarwinLse
8721		else
8722		cas8RelNondarwinNolse;
8723		fn swp8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8724		@setRuntimeSafety(false);
8725		__init_aarch64_have_lse_atomics();
8726
8727		return asm volatile (
8728		\\ cbz w16, 8f
8729		\\ swpl x0, x0, [x1]
8730		\\ cbz wzr, 1f
8731		\\8:
8732		\\ mov x16, x0
8733		\\0:
8734		\\ ldxr x0, [x1]
8735		\\ stlxr w17, x16, [x1]
8736		\\ cbnz w17, 0b
8737		\\1:
8738		: [ret] "={x0}" (-> u64),
8739		: [val] "{x0}" (val),
8740		[ptr] "{x1}" (ptr),
8741		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8742		: "w15", "w16", "w17", "memory"
8743		);
8744		}
8745		fn swp8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8746		@setRuntimeSafety(false);
8747		__init_aarch64_have_lse_atomics();
8748
8749		return asm volatile (
8750		\\ cbz w16, 8f
8751		\\ .inst 0x38208020 + 0xc0000000 + 0x400000
8752		\\ cbz wzr, 1f
8753		\\8:
8754		\\ mov x16, x0
8755		\\0:
8756		\\ ldxr x0, [x1]
8757		\\ stlxr w17, x16, [x1]
8758		\\ cbnz w17, 0b
8759		\\1:
8760		: [ret] "={x0}" (-> u64),
8761		: [val] "{x0}" (val),
8762		[ptr] "{x1}" (ptr),
8763		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8764		: "w15", "w16", "w17", "memory"
8765		);
8766		}
8767		fn swp8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8768		@setRuntimeSafety(false);
8769		__init_aarch64_have_lse_atomics();
8770
8771		return asm volatile (
8772		\\ cbz w16, 8f
8773		\\ swpl x0, x0, [x1]
8774		\\ cbz wzr, 1f
8775		\\8:
8776		\\ mov x16, x0
8777		\\0:
8778		\\ ldxr x0, [x1]
8779		\\ stlxr w17, x16, [x1]
8780		\\ cbnz w17, 0b
8781		\\1:
8782		: [ret] "={x0}" (-> u64),
8783		: [val] "{x0}" (val),
8784		[ptr] "{x1}" (ptr),
8785		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8786		: "w15", "w16", "w17", "memory"
8787		);
8788		}
8789		fn swp8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8790		@setRuntimeSafety(false);
8791		__init_aarch64_have_lse_atomics();
8792
8793		return asm volatile (
8794		\\ cbz w16, 8f
8795		\\ .inst 0x38208020 + 0xc0000000 + 0x400000
8796		\\ cbz wzr, 1f
8797		\\8:
8798		\\ mov x16, x0
8799		\\0:
8800		\\ ldxr x0, [x1]
8801		\\ stlxr w17, x16, [x1]
8802		\\ cbnz w17, 0b
8803		\\1:
8804		: [ret] "={x0}" (-> u64),
8805		: [val] "{x0}" (val),
8806		[ptr] "{x1}" (ptr),
8807		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8808		: "w15", "w16", "w17", "memory"
8809		);
8810		}
8811		const __aarch64_swp8_rel = if (is_darwin)
8812		if (has_lse)
8813		swp8RelDarwinLse
8814		else
8815		swp8RelDarwinNolse
8816		else if (has_lse)
8817		swp8RelNondarwinLse
8818		else
8819		swp8RelNondarwinNolse;
8820		fn ldadd8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8821		@setRuntimeSafety(false);
8822		__init_aarch64_have_lse_atomics();
8823
8824		return asm volatile (
8825		\\ cbz w16, 8f
8826		\\ ldaddl x0, x0, [x1]
8827		\\ cbz wzr, 1f
8828		\\8:
8829		\\ mov x16, x0
8830		\\0:
8831		\\ ldxr x0, [x1]
8832		\\ add x17, x0, x16
8833		\\ stlxr w15, x17, [x1]
8834		\\ cbnz w15, 0b
8835		\\1:
8836		: [ret] "={x0}" (-> u64),
8837		: [val] "{x0}" (val),
8838		[ptr] "{x1}" (ptr),
8839		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8840		: "w15", "w16", "w17", "memory"
8841		);
8842		}
8843		fn ldadd8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8844		@setRuntimeSafety(false);
8845		__init_aarch64_have_lse_atomics();
8846
8847		return asm volatile (
8848		\\ cbz w16, 8f
8849		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
8850		\\ cbz wzr, 1f
8851		\\8:
8852		\\ mov x16, x0
8853		\\0:
8854		\\ ldxr x0, [x1]
8855		\\ add x17, x0, x16
8856		\\ stlxr w15, x17, [x1]
8857		\\ cbnz w15, 0b
8858		\\1:
8859		: [ret] "={x0}" (-> u64),
8860		: [val] "{x0}" (val),
8861		[ptr] "{x1}" (ptr),
8862		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8863		: "w15", "w16", "w17", "memory"
8864		);
8865		}
8866		fn ldadd8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8867		@setRuntimeSafety(false);
8868		__init_aarch64_have_lse_atomics();
8869
8870		return asm volatile (
8871		\\ cbz w16, 8f
8872		\\ ldaddl x0, x0, [x1]
8873		\\ cbz wzr, 1f
8874		\\8:
8875		\\ mov x16, x0
8876		\\0:
8877		\\ ldxr x0, [x1]
8878		\\ add x17, x0, x16
8879		\\ stlxr w15, x17, [x1]
8880		\\ cbnz w15, 0b
8881		\\1:
8882		: [ret] "={x0}" (-> u64),
8883		: [val] "{x0}" (val),
8884		[ptr] "{x1}" (ptr),
8885		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8886		: "w15", "w16", "w17", "memory"
8887		);
8888		}
8889		fn ldadd8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8890		@setRuntimeSafety(false);
8891		__init_aarch64_have_lse_atomics();
8892
8893		return asm volatile (
8894		\\ cbz w16, 8f
8895		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
8896		\\ cbz wzr, 1f
8897		\\8:
8898		\\ mov x16, x0
8899		\\0:
8900		\\ ldxr x0, [x1]
8901		\\ add x17, x0, x16
8902		\\ stlxr w15, x17, [x1]
8903		\\ cbnz w15, 0b
8904		\\1:
8905		: [ret] "={x0}" (-> u64),
8906		: [val] "{x0}" (val),
8907		[ptr] "{x1}" (ptr),
8908		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8909		: "w15", "w16", "w17", "memory"
8910		);
8911		}
8912		const __aarch64_ldadd8_rel = if (is_darwin)
8913		if (has_lse)
8914		ldadd8RelDarwinLse
8915		else
8916		ldadd8RelDarwinNolse
8917		else if (has_lse)
8918		ldadd8RelNondarwinLse
8919		else
8920		ldadd8RelNondarwinNolse;
8921		fn ldclr8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8922		@setRuntimeSafety(false);
8923		__init_aarch64_have_lse_atomics();
8924
8925		return asm volatile (
8926		\\ cbz w16, 8f
8927		\\ ldclrl x0, x0, [x1]
8928		\\ cbz wzr, 1f
8929		\\8:
8930		\\ mov x16, x0
8931		\\0:
8932		\\ ldxr x0, [x1]
8933		\\ bic x17, x0, x16
8934		\\ stlxr w15, x17, [x1]
8935		\\ cbnz w15, 0b
8936		\\1:
8937		: [ret] "={x0}" (-> u64),
8938		: [val] "{x0}" (val),
8939		[ptr] "{x1}" (ptr),
8940		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8941		: "w15", "w16", "w17", "memory"
8942		);
8943		}
8944		fn ldclr8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8945		@setRuntimeSafety(false);
8946		__init_aarch64_have_lse_atomics();
8947
8948		return asm volatile (
8949		\\ cbz w16, 8f
8950		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
8951		\\ cbz wzr, 1f
8952		\\8:
8953		\\ mov x16, x0
8954		\\0:
8955		\\ ldxr x0, [x1]
8956		\\ bic x17, x0, x16
8957		\\ stlxr w15, x17, [x1]
8958		\\ cbnz w15, 0b
8959		\\1:
8960		: [ret] "={x0}" (-> u64),
8961		: [val] "{x0}" (val),
8962		[ptr] "{x1}" (ptr),
8963		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8964		: "w15", "w16", "w17", "memory"
8965		);
8966		}
8967		fn ldclr8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
8968		@setRuntimeSafety(false);
8969		__init_aarch64_have_lse_atomics();
8970
8971		return asm volatile (
8972		\\ cbz w16, 8f
8973		\\ ldclrl x0, x0, [x1]
8974		\\ cbz wzr, 1f
8975		\\8:
8976		\\ mov x16, x0
8977		\\0:
8978		\\ ldxr x0, [x1]
8979		\\ bic x17, x0, x16
8980		\\ stlxr w15, x17, [x1]
8981		\\ cbnz w15, 0b
8982		\\1:
8983		: [ret] "={x0}" (-> u64),
8984		: [val] "{x0}" (val),
8985		[ptr] "{x1}" (ptr),
8986		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
8987		: "w15", "w16", "w17", "memory"
8988		);
8989		}
8990		fn ldclr8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
8991		@setRuntimeSafety(false);
8992		__init_aarch64_have_lse_atomics();
8993
8994		return asm volatile (
8995		\\ cbz w16, 8f
8996		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
8997		\\ cbz wzr, 1f
8998		\\8:
8999		\\ mov x16, x0
9000		\\0:
9001		\\ ldxr x0, [x1]
9002		\\ bic x17, x0, x16
9003		\\ stlxr w15, x17, [x1]
9004		\\ cbnz w15, 0b
9005		\\1:
9006		: [ret] "={x0}" (-> u64),
9007		: [val] "{x0}" (val),
9008		[ptr] "{x1}" (ptr),
9009		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9010		: "w15", "w16", "w17", "memory"
9011		);
9012		}
9013		const __aarch64_ldclr8_rel = if (is_darwin)
9014		if (has_lse)
9015		ldclr8RelDarwinLse
9016		else
9017		ldclr8RelDarwinNolse
9018		else if (has_lse)
9019		ldclr8RelNondarwinLse
9020		else
9021		ldclr8RelNondarwinNolse;
9022		fn ldeor8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9023		@setRuntimeSafety(false);
9024		__init_aarch64_have_lse_atomics();
9025
9026		return asm volatile (
9027		\\ cbz w16, 8f
9028		\\ ldeorl x0, x0, [x1]
9029		\\ cbz wzr, 1f
9030		\\8:
9031		\\ mov x16, x0
9032		\\0:
9033		\\ ldxr x0, [x1]
9034		\\ eor x17, x0, x16
9035		\\ stlxr w15, x17, [x1]
9036		\\ cbnz w15, 0b
9037		\\1:
9038		: [ret] "={x0}" (-> u64),
9039		: [val] "{x0}" (val),
9040		[ptr] "{x1}" (ptr),
9041		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9042		: "w15", "w16", "w17", "memory"
9043		);
9044		}
9045		fn ldeor8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9046		@setRuntimeSafety(false);
9047		__init_aarch64_have_lse_atomics();
9048
9049		return asm volatile (
9050		\\ cbz w16, 8f
9051		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
9052		\\ cbz wzr, 1f
9053		\\8:
9054		\\ mov x16, x0
9055		\\0:
9056		\\ ldxr x0, [x1]
9057		\\ eor x17, x0, x16
9058		\\ stlxr w15, x17, [x1]
9059		\\ cbnz w15, 0b
9060		\\1:
9061		: [ret] "={x0}" (-> u64),
9062		: [val] "{x0}" (val),
9063		[ptr] "{x1}" (ptr),
9064		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9065		: "w15", "w16", "w17", "memory"
9066		);
9067		}
9068		fn ldeor8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9069		@setRuntimeSafety(false);
9070		__init_aarch64_have_lse_atomics();
9071
9072		return asm volatile (
9073		\\ cbz w16, 8f
9074		\\ ldeorl x0, x0, [x1]
9075		\\ cbz wzr, 1f
9076		\\8:
9077		\\ mov x16, x0
9078		\\0:
9079		\\ ldxr x0, [x1]
9080		\\ eor x17, x0, x16
9081		\\ stlxr w15, x17, [x1]
9082		\\ cbnz w15, 0b
9083		\\1:
9084		: [ret] "={x0}" (-> u64),
9085		: [val] "{x0}" (val),
9086		[ptr] "{x1}" (ptr),
9087		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9088		: "w15", "w16", "w17", "memory"
9089		);
9090		}
9091		fn ldeor8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9092		@setRuntimeSafety(false);
9093		__init_aarch64_have_lse_atomics();
9094
9095		return asm volatile (
9096		\\ cbz w16, 8f
9097		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
9098		\\ cbz wzr, 1f
9099		\\8:
9100		\\ mov x16, x0
9101		\\0:
9102		\\ ldxr x0, [x1]
9103		\\ eor x17, x0, x16
9104		\\ stlxr w15, x17, [x1]
9105		\\ cbnz w15, 0b
9106		\\1:
9107		: [ret] "={x0}" (-> u64),
9108		: [val] "{x0}" (val),
9109		[ptr] "{x1}" (ptr),
9110		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9111		: "w15", "w16", "w17", "memory"
9112		);
9113		}
9114		const __aarch64_ldeor8_rel = if (is_darwin)
9115		if (has_lse)
9116		ldeor8RelDarwinLse
9117		else
9118		ldeor8RelDarwinNolse
9119		else if (has_lse)
9120		ldeor8RelNondarwinLse
9121		else
9122		ldeor8RelNondarwinNolse;
9123		fn ldset8RelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9124		@setRuntimeSafety(false);
9125		__init_aarch64_have_lse_atomics();
9126
9127		return asm volatile (
9128		\\ cbz w16, 8f
9129		\\ ldsetl x0, x0, [x1]
9130		\\ cbz wzr, 1f
9131		\\8:
9132		\\ mov x16, x0
9133		\\0:
9134		\\ ldxr x0, [x1]
9135		\\ orr x17, x0, x16
9136		\\ stlxr w15, x17, [x1]
9137		\\ cbnz w15, 0b
9138		\\1:
9139		: [ret] "={x0}" (-> u64),
9140		: [val] "{x0}" (val),
9141		[ptr] "{x1}" (ptr),
9142		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9143		: "w15", "w16", "w17", "memory"
9144		);
9145		}
9146		fn ldset8RelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9147		@setRuntimeSafety(false);
9148		__init_aarch64_have_lse_atomics();
9149
9150		return asm volatile (
9151		\\ cbz w16, 8f
9152		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
9153		\\ cbz wzr, 1f
9154		\\8:
9155		\\ mov x16, x0
9156		\\0:
9157		\\ ldxr x0, [x1]
9158		\\ orr x17, x0, x16
9159		\\ stlxr w15, x17, [x1]
9160		\\ cbnz w15, 0b
9161		\\1:
9162		: [ret] "={x0}" (-> u64),
9163		: [val] "{x0}" (val),
9164		[ptr] "{x1}" (ptr),
9165		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9166		: "w15", "w16", "w17", "memory"
9167		);
9168		}
9169		fn ldset8RelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9170		@setRuntimeSafety(false);
9171		__init_aarch64_have_lse_atomics();
9172
9173		return asm volatile (
9174		\\ cbz w16, 8f
9175		\\ ldsetl x0, x0, [x1]
9176		\\ cbz wzr, 1f
9177		\\8:
9178		\\ mov x16, x0
9179		\\0:
9180		\\ ldxr x0, [x1]
9181		\\ orr x17, x0, x16
9182		\\ stlxr w15, x17, [x1]
9183		\\ cbnz w15, 0b
9184		\\1:
9185		: [ret] "={x0}" (-> u64),
9186		: [val] "{x0}" (val),
9187		[ptr] "{x1}" (ptr),
9188		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9189		: "w15", "w16", "w17", "memory"
9190		);
9191		}
9192		fn ldset8RelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9193		@setRuntimeSafety(false);
9194		__init_aarch64_have_lse_atomics();
9195
9196		return asm volatile (
9197		\\ cbz w16, 8f
9198		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
9199		\\ cbz wzr, 1f
9200		\\8:
9201		\\ mov x16, x0
9202		\\0:
9203		\\ ldxr x0, [x1]
9204		\\ orr x17, x0, x16
9205		\\ stlxr w15, x17, [x1]
9206		\\ cbnz w15, 0b
9207		\\1:
9208		: [ret] "={x0}" (-> u64),
9209		: [val] "{x0}" (val),
9210		[ptr] "{x1}" (ptr),
9211		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9212		: "w15", "w16", "w17", "memory"
9213		);
9214		}
9215		const __aarch64_ldset8_rel = if (is_darwin)
9216		if (has_lse)
9217		ldset8RelDarwinLse
9218		else
9219		ldset8RelDarwinNolse
9220		else if (has_lse)
9221		ldset8RelNondarwinLse
9222		else
9223		ldset8RelNondarwinNolse;
9224		fn cas8AcqRelDarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9225		@setRuntimeSafety(false);
9226		__init_aarch64_have_lse_atomics();
9227
9228		return asm volatile (
9229		\\ cbz w16, 8f
9230		\\ casal x0, x1, [x2]
9231		\\
9232		\\ cbz wzr, 1f
9233		\\8:
9234		\\ mov x16, x0
9235		\\0:
9236		\\ ldaxr x0, [x2]
9237		\\ cmp x0, x16
9238		\\ bne 1f
9239		\\ stlxr w17, x1, [x2]
9240		\\ cbnz w17, 0b
9241		\\1:
9242		: [ret] "={x0}" (-> u64),
9243		: [expected] "{x0}" (expected),
9244		[desired] "{x1}" (desired),
9245		[ptr] "{x2}" (ptr),
9246		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9247		: "w15", "w16", "w17", "memory"
9248		);
9249		}
9250		fn cas8AcqRelDarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9251		@setRuntimeSafety(false);
9252		__init_aarch64_have_lse_atomics();
9253
9254		return asm volatile (
9255		\\ cbz w16, 8f
9256		\\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
9257		\\
9258		\\ cbz wzr, 1f
9259		\\8:
9260		\\ mov x16, x0
9261		\\0:
9262		\\ ldaxr x0, [x2]
9263		\\ cmp x0, x16
9264		\\ bne 1f
9265		\\ stlxr w17, x1, [x2]
9266		\\ cbnz w17, 0b
9267		\\1:
9268		: [ret] "={x0}" (-> u64),
9269		: [expected] "{x0}" (expected),
9270		[desired] "{x1}" (desired),
9271		[ptr] "{x2}" (ptr),
9272		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9273		: "w15", "w16", "w17", "memory"
9274		);
9275		}
9276		fn cas8AcqRelNondarwinLse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9277		@setRuntimeSafety(false);
9278		__init_aarch64_have_lse_atomics();
9279
9280		return asm volatile (
9281		\\ cbz w16, 8f
9282		\\ casal x0, x1, [x2]
9283		\\
9284		\\ cbz wzr, 1f
9285		\\8:
9286		\\ mov x16, x0
9287		\\0:
9288		\\ ldaxr x0, [x2]
9289		\\ cmp x0, x16
9290		\\ bne 1f
9291		\\ stlxr w17, x1, [x2]
9292		\\ cbnz w17, 0b
9293		\\1:
9294		: [ret] "={x0}" (-> u64),
9295		: [expected] "{x0}" (expected),
9296		[desired] "{x1}" (desired),
9297		[ptr] "{x2}" (ptr),
9298		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9299		: "w15", "w16", "w17", "memory"
9300		);
9301		}
9302		fn cas8AcqRelNondarwinNolse(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9303		@setRuntimeSafety(false);
9304		__init_aarch64_have_lse_atomics();
9305
9306		return asm volatile (
9307		\\ cbz w16, 8f
9308		\\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
	1425	\\ .inst 0x08a07c41 + 0x80000000 + 0x408000
9309	1426	\\
9310	1427	\\ cbz wzr, 1f
9311	1428	\\8:
9312		\\ mov x16, x0
	1429	\\ mov w16, w0
9313	1430	\\0:
9314		\\ ldaxr x0, [x2]
9315		\\ cmp x0, x16
	1431	\\ ldaxr w0, [x2]
	1432	\\ cmp w0, w16
9316	1433	\\ bne 1f
9317		\\ stlxr w17, x1, [x2]
	1434	\\ stlxr w17, w1, [x2]
9318	1435	\\ cbnz w17, 0b
9319	1436	\\1:
9320		: [ret] "={x0}" (-> u64),
9321		: [expected] "{x0}" (expected),
9322		[desired] "{x1}" (desired),
	1437	: [ret] "={w0}" (-> u32),
	1438	: [expected] "{w0}" (expected),
	1439	[desired] "{w1}" (desired),
9323	1440	[ptr] "{x2}" (ptr),
9324	1441	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9325	1442	: "w15", "w16", "w17", "memory"
9326	1443	);
9327	1444	}
9328		const __aarch64_cas8_acq_rel = if (is_darwin)
9329		if (has_lse)
9330		cas8AcqRelDarwinLse
9331		else
9332		cas8AcqRelDarwinNolse
9333		else if (has_lse)
9334		cas8AcqRelNondarwinLse
9335		else
9336		cas8AcqRelNondarwinNolse;
9337		fn swp8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9338		@setRuntimeSafety(false);
9339		__init_aarch64_have_lse_atomics();
9340
9341		return asm volatile (
9342		\\ cbz w16, 8f
9343		\\ swpal x0, x0, [x1]
9344		\\ cbz wzr, 1f
9345		\\8:
9346		\\ mov x16, x0
9347		\\0:
9348		\\ ldaxr x0, [x1]
9349		\\ stlxr w17, x16, [x1]
9350		\\ cbnz w17, 0b
9351		\\1:
9352		: [ret] "={x0}" (-> u64),
9353		: [val] "{x0}" (val),
9354		[ptr] "{x1}" (ptr),
9355		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9356		: "w15", "w16", "w17", "memory"
9357		);
9358		}
9359		fn swp8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9360		@setRuntimeSafety(false);
9361		__init_aarch64_have_lse_atomics();
9362
9363		return asm volatile (
9364		\\ cbz w16, 8f
9365		\\ .inst 0x38208020 + 0xc0000000 + 0xc00000
9366		\\ cbz wzr, 1f
9367		\\8:
9368		\\ mov x16, x0
9369		\\0:
9370		\\ ldaxr x0, [x1]
9371		\\ stlxr w17, x16, [x1]
9372		\\ cbnz w17, 0b
9373		\\1:
9374		: [ret] "={x0}" (-> u64),
9375		: [val] "{x0}" (val),
9376		[ptr] "{x1}" (ptr),
9377		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9378		: "w15", "w16", "w17", "memory"
9379		);
9380		}
9381		fn swp8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9382		@setRuntimeSafety(false);
9383		__init_aarch64_have_lse_atomics();
9384
9385		return asm volatile (
9386		\\ cbz w16, 8f
9387		\\ swpal x0, x0, [x1]
9388		\\ cbz wzr, 1f
9389		\\8:
9390		\\ mov x16, x0
9391		\\0:
9392		\\ ldaxr x0, [x1]
9393		\\ stlxr w17, x16, [x1]
9394		\\ cbnz w17, 0b
9395		\\1:
9396		: [ret] "={x0}" (-> u64),
9397		: [val] "{x0}" (val),
9398		[ptr] "{x1}" (ptr),
9399		[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9400		: "w15", "w16", "w17", "memory"
9401		);
9402		}
9403		fn swp8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1445	fn __aarch64_swp4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
9404	1446	@setRuntimeSafety(false);
9405		__init_aarch64_have_lse_atomics();
9406
9407	1447	return asm volatile (
9408	1448	\\ cbz w16, 8f
9409		\\ .inst 0x38208020 + 0xc0000000 + 0xc00000
	1449	\\ .inst 0x38208020 + 0x80000000 + 0xc00000
9410	1450	\\ cbz wzr, 1f
9411	1451	\\8:
9412		\\ mov x16, x0
	1452	\\ mov w16, w0
9413	1453	\\0:
9414		\\ ldaxr x0, [x1]
9415		\\ stlxr w17, x16, [x1]
	1454	\\ ldaxr w0, [x1]
	1455	\\ stlxr w17, w16, [x1]
9416	1456	\\ cbnz w17, 0b
9417	1457	\\1:
9418		: [ret] "={x0}" (-> u64),
9419		: [val] "{x0}" (val),
	1458	: [ret] "={w0}" (-> u32),
	1459	: [val] "{w0}" (val),
9420	1460	[ptr] "{x1}" (ptr),
9421	1461	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9422	1462	: "w15", "w16", "w17", "memory"
9423	1463	);
9424	1464	}
9425		const __aarch64_swp8_acq_rel = if (is_darwin)
9426		if (has_lse)
9427		swp8AcqRelDarwinLse
9428		else
9429		swp8AcqRelDarwinNolse
9430		else if (has_lse)
9431		swp8AcqRelNondarwinLse
9432		else
9433		swp8AcqRelNondarwinNolse;
9434		fn ldadd8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1465	fn __aarch64_ldadd4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
9435	1466	@setRuntimeSafety(false);
9436		__init_aarch64_have_lse_atomics();
9437
9438	1467	return asm volatile (
9439	1468	\\ cbz w16, 8f
9440		\\ ldaddal x0, x0, [x1]
	1469	\\ .inst 0x38200020 + 0x0000 + 0x80000000 + 0xc00000
9441	1470	\\ cbz wzr, 1f
9442	1471	\\8:
9443		\\ mov x16, x0
	1472	\\ mov w16, w0
9444	1473	\\0:
9445		\\ ldaxr x0, [x1]
9446		\\ add x17, x0, x16
9447		\\ stlxr w15, x17, [x1]
	1474	\\ ldaxr w0, [x1]
	1475	\\ add w17, w0, w16
	1476	\\ stlxr w15, w17, [x1]
9448	1477	\\ cbnz w15, 0b
9449	1478	\\1:
9450		: [ret] "={x0}" (-> u64),
9451		: [val] "{x0}" (val),
	1479	: [ret] "={w0}" (-> u32),
	1480	: [val] "{w0}" (val),
9452	1481	[ptr] "{x1}" (ptr),
9453	1482	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9454	1483	: "w15", "w16", "w17", "memory"
9455	1484	);
9456	1485	}
9457		fn ldadd8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1486	fn __aarch64_ldclr4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
9458	1487	@setRuntimeSafety(false);
9459		__init_aarch64_have_lse_atomics();
9460
9461	1488	return asm volatile (
9462	1489	\\ cbz w16, 8f
9463		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
	1490	\\ .inst 0x38200020 + 0x1000 + 0x80000000 + 0xc00000
9464	1491	\\ cbz wzr, 1f
9465	1492	\\8:
9466		\\ mov x16, x0
	1493	\\ mov w16, w0
9467	1494	\\0:
9468		\\ ldaxr x0, [x1]
9469		\\ add x17, x0, x16
9470		\\ stlxr w15, x17, [x1]
	1495	\\ ldaxr w0, [x1]
	1496	\\ bic w17, w0, w16
	1497	\\ stlxr w15, w17, [x1]
9471	1498	\\ cbnz w15, 0b
9472	1499	\\1:
9473		: [ret] "={x0}" (-> u64),
9474		: [val] "{x0}" (val),
	1500	: [ret] "={w0}" (-> u32),
	1501	: [val] "{w0}" (val),
9475	1502	[ptr] "{x1}" (ptr),
9476	1503	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9477	1504	: "w15", "w16", "w17", "memory"
9478	1505	);
9479	1506	}
9480		fn ldadd8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1507	fn __aarch64_ldeor4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
9481	1508	@setRuntimeSafety(false);
9482		__init_aarch64_have_lse_atomics();
9483
9484	1509	return asm volatile (
9485	1510	\\ cbz w16, 8f
9486		\\ ldaddal x0, x0, [x1]
	1511	\\ .inst 0x38200020 + 0x2000 + 0x80000000 + 0xc00000
9487	1512	\\ cbz wzr, 1f
9488	1513	\\8:
9489		\\ mov x16, x0
	1514	\\ mov w16, w0
9490	1515	\\0:
9491		\\ ldaxr x0, [x1]
9492		\\ add x17, x0, x16
9493		\\ stlxr w15, x17, [x1]
	1516	\\ ldaxr w0, [x1]
	1517	\\ eor w17, w0, w16
	1518	\\ stlxr w15, w17, [x1]
9494	1519	\\ cbnz w15, 0b
9495	1520	\\1:
9496		: [ret] "={x0}" (-> u64),
9497		: [val] "{x0}" (val),
	1521	: [ret] "={w0}" (-> u32),
	1522	: [val] "{w0}" (val),
9498	1523	[ptr] "{x1}" (ptr),
9499	1524	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9500	1525	: "w15", "w16", "w17", "memory"
9501	1526	);
9502	1527	}
9503		fn ldadd8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1528	fn __aarch64_ldset4_acq_rel(val: u32, ptr: *u32) callconv(.C) u32 {
9504	1529	@setRuntimeSafety(false);
9505		__init_aarch64_have_lse_atomics();
9506
9507	1530	return asm volatile (
9508	1531	\\ cbz w16, 8f
9509		\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
	1532	\\ .inst 0x38200020 + 0x3000 + 0x80000000 + 0xc00000
9510	1533	\\ cbz wzr, 1f
9511	1534	\\8:
9512		\\ mov x16, x0
	1535	\\ mov w16, w0
9513	1536	\\0:
9514		\\ ldaxr x0, [x1]
9515		\\ add x17, x0, x16
9516		\\ stlxr w15, x17, [x1]
	1537	\\ ldaxr w0, [x1]
	1538	\\ orr w17, w0, w16
	1539	\\ stlxr w15, w17, [x1]
9517	1540	\\ cbnz w15, 0b
9518	1541	\\1:
9519		: [ret] "={x0}" (-> u64),
9520		: [val] "{x0}" (val),
	1542	: [ret] "={w0}" (-> u32),
	1543	: [val] "{w0}" (val),
9521	1544	[ptr] "{x1}" (ptr),
9522	1545	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9523	1546	: "w15", "w16", "w17", "memory"
9524	1547	);
9525	1548	}
9526		const __aarch64_ldadd8_acq_rel = if (is_darwin)
9527		if (has_lse)
9528		ldadd8AcqRelDarwinLse
9529		else
9530		ldadd8AcqRelDarwinNolse
9531		else if (has_lse)
9532		ldadd8AcqRelNondarwinLse
9533		else
9534		ldadd8AcqRelNondarwinNolse;
9535		fn ldclr8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1549	fn __aarch64_cas8_relax(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9536	1550	@setRuntimeSafety(false);
9537		__init_aarch64_have_lse_atomics();
9538
9539	1551	return asm volatile (
9540	1552	\\ cbz w16, 8f
9541		\\ ldclral x0, x0, [x1]
	1553	\\ .inst 0x08a07c41 + 0xc0000000 + 0x000000
	1554	\\
9542	1555	\\ cbz wzr, 1f
9543	1556	\\8:
9544	1557	\\ mov x16, x0
9545	1558	\\0:
9546		\\ ldaxr x0, [x1]
9547		\\ bic x17, x0, x16
9548		\\ stlxr w15, x17, [x1]
9549		\\ cbnz w15, 0b
	1559	\\ ldxr x0, [x2]
	1560	\\ cmp x0, x16
	1561	\\ bne 1f
	1562	\\ stxr w17, x1, [x2]
	1563	\\ cbnz w17, 0b
9550	1564	\\1:
9551	1565	: [ret] "={x0}" (-> u64),
9552		: [val] "{x0}" (val),
9553		[ptr] "{x1}" (ptr),
	1566	: [expected] "{x0}" (expected),
	1567	[desired] "{x1}" (desired),
	1568	[ptr] "{x2}" (ptr),
9554	1569	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9555	1570	: "w15", "w16", "w17", "memory"
9556	1571	);
9557	1572	}
9558		fn ldclr8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1573	fn __aarch64_swp8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
9559	1574	@setRuntimeSafety(false);
9560		__init_aarch64_have_lse_atomics();
9561
9562	1575	return asm volatile (
9563	1576	\\ cbz w16, 8f
9564		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
	1577	\\ .inst 0x38208020 + 0xc0000000 + 0x000000
9565	1578	\\ cbz wzr, 1f
9566	1579	\\8:
9567	1580	\\ mov x16, x0
9568	1581	\\0:
9569		\\ ldaxr x0, [x1]
9570		\\ bic x17, x0, x16
9571		\\ stlxr w15, x17, [x1]
9572		\\ cbnz w15, 0b
	1582	\\ ldxr x0, [x1]
	1583	\\ stxr w17, x16, [x1]
	1584	\\ cbnz w17, 0b
9573	1585	\\1:
9574	1586	: [ret] "={x0}" (-> u64),
9575	1587	: [val] "{x0}" (val),
...	...	@@ -9578,20 +1590,18 @@ fn ldclr8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9578	1590	: "w15", "w16", "w17", "memory"
9579	1591	);
9580	1592	}
9581		fn ldclr8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1593	fn __aarch64_ldadd8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
9582	1594	@setRuntimeSafety(false);
9583		__init_aarch64_have_lse_atomics();
9584
9585	1595	return asm volatile (
9586	1596	\\ cbz w16, 8f
9587		\\ ldclral x0, x0, [x1]
	1597	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x000000
9588	1598	\\ cbz wzr, 1f
9589	1599	\\8:
9590	1600	\\ mov x16, x0
9591	1601	\\0:
9592		\\ ldaxr x0, [x1]
9593		\\ bic x17, x0, x16
9594		\\ stlxr w15, x17, [x1]
	1602	\\ ldxr x0, [x1]
	1603	\\ add x17, x0, x16
	1604	\\ stxr w15, x17, [x1]
9595	1605	\\ cbnz w15, 0b
9596	1606	\\1:
9597	1607	: [ret] "={x0}" (-> u64),
...	...	@@ -9601,20 +1611,18 @@ fn ldclr8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9601	1611	: "w15", "w16", "w17", "memory"
9602	1612	);
9603	1613	}
9604		fn ldclr8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1614	fn __aarch64_ldclr8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
9605	1615	@setRuntimeSafety(false);
9606		__init_aarch64_have_lse_atomics();
9607
9608	1616	return asm volatile (
9609	1617	\\ cbz w16, 8f
9610		\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
	1618	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x000000
9611	1619	\\ cbz wzr, 1f
9612	1620	\\8:
9613	1621	\\ mov x16, x0
9614	1622	\\0:
9615		\\ ldaxr x0, [x1]
	1623	\\ ldxr x0, [x1]
9616	1624	\\ bic x17, x0, x16
9617		\\ stlxr w15, x17, [x1]
	1625	\\ stxr w15, x17, [x1]
9618	1626	\\ cbnz w15, 0b
9619	1627	\\1:
9620	1628	: [ret] "={x0}" (-> u64),
...	...	@@ -9624,29 +1632,18 @@ fn ldclr8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9624	1632	: "w15", "w16", "w17", "memory"
9625	1633	);
9626	1634	}
9627		const __aarch64_ldclr8_acq_rel = if (is_darwin)
9628		if (has_lse)
9629		ldclr8AcqRelDarwinLse
9630		else
9631		ldclr8AcqRelDarwinNolse
9632		else if (has_lse)
9633		ldclr8AcqRelNondarwinLse
9634		else
9635		ldclr8AcqRelNondarwinNolse;
9636		fn ldeor8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1635	fn __aarch64_ldeor8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
9637	1636	@setRuntimeSafety(false);
9638		__init_aarch64_have_lse_atomics();
9639
9640	1637	return asm volatile (
9641	1638	\\ cbz w16, 8f
9642		\\ ldeoral x0, x0, [x1]
	1639	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x000000
9643	1640	\\ cbz wzr, 1f
9644	1641	\\8:
9645	1642	\\ mov x16, x0
9646	1643	\\0:
9647		\\ ldaxr x0, [x1]
	1644	\\ ldxr x0, [x1]
9648	1645	\\ eor x17, x0, x16
9649		\\ stlxr w15, x17, [x1]
	1646	\\ stxr w15, x17, [x1]
9650	1647	\\ cbnz w15, 0b
9651	1648	\\1:
9652	1649	: [ret] "={x0}" (-> u64),
...	...	@@ -9656,20 +1653,18 @@ fn ldeor8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9656	1653	: "w15", "w16", "w17", "memory"
9657	1654	);
9658	1655	}
9659		fn ldeor8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1656	fn __aarch64_ldset8_relax(val: u64, ptr: *u64) callconv(.C) u64 {
9660	1657	@setRuntimeSafety(false);
9661		__init_aarch64_have_lse_atomics();
9662
9663	1658	return asm volatile (
9664	1659	\\ cbz w16, 8f
9665		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
	1660	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x000000
9666	1661	\\ cbz wzr, 1f
9667	1662	\\8:
9668	1663	\\ mov x16, x0
9669	1664	\\0:
9670		\\ ldaxr x0, [x1]
9671		\\ eor x17, x0, x16
9672		\\ stlxr w15, x17, [x1]
	1665	\\ ldxr x0, [x1]
	1666	\\ orr x17, x0, x16
	1667	\\ stxr w15, x17, [x1]
9673	1668	\\ cbnz w15, 0b
9674	1669	\\1:
9675	1670	: [ret] "={x0}" (-> u64),
...	...	@@ -9679,44 +1674,42 @@ fn ldeor8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9679	1674	: "w15", "w16", "w17", "memory"
9680	1675	);
9681	1676	}
9682		fn ldeor8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1677	fn __aarch64_cas8_acq(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9683	1678	@setRuntimeSafety(false);
9684		__init_aarch64_have_lse_atomics();
9685
9686	1679	return asm volatile (
9687	1680	\\ cbz w16, 8f
9688		\\ ldeoral x0, x0, [x1]
	1681	\\ .inst 0x08a07c41 + 0xc0000000 + 0x400000
	1682	\\
9689	1683	\\ cbz wzr, 1f
9690	1684	\\8:
9691	1685	\\ mov x16, x0
9692	1686	\\0:
9693		\\ ldaxr x0, [x1]
9694		\\ eor x17, x0, x16
9695		\\ stlxr w15, x17, [x1]
9696		\\ cbnz w15, 0b
	1687	\\ ldaxr x0, [x2]
	1688	\\ cmp x0, x16
	1689	\\ bne 1f
	1690	\\ stxr w17, x1, [x2]
	1691	\\ cbnz w17, 0b
9697	1692	\\1:
9698	1693	: [ret] "={x0}" (-> u64),
9699		: [val] "{x0}" (val),
9700		[ptr] "{x1}" (ptr),
	1694	: [expected] "{x0}" (expected),
	1695	[desired] "{x1}" (desired),
	1696	[ptr] "{x2}" (ptr),
9701	1697	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9702	1698	: "w15", "w16", "w17", "memory"
9703	1699	);
9704	1700	}
9705		fn ldeor8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1701	fn __aarch64_swp8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
9706	1702	@setRuntimeSafety(false);
9707		__init_aarch64_have_lse_atomics();
9708
9709	1703	return asm volatile (
9710	1704	\\ cbz w16, 8f
9711		\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
	1705	\\ .inst 0x38208020 + 0xc0000000 + 0x800000
9712	1706	\\ cbz wzr, 1f
9713	1707	\\8:
9714	1708	\\ mov x16, x0
9715	1709	\\0:
9716	1710	\\ ldaxr x0, [x1]
9717		\\ eor x17, x0, x16
9718		\\ stlxr w15, x17, [x1]
9719		\\ cbnz w15, 0b
	1711	\\ stxr w17, x16, [x1]
	1712	\\ cbnz w17, 0b
9720	1713	\\1:
9721	1714	: [ret] "={x0}" (-> u64),
9722	1715	: [val] "{x0}" (val),
...	...	@@ -9725,29 +1718,18 @@ fn ldeor8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9725	1718	: "w15", "w16", "w17", "memory"
9726	1719	);
9727	1720	}
9728		const __aarch64_ldeor8_acq_rel = if (is_darwin)
9729		if (has_lse)
9730		ldeor8AcqRelDarwinLse
9731		else
9732		ldeor8AcqRelDarwinNolse
9733		else if (has_lse)
9734		ldeor8AcqRelNondarwinLse
9735		else
9736		ldeor8AcqRelNondarwinNolse;
9737		fn ldset8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1721	fn __aarch64_ldadd8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
9738	1722	@setRuntimeSafety(false);
9739		__init_aarch64_have_lse_atomics();
9740
9741	1723	return asm volatile (
9742	1724	\\ cbz w16, 8f
9743		\\ ldsetal x0, x0, [x1]
	1725	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x800000
9744	1726	\\ cbz wzr, 1f
9745	1727	\\8:
9746	1728	\\ mov x16, x0
9747	1729	\\0:
9748	1730	\\ ldaxr x0, [x1]
9749		\\ orr x17, x0, x16
9750		\\ stlxr w15, x17, [x1]
	1731	\\ add x17, x0, x16
	1732	\\ stxr w15, x17, [x1]
9751	1733	\\ cbnz w15, 0b
9752	1734	\\1:
9753	1735	: [ret] "={x0}" (-> u64),
...	...	@@ -9757,20 +1739,18 @@ fn ldset8AcqRelDarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9757	1739	: "w15", "w16", "w17", "memory"
9758	1740	);
9759	1741	}
9760		fn ldset8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1742	fn __aarch64_ldclr8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
9761	1743	@setRuntimeSafety(false);
9762		__init_aarch64_have_lse_atomics();
9763
9764	1744	return asm volatile (
9765	1745	\\ cbz w16, 8f
9766		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
	1746	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x800000
9767	1747	\\ cbz wzr, 1f
9768	1748	\\8:
9769	1749	\\ mov x16, x0
9770	1750	\\0:
9771	1751	\\ ldaxr x0, [x1]
9772		\\ orr x17, x0, x16
9773		\\ stlxr w15, x17, [x1]
	1752	\\ bic x17, x0, x16
	1753	\\ stxr w15, x17, [x1]
9774	1754	\\ cbnz w15, 0b
9775	1755	\\1:
9776	1756	: [ret] "={x0}" (-> u64),
...	...	@@ -9780,20 +1760,18 @@ fn ldset8AcqRelDarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9780	1760	: "w15", "w16", "w17", "memory"
9781	1761	);
9782	1762	}
9783		fn ldset8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
	1763	fn __aarch64_ldeor8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
9784	1764	@setRuntimeSafety(false);
9785		__init_aarch64_have_lse_atomics();
9786
9787	1765	return asm volatile (
9788	1766	\\ cbz w16, 8f
9789		\\ ldsetal x0, x0, [x1]
	1767	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x800000
9790	1768	\\ cbz wzr, 1f
9791	1769	\\8:
9792	1770	\\ mov x16, x0
9793	1771	\\0:
9794	1772	\\ ldaxr x0, [x1]
9795		\\ orr x17, x0, x16
9796		\\ stlxr w15, x17, [x1]
	1773	\\ eor x17, x0, x16
	1774	\\ stxr w15, x17, [x1]
9797	1775	\\ cbnz w15, 0b
9798	1776	\\1:
9799	1777	: [ret] "={x0}" (-> u64),
...	...	@@ -9803,20 +1781,18 @@ fn ldset8AcqRelNondarwinLse(val: u64, ptr: *u64) callconv(.C) u64 {
9803	1781	: "w15", "w16", "w17", "memory"
9804	1782	);
9805	1783	}
9806		fn ldset8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
	1784	fn __aarch64_ldset8_acq(val: u64, ptr: *u64) callconv(.C) u64 {
9807	1785	@setRuntimeSafety(false);
9808		__init_aarch64_have_lse_atomics();
9809
9810	1786	return asm volatile (
9811	1787	\\ cbz w16, 8f
9812		\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
	1788	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x800000
9813	1789	\\ cbz wzr, 1f
9814	1790	\\8:
9815	1791	\\ mov x16, x0
9816	1792	\\0:
9817	1793	\\ ldaxr x0, [x1]
9818	1794	\\ orr x17, x0, x16
9819		\\ stlxr w15, x17, [x1]
	1795	\\ stxr w15, x17, [x1]
9820	1796	\\ cbnz w15, 0b
9821	1797	\\1:
9822	1798	: [ret] "={x0}" (-> u64),
...	...	@@ -9826,36 +1802,23 @@ fn ldset8AcqRelNondarwinNolse(val: u64, ptr: *u64) callconv(.C) u64 {
9826	1802	: "w15", "w16", "w17", "memory"
9827	1803	);
9828	1804	}
9829		const __aarch64_ldset8_acq_rel = if (is_darwin)
9830		if (has_lse)
9831		ldset8AcqRelDarwinLse
9832		else
9833		ldset8AcqRelDarwinNolse
9834		else if (has_lse)
9835		ldset8AcqRelNondarwinLse
9836		else
9837		ldset8AcqRelNondarwinNolse;
9838		fn cas16RelaxDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1805	fn __aarch64_cas8_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
9839	1806	@setRuntimeSafety(false);
9840		__init_aarch64_have_lse_atomics();
9841
9842	1807	return asm volatile (
9843	1808	\\ cbz w16, 8f
9844		\\ casp x0, x1, x2, x3, [x4]
	1809	\\ .inst 0x08a07c41 + 0xc0000000 + 0x008000
9845	1810	\\
9846	1811	\\ cbz wzr, 1f
9847	1812	\\8:
9848	1813	\\ mov x16, x0
9849		\\ mov x17, x1
9850	1814	\\0:
9851		\\ ldxp x0, x1, [x4]
	1815	\\ ldxr x0, [x2]
9852	1816	\\ cmp x0, x16
9853		\\ ccmp x1, x17, #0, eq
9854	1817	\\ bne 1f
9855		\\ stxp w15, x2, x3, [x4]
9856		\\ cbnz w15, 0b
	1818	\\ stlxr w17, x1, [x2]
	1819	\\ cbnz w17, 0b
9857	1820	\\1:
9858		: [ret] "={x0}" (-> u128),
	1821	: [ret] "={x0}" (-> u64),
9859	1822	: [expected] "{x0}" (expected),
9860	1823	[desired] "{x1}" (desired),
9861	1824	[ptr] "{x2}" (ptr),
...	...	@@ -9863,176 +1826,127 @@ fn cas16RelaxDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u
9863	1826	: "w15", "w16", "w17", "memory"
9864	1827	);
9865	1828	}
9866		fn cas16RelaxDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1829	fn __aarch64_swp8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
9867	1830	@setRuntimeSafety(false);
9868		__init_aarch64_have_lse_atomics();
9869
9870	1831	return asm volatile (
9871	1832	\\ cbz w16, 8f
9872		\\ .inst 0x48207c82 + 0x000000
9873		\\
	1833	\\ .inst 0x38208020 + 0xc0000000 + 0x400000
9874	1834	\\ cbz wzr, 1f
9875	1835	\\8:
9876	1836	\\ mov x16, x0
9877		\\ mov x17, x1
9878	1837	\\0:
9879		\\ ldxp x0, x1, [x4]
9880		\\ cmp x0, x16
9881		\\ ccmp x1, x17, #0, eq
9882		\\ bne 1f
9883		\\ stxp w15, x2, x3, [x4]
9884		\\ cbnz w15, 0b
	1838	\\ ldxr x0, [x1]
	1839	\\ stlxr w17, x16, [x1]
	1840	\\ cbnz w17, 0b
9885	1841	\\1:
9886		: [ret] "={x0}" (-> u128),
9887		: [expected] "{x0}" (expected),
9888		[desired] "{x1}" (desired),
9889		[ptr] "{x2}" (ptr),
	1842	: [ret] "={x0}" (-> u64),
	1843	: [val] "{x0}" (val),
	1844	[ptr] "{x1}" (ptr),
9890	1845	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9891	1846	: "w15", "w16", "w17", "memory"
9892	1847	);
9893	1848	}
9894		fn cas16RelaxNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1849	fn __aarch64_ldadd8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
9895	1850	@setRuntimeSafety(false);
9896		__init_aarch64_have_lse_atomics();
9897
9898	1851	return asm volatile (
9899	1852	\\ cbz w16, 8f
9900		\\ casp x0, x1, x2, x3, [x4]
9901		\\
	1853	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0x400000
9902	1854	\\ cbz wzr, 1f
9903	1855	\\8:
9904	1856	\\ mov x16, x0
9905		\\ mov x17, x1
9906	1857	\\0:
9907		\\ ldxp x0, x1, [x4]
9908		\\ cmp x0, x16
9909		\\ ccmp x1, x17, #0, eq
9910		\\ bne 1f
9911		\\ stxp w15, x2, x3, [x4]
	1858	\\ ldxr x0, [x1]
	1859	\\ add x17, x0, x16
	1860	\\ stlxr w15, x17, [x1]
9912	1861	\\ cbnz w15, 0b
9913	1862	\\1:
9914		: [ret] "={x0}" (-> u128),
9915		: [expected] "{x0}" (expected),
9916		[desired] "{x1}" (desired),
9917		[ptr] "{x2}" (ptr),
	1863	: [ret] "={x0}" (-> u64),
	1864	: [val] "{x0}" (val),
	1865	[ptr] "{x1}" (ptr),
9918	1866	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9919	1867	: "w15", "w16", "w17", "memory"
9920	1868	);
9921	1869	}
9922		fn cas16RelaxNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1870	fn __aarch64_ldclr8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
9923	1871	@setRuntimeSafety(false);
9924		__init_aarch64_have_lse_atomics();
9925
9926	1872	return asm volatile (
9927	1873	\\ cbz w16, 8f
9928		\\ .inst 0x48207c82 + 0x000000
9929		\\
	1874	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0x400000
9930	1875	\\ cbz wzr, 1f
9931	1876	\\8:
9932	1877	\\ mov x16, x0
9933		\\ mov x17, x1
9934	1878	\\0:
9935		\\ ldxp x0, x1, [x4]
9936		\\ cmp x0, x16
9937		\\ ccmp x1, x17, #0, eq
9938		\\ bne 1f
9939		\\ stxp w15, x2, x3, [x4]
	1879	\\ ldxr x0, [x1]
	1880	\\ bic x17, x0, x16
	1881	\\ stlxr w15, x17, [x1]
9940	1882	\\ cbnz w15, 0b
9941	1883	\\1:
9942		: [ret] "={x0}" (-> u128),
9943		: [expected] "{x0}" (expected),
9944		[desired] "{x1}" (desired),
9945		[ptr] "{x2}" (ptr),
	1884	: [ret] "={x0}" (-> u64),
	1885	: [val] "{x0}" (val),
	1886	[ptr] "{x1}" (ptr),
9946	1887	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9947	1888	: "w15", "w16", "w17", "memory"
9948	1889	);
9949	1890	}
9950		const __aarch64_cas16_relax = if (is_darwin)
9951		if (has_lse)
9952		cas16RelaxDarwinLse
9953		else
9954		cas16RelaxDarwinNolse
9955		else if (has_lse)
9956		cas16RelaxNondarwinLse
9957		else
9958		cas16RelaxNondarwinNolse;
9959		fn cas16AcqDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1891	fn __aarch64_ldeor8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
9960	1892	@setRuntimeSafety(false);
9961		__init_aarch64_have_lse_atomics();
9962
9963	1893	return asm volatile (
9964	1894	\\ cbz w16, 8f
9965		\\ caspa x0, x1, x2, x3, [x4]
9966		\\
	1895	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0x400000
9967	1896	\\ cbz wzr, 1f
9968	1897	\\8:
9969	1898	\\ mov x16, x0
9970		\\ mov x17, x1
9971	1899	\\0:
9972		\\ ldaxp x0, x1, [x4]
9973		\\ cmp x0, x16
9974		\\ ccmp x1, x17, #0, eq
9975		\\ bne 1f
9976		\\ stxp w15, x2, x3, [x4]
	1900	\\ ldxr x0, [x1]
	1901	\\ eor x17, x0, x16
	1902	\\ stlxr w15, x17, [x1]
9977	1903	\\ cbnz w15, 0b
9978	1904	\\1:
9979		: [ret] "={x0}" (-> u128),
9980		: [expected] "{x0}" (expected),
9981		[desired] "{x1}" (desired),
9982		[ptr] "{x2}" (ptr),
	1905	: [ret] "={x0}" (-> u64),
	1906	: [val] "{x0}" (val),
	1907	[ptr] "{x1}" (ptr),
9983	1908	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
9984	1909	: "w15", "w16", "w17", "memory"
9985	1910	);
9986	1911	}
9987		fn cas16AcqDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1912	fn __aarch64_ldset8_rel(val: u64, ptr: *u64) callconv(.C) u64 {
9988	1913	@setRuntimeSafety(false);
9989		__init_aarch64_have_lse_atomics();
9990
9991	1914	return asm volatile (
9992	1915	\\ cbz w16, 8f
9993		\\ .inst 0x48207c82 + 0x400000
9994		\\
	1916	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0x400000
9995	1917	\\ cbz wzr, 1f
9996	1918	\\8:
9997	1919	\\ mov x16, x0
9998		\\ mov x17, x1
9999	1920	\\0:
10000		\\ ldaxp x0, x1, [x4]
10001		\\ cmp x0, x16
10002		\\ ccmp x1, x17, #0, eq
10003		\\ bne 1f
10004		\\ stxp w15, x2, x3, [x4]
	1921	\\ ldxr x0, [x1]
	1922	\\ orr x17, x0, x16
	1923	\\ stlxr w15, x17, [x1]
10005	1924	\\ cbnz w15, 0b
10006	1925	\\1:
10007		: [ret] "={x0}" (-> u128),
10008		: [expected] "{x0}" (expected),
10009		[desired] "{x1}" (desired),
10010		[ptr] "{x2}" (ptr),
	1926	: [ret] "={x0}" (-> u64),
	1927	: [val] "{x0}" (val),
	1928	[ptr] "{x1}" (ptr),
10011	1929	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10012	1930	: "w15", "w16", "w17", "memory"
10013	1931	);
10014	1932	}
10015		fn cas16AcqNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1933	fn __aarch64_cas8_acq_rel(expected: u64, desired: u64, ptr: *u64) callconv(.C) u64 {
10016	1934	@setRuntimeSafety(false);
10017		__init_aarch64_have_lse_atomics();
10018
10019	1935	return asm volatile (
10020	1936	\\ cbz w16, 8f
10021		\\ caspa x0, x1, x2, x3, [x4]
	1937	\\ .inst 0x08a07c41 + 0xc0000000 + 0x408000
10022	1938	\\
10023	1939	\\ cbz wzr, 1f
10024	1940	\\8:
10025	1941	\\ mov x16, x0
10026		\\ mov x17, x1
10027	1942	\\0:
10028		\\ ldaxp x0, x1, [x4]
	1943	\\ ldaxr x0, [x2]
10029	1944	\\ cmp x0, x16
10030		\\ ccmp x1, x17, #0, eq
10031	1945	\\ bne 1f
10032		\\ stxp w15, x2, x3, [x4]
10033		\\ cbnz w15, 0b
	1946	\\ stlxr w17, x1, [x2]
	1947	\\ cbnz w17, 0b
10034	1948	\\1:
10035		: [ret] "={x0}" (-> u128),
	1949	: [ret] "={x0}" (-> u64),
10036	1950	: [expected] "{x0}" (expected),
10037	1951	[desired] "{x1}" (desired),
10038	1952	[ptr] "{x2}" (ptr),
...	...	@@ -10040,182 +1954,126 @@ fn cas16AcqNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C)
10040	1954	: "w15", "w16", "w17", "memory"
10041	1955	);
10042	1956	}
10043		fn cas16AcqNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1957	fn __aarch64_swp8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
10044	1958	@setRuntimeSafety(false);
10045		__init_aarch64_have_lse_atomics();
10046
10047	1959	return asm volatile (
10048	1960	\\ cbz w16, 8f
10049		\\ .inst 0x48207c82 + 0x400000
10050		\\
	1961	\\ .inst 0x38208020 + 0xc0000000 + 0xc00000
10051	1962	\\ cbz wzr, 1f
10052	1963	\\8:
10053	1964	\\ mov x16, x0
10054		\\ mov x17, x1
10055	1965	\\0:
10056		\\ ldaxp x0, x1, [x4]
10057		\\ cmp x0, x16
10058		\\ ccmp x1, x17, #0, eq
10059		\\ bne 1f
10060		\\ stxp w15, x2, x3, [x4]
10061		\\ cbnz w15, 0b
	1966	\\ ldaxr x0, [x1]
	1967	\\ stlxr w17, x16, [x1]
	1968	\\ cbnz w17, 0b
10062	1969	\\1:
10063		: [ret] "={x0}" (-> u128),
10064		: [expected] "{x0}" (expected),
10065		[desired] "{x1}" (desired),
10066		[ptr] "{x2}" (ptr),
	1970	: [ret] "={x0}" (-> u64),
	1971	: [val] "{x0}" (val),
	1972	[ptr] "{x1}" (ptr),
10067	1973	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10068	1974	: "w15", "w16", "w17", "memory"
10069	1975	);
10070	1976	}
10071		const __aarch64_cas16_acq = if (is_darwin)
10072		if (has_lse)
10073		cas16AcqDarwinLse
10074		else
10075		cas16AcqDarwinNolse
10076		else if (has_lse)
10077		cas16AcqNondarwinLse
10078		else
10079		cas16AcqNondarwinNolse;
10080		fn cas16RelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1977	fn __aarch64_ldadd8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
10081	1978	@setRuntimeSafety(false);
10082		__init_aarch64_have_lse_atomics();
10083
10084	1979	return asm volatile (
10085	1980	\\ cbz w16, 8f
10086		\\ caspl x0, x1, x2, x3, [x4]
10087		\\
	1981	\\ .inst 0x38200020 + 0x0000 + 0xc0000000 + 0xc00000
10088	1982	\\ cbz wzr, 1f
10089	1983	\\8:
10090	1984	\\ mov x16, x0
10091		\\ mov x17, x1
10092	1985	\\0:
10093		\\ ldxp x0, x1, [x4]
10094		\\ cmp x0, x16
10095		\\ ccmp x1, x17, #0, eq
10096		\\ bne 1f
10097		\\ stlxp w15, x2, x3, [x4]
	1986	\\ ldaxr x0, [x1]
	1987	\\ add x17, x0, x16
	1988	\\ stlxr w15, x17, [x1]
10098	1989	\\ cbnz w15, 0b
10099	1990	\\1:
10100		: [ret] "={x0}" (-> u128),
10101		: [expected] "{x0}" (expected),
10102		[desired] "{x1}" (desired),
10103		[ptr] "{x2}" (ptr),
	1991	: [ret] "={x0}" (-> u64),
	1992	: [val] "{x0}" (val),
	1993	[ptr] "{x1}" (ptr),
10104	1994	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10105	1995	: "w15", "w16", "w17", "memory"
10106	1996	);
10107	1997	}
10108		fn cas16RelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	1998	fn __aarch64_ldclr8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
10109	1999	@setRuntimeSafety(false);
10110		__init_aarch64_have_lse_atomics();
10111
10112	2000	return asm volatile (
10113	2001	\\ cbz w16, 8f
10114		\\ .inst 0x48207c82 + 0x008000
10115		\\
	2002	\\ .inst 0x38200020 + 0x1000 + 0xc0000000 + 0xc00000
10116	2003	\\ cbz wzr, 1f
10117	2004	\\8:
10118	2005	\\ mov x16, x0
10119		\\ mov x17, x1
10120	2006	\\0:
10121		\\ ldxp x0, x1, [x4]
10122		\\ cmp x0, x16
10123		\\ ccmp x1, x17, #0, eq
10124		\\ bne 1f
10125		\\ stlxp w15, x2, x3, [x4]
	2007	\\ ldaxr x0, [x1]
	2008	\\ bic x17, x0, x16
	2009	\\ stlxr w15, x17, [x1]
10126	2010	\\ cbnz w15, 0b
10127	2011	\\1:
10128		: [ret] "={x0}" (-> u128),
10129		: [expected] "{x0}" (expected),
10130		[desired] "{x1}" (desired),
10131		[ptr] "{x2}" (ptr),
	2012	: [ret] "={x0}" (-> u64),
	2013	: [val] "{x0}" (val),
	2014	[ptr] "{x1}" (ptr),
10132	2015	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10133	2016	: "w15", "w16", "w17", "memory"
10134	2017	);
10135	2018	}
10136		fn cas16RelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2019	fn __aarch64_ldeor8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
10137	2020	@setRuntimeSafety(false);
10138		__init_aarch64_have_lse_atomics();
10139
10140	2021	return asm volatile (
10141	2022	\\ cbz w16, 8f
10142		\\ caspl x0, x1, x2, x3, [x4]
10143		\\
	2023	\\ .inst 0x38200020 + 0x2000 + 0xc0000000 + 0xc00000
10144	2024	\\ cbz wzr, 1f
10145	2025	\\8:
10146	2026	\\ mov x16, x0
10147		\\ mov x17, x1
10148	2027	\\0:
10149		\\ ldxp x0, x1, [x4]
10150		\\ cmp x0, x16
10151		\\ ccmp x1, x17, #0, eq
10152		\\ bne 1f
10153		\\ stlxp w15, x2, x3, [x4]
	2028	\\ ldaxr x0, [x1]
	2029	\\ eor x17, x0, x16
	2030	\\ stlxr w15, x17, [x1]
10154	2031	\\ cbnz w15, 0b
10155	2032	\\1:
10156		: [ret] "={x0}" (-> u128),
10157		: [expected] "{x0}" (expected),
10158		[desired] "{x1}" (desired),
10159		[ptr] "{x2}" (ptr),
	2033	: [ret] "={x0}" (-> u64),
	2034	: [val] "{x0}" (val),
	2035	[ptr] "{x1}" (ptr),
10160	2036	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10161	2037	: "w15", "w16", "w17", "memory"
10162	2038	);
10163	2039	}
10164		fn cas16RelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2040	fn __aarch64_ldset8_acq_rel(val: u64, ptr: *u64) callconv(.C) u64 {
10165	2041	@setRuntimeSafety(false);
10166		__init_aarch64_have_lse_atomics();
10167
10168	2042	return asm volatile (
10169	2043	\\ cbz w16, 8f
10170		\\ .inst 0x48207c82 + 0x008000
10171		\\
	2044	\\ .inst 0x38200020 + 0x3000 + 0xc0000000 + 0xc00000
10172	2045	\\ cbz wzr, 1f
10173	2046	\\8:
10174	2047	\\ mov x16, x0
10175		\\ mov x17, x1
10176	2048	\\0:
10177		\\ ldxp x0, x1, [x4]
10178		\\ cmp x0, x16
10179		\\ ccmp x1, x17, #0, eq
10180		\\ bne 1f
10181		\\ stlxp w15, x2, x3, [x4]
	2049	\\ ldaxr x0, [x1]
	2050	\\ orr x17, x0, x16
	2051	\\ stlxr w15, x17, [x1]
10182	2052	\\ cbnz w15, 0b
10183	2053	\\1:
10184		: [ret] "={x0}" (-> u128),
10185		: [expected] "{x0}" (expected),
10186		[desired] "{x1}" (desired),
10187		[ptr] "{x2}" (ptr),
	2054	: [ret] "={x0}" (-> u64),
	2055	: [val] "{x0}" (val),
	2056	[ptr] "{x1}" (ptr),
10188	2057	[__aarch64_have_lse_atomics] "{w16}" (__aarch64_have_lse_atomics),
10189	2058	: "w15", "w16", "w17", "memory"
10190	2059	);
10191	2060	}
10192		const __aarch64_cas16_rel = if (is_darwin)
10193		if (has_lse)
10194		cas16RelDarwinLse
10195		else
10196		cas16RelDarwinNolse
10197		else if (has_lse)
10198		cas16RelNondarwinLse
10199		else
10200		cas16RelNondarwinNolse;
10201		fn cas16AcqRelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2061	fn __aarch64_cas16_relax(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10202	2062	@setRuntimeSafety(false);
10203		__init_aarch64_have_lse_atomics();
10204
10205	2063	return asm volatile (
10206	2064	\\ cbz w16, 8f
10207		\\ caspal x0, x1, x2, x3, [x4]
	2065	\\ .inst 0x48207c82 + 0x000000
10208	2066	\\
10209	2067	\\ cbz wzr, 1f
10210	2068	\\8:
10211	2069	\\ mov x16, x0
10212	2070	\\ mov x17, x1
10213	2071	\\0:
10214		\\ ldaxp x0, x1, [x4]
	2072	\\ ldxp x0, x1, [x4]
10215	2073	\\ cmp x0, x16
10216	2074	\\ ccmp x1, x17, #0, eq
10217	2075	\\ bne 1f
10218		\\ stlxp w15, x2, x3, [x4]
	2076	\\ stxp w15, x2, x3, [x4]
10219	2077	\\ cbnz w15, 0b
10220	2078	\\1:
10221	2079	: [ret] "={x0}" (-> u128),
...	...	@@ -10226,13 +2084,11 @@ fn cas16AcqRelDarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C)
10226	2084	: "w15", "w16", "w17", "memory"
10227	2085	);
10228	2086	}
10229		fn cas16AcqRelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2087	fn __aarch64_cas16_acq(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10230	2088	@setRuntimeSafety(false);
10231		__init_aarch64_have_lse_atomics();
10232
10233	2089	return asm volatile (
10234	2090	\\ cbz w16, 8f
10235		\\ .inst 0x48207c82 + 0x408000
	2091	\\ .inst 0x48207c82 + 0x400000
10236	2092	\\
10237	2093	\\ cbz wzr, 1f
10238	2094	\\8:
...	...	@@ -10243,7 +2099,7 @@ fn cas16AcqRelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C
10243	2099	\\ cmp x0, x16
10244	2100	\\ ccmp x1, x17, #0, eq
10245	2101	\\ bne 1f
10246		\\ stlxp w15, x2, x3, [x4]
	2102	\\ stxp w15, x2, x3, [x4]
10247	2103	\\ cbnz w15, 0b
10248	2104	\\1:
10249	2105	: [ret] "={x0}" (-> u128),
...	...	@@ -10254,20 +2110,18 @@ fn cas16AcqRelDarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C
10254	2110	: "w15", "w16", "w17", "memory"
10255	2111	);
10256	2112	}
10257		fn cas16AcqRelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2113	fn __aarch64_cas16_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10258	2114	@setRuntimeSafety(false);
10259		__init_aarch64_have_lse_atomics();
10260
10261	2115	return asm volatile (
10262	2116	\\ cbz w16, 8f
10263		\\ caspal x0, x1, x2, x3, [x4]
	2117	\\ .inst 0x48207c82 + 0x008000
10264	2118	\\
10265	2119	\\ cbz wzr, 1f
10266	2120	\\8:
10267	2121	\\ mov x16, x0
10268	2122	\\ mov x17, x1
10269	2123	\\0:
10270		\\ ldaxp x0, x1, [x4]
	2124	\\ ldxp x0, x1, [x4]
10271	2125	\\ cmp x0, x16
10272	2126	\\ ccmp x1, x17, #0, eq
10273	2127	\\ bne 1f
...	...	@@ -10282,10 +2136,8 @@ fn cas16AcqRelNondarwinLse(expected: u128, desired: u128, ptr: *u128) callconv(.
10282	2136	: "w15", "w16", "w17", "memory"
10283	2137	);
10284	2138	}
10285		fn cas16AcqRelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
	2139	fn __aarch64_cas16_acq_rel(expected: u128, desired: u128, ptr: *u128) callconv(.C) u128 {
10286	2140	@setRuntimeSafety(false);
10287		__init_aarch64_have_lse_atomics();
10288
10289	2141	return asm volatile (
10290	2142	\\ cbz w16, 8f
10291	2143	\\ .inst 0x48207c82 + 0x408000
...	...	@@ -10310,126 +2162,106 @@ fn cas16AcqRelNondarwinNolse(expected: u128, desired: u128, ptr: *u128) callconv
10310	2162	: "w15", "w16", "w17", "memory"
10311	2163	);
10312	2164	}
10313		const __aarch64_cas16_acq_rel = if (is_darwin)
10314		if (has_lse)
10315		cas16AcqRelDarwinLse
10316		else
10317		cas16AcqRelDarwinNolse
10318		else if (has_lse)
10319		cas16AcqRelNondarwinLse
10320		else
10321		cas16AcqRelNondarwinNolse;
10322		//TODO: Add linksection once implemented and remove init at writeFunction
10323		fn __init_aarch64_have_lse_atomics() callconv(.C) void {
10324		const AT_HWCAP = 16;
10325		const HWCAP_ATOMICS = 1 << 8;
10326		const hwcap = std.os.linux.getauxval(AT_HWCAP);
10327		__aarch64_have_lse_atomics = @boolToInt((hwcap & HWCAP_ATOMICS) != 0);
10328		}
10329
10330		var __aarch64_have_lse_atomics: u8 = @boolToInt(has_lse);
10331	2165
10332	2166	comptime {
10333		if (arch.isAARCH64()) {
10334		@export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage });
10335		@export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage });
10336		@export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage });
10337		@export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage });
10338		@export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage });
10339		@export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage });
10340		@export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage });
10341		@export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage });
10342		@export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage });
10343		@export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage });
10344		@export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage });
10345		@export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage });
10346		@export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage });
10347		@export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage });
10348		@export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage });
10349		@export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage });
10350		@export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage });
10351		@export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage });
10352		@export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage });
10353		@export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage });
10354		@export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage });
10355		@export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage });
10356		@export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage });
10357		@export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage });
10358		@export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage });
10359		@export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage });
10360		@export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage });
10361		@export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage });
10362		@export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage });
10363		@export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage });
10364		@export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage });
10365		@export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage });
10366		@export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage });
10367		@export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage });
10368		@export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage });
10369		@export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage });
10370		@export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage });
10371		@export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage });
10372		@export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage });
10373		@export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage });
10374		@export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage });
10375		@export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage });
10376		@export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage });
10377		@export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage });
10378		@export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage });
10379		@export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage });
10380		@export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage });
10381		@export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage });
10382		@export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage });
10383		@export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage });
10384		@export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage });
10385		@export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage });
10386		@export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage });
10387		@export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage });
10388		@export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage });
10389		@export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage });
10390		@export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage });
10391		@export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage });
10392		@export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage });
10393		@export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage });
10394		@export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage });
10395		@export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage });
10396		@export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage });
10397		@export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage });
10398		@export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage });
10399		@export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage });
10400		@export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage });
10401		@export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage });
10402		@export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage });
10403		@export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage });
10404		@export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage });
10405		@export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage });
10406		@export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage });
10407		@export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage });
10408		@export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage });
10409		@export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage });
10410		@export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage });
10411		@export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage });
10412		@export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage });
10413		@export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage });
10414		@export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage });
10415		@export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage });
10416		@export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage });
10417		@export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage });
10418		@export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage });
10419		@export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage });
10420		@export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage });
10421		@export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage });
10422		@export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage });
10423		@export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage });
10424		@export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage });
10425		@export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage });
10426		@export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage });
10427		@export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage });
10428		@export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage });
10429		@export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage });
10430		@export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage });
10431		@export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage });
10432		@export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage });
10433		@export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage });
10434		}
	2167	@export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage });
	2168	@export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage });
	2169	@export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage });
	2170	@export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage });
	2171	@export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage });
	2172	@export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage });
	2173	@export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage });
	2174	@export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage });
	2175	@export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage });
	2176	@export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage });
	2177	@export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage });
	2178	@export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage });
	2179	@export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage });
	2180	@export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage });
	2181	@export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage });
	2182	@export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage });
	2183	@export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage });
	2184	@export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage });
	2185	@export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage });
	2186	@export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage });
	2187	@export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage });
	2188	@export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage });
	2189	@export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage });
	2190	@export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage });
	2191	@export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage });
	2192	@export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage });
	2193	@export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage });
	2194	@export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage });
	2195	@export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage });
	2196	@export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage });
	2197	@export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage });
	2198	@export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage });
	2199	@export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage });
	2200	@export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage });
	2201	@export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage });
	2202	@export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage });
	2203	@export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage });
	2204	@export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage });
	2205	@export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage });
	2206	@export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage });
	2207	@export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage });
	2208	@export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage });
	2209	@export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage });
	2210	@export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage });
	2211	@export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage });
	2212	@export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage });
	2213	@export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage });
	2214	@export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage });
	2215	@export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage });
	2216	@export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage });
	2217	@export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage });
	2218	@export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage });
	2219	@export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage });
	2220	@export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage });
	2221	@export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage });
	2222	@export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage });
	2223	@export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage });
	2224	@export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage });
	2225	@export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage });
	2226	@export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage });
	2227	@export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage });
	2228	@export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage });
	2229	@export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage });
	2230	@export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage });
	2231	@export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage });
	2232	@export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage });
	2233	@export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage });
	2234	@export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage });
	2235	@export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage });
	2236	@export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage });
	2237	@export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage });
	2238	@export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage });
	2239	@export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage });
	2240	@export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage });
	2241	@export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage });
	2242	@export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage });
	2243	@export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage });
	2244	@export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage });
	2245	@export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage });
	2246	@export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage });
	2247	@export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage });
	2248	@export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage });
	2249	@export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage });
	2250	@export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage });
	2251	@export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage });
	2252	@export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage });
	2253	@export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage });
	2254	@export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage });
	2255	@export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage });
	2256	@export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage });
	2257	@export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage });
	2258	@export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage });
	2259	@export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage });
	2260	@export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage });
	2261	@export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage });
	2262	@export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage });
	2263	@export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage });
	2264	@export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage });
	2265	@export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage });
	2266	@export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage });
10435	2267	}

tools/gen_outline_atomics.zig+113-342

...	...	@@ -11,217 +11,97 @@ const AtomicOp = enum {
11	11	};
12	12
13	13	pub fn main() !void {
14		var gpa = std.heap.GeneralPurposeAllocator(.{}){};
15		defer std.debug.assert(!gpa.deinit());
16		var allocator = gpa.allocator();
	14	var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator);
	15	defer arena_instance.deinit();
	16	const arena = arena_instance.allocator();
17	17
18		const args = try std.process.argsAlloc(allocator);
19		defer std.process.argsFree(allocator, args);
20		if (args.len != 2) {
21		usageAndExit(std.io.getStdErr(), args[0], 1);
22		}
	18	//const args = try std.process.argsAlloc(arena);
23	19
24		var file = try std.fs.cwd().createFile(args[1], .{ .truncate = true });
	20	var bw = std.io.bufferedWriter(std.io.getStdOut().writer());
	21	const w = bw.writer();
25	22
26		try file.writeAll(
27		\\const std = @import("std");
	23	try w.writeAll(
	24	\\//! This file is generated by tools/gen_outline_atomics.zig.
28	25	\\const builtin = @import("builtin");
29		\\const arch = builtin.cpu.arch;
30		\\const is_test = builtin.is_test;
31		\\const target = std.Target;
32		\\const os_tag = builtin.os.tag;
33		\\const is_darwin = target.Os.Tag.isDarwin(os_tag);
34		\\const has_lse = target.aarch64.featureSetHas(builtin.target.cpu.features, .lse);
35		\\const linkage = if (is_test)
36		\\ std.builtin.GlobalLinkage.Internal
37		\\else
38		\\ std.builtin.GlobalLinkage.Strong;
	26	\\const std = @import("std");
	27	\\const linkage = @import("./common.zig").linkage;
	28	\\const always_has_lse = std.Target.aarch64.featureSetHas(builtin.cpu.features, .lse);
	29	\\
	30	\\/// This default is overridden at runtime after inspecting CPU properties.
	31	\\/// It is intentionally not exported in order to make the machine code that
	32	\\/// uses it a statically predicted direct branch rather than using the PLT,
	33	\\/// which ARM is concerned would have too much overhead.
	34	\\var __aarch64_have_lse_atomics: u8 = @boolToInt(always_has_lse);
39	35	\\
40	36	\\
41	37	);
42	38
	39	var footer = std.ArrayList(u8).init(arena);
	40	try footer.appendSlice("\ncomptime {\n");
	41
43	42	for ([_]N{ .one, .two, .four, .eight, .sixteen }) \|n\| {
44	43	for ([_]Ordering{ .relax, .acq, .rel, .acq_rel }) \|order\| {
45		for ([_]AtomicOp{ .cas, .swp, .ldadd, .ldclr, .ldeor, .ldset }) \|pat\| {
46		if (pat == .cas or n != .sixteen) {
47		for ([_]bool{ true, false }) \|darwin\| {
48		for ([_]bool{ true, false }) \|lse\| {
49		const darwin_name = if (darwin) "Darwin" else "Nondarwin";
50		const lse_name = if (lse) "Lse" else "Nolse";
51		var buf: [100:0]u8 = undefined;
52		const name = try std.fmt.bufPrintZ(&buf, "{s}{s}{s}{s}{s}", .{ @tagName(pat), n.toBytes(), order.capName(), darwin_name, lse_name });
53		const body = switch (pat) {
54		.cas => try generateCas(&allocator, n, order, lse),
55		.swp => try generateSwp(&allocator, n, order, lse),
56		.ldadd => try generateLd(&allocator, n, order, .ldadd, lse),
57		.ldclr => try generateLd(&allocator, n, order, .ldclr, lse),
58		.ldeor => try generateLd(&allocator, n, order, .ldeor, lse),
59		.ldset => try generateLd(&allocator, n, order, .ldset, lse),
60		};
61		defer allocator.destroy(body.ptr);
62		try writeFunction(&file, name, pat, n, body);
63		}
64		}
65		try writeExport(&file, @tagName(pat), n.toBytes(), order);
66		}
	44	for ([_]AtomicOp{ .cas, .swp, .ldadd, .ldclr, .ldeor, .ldset }) \|op\| {
	45	if (n == .sixteen and op != .cas) continue;
	46
	47	const name = try std.fmt.allocPrint(arena, "__aarch64_{s}{d}_{s}", .{
	48	@tagName(op), n.toBytes(), @tagName(order),
	49	});
	50	try writeFunction(arena, w, name, op, n, order);
	51	try footer.writer().print(" @export({s}, .{{ .name = \"{s}\", .linkage = linkage }});\n", .{
	52	name, name,
	53	});
67	54	}
68	55	}
69	56	}
70	57
71		try file.writeAll(
72		\\//TODO: Add linksection once implemented and remove init at writeFunction
73		\\fn __init_aarch64_have_lse_atomics() callconv(.C) void {
74		\\ const AT_HWCAP = 16;
75		\\ const HWCAP_ATOMICS = 1 << 8;
76		\\ const hwcap = std.os.linux.getauxval(AT_HWCAP);
77		\\ __aarch64_have_lse_atomics = @boolToInt((hwcap & HWCAP_ATOMICS) != 0);
78		\\}
79		\\
80		\\var __aarch64_have_lse_atomics: u8 = @boolToInt(has_lse);
81		\\
82		\\comptime {
83		\\ if (arch.isAARCH64()) {
84		\\ @export(__aarch64_cas1_relax, .{ .name = "__aarch64_cas1_relax", .linkage = linkage });
85		\\ @export(__aarch64_cas1_acq, .{ .name = "__aarch64_cas1_acq", .linkage = linkage });
86		\\ @export(__aarch64_cas1_rel, .{ .name = "__aarch64_cas1_rel", .linkage = linkage });
87		\\ @export(__aarch64_cas1_acq_rel, .{ .name = "__aarch64_cas1_acq_rel", .linkage = linkage });
88		\\ @export(__aarch64_cas2_relax, .{ .name = "__aarch64_cas2_relax", .linkage = linkage });
89		\\ @export(__aarch64_cas2_acq, .{ .name = "__aarch64_cas2_acq", .linkage = linkage });
90		\\ @export(__aarch64_cas2_rel, .{ .name = "__aarch64_cas2_rel", .linkage = linkage });
91		\\ @export(__aarch64_cas2_acq_rel, .{ .name = "__aarch64_cas2_acq_rel", .linkage = linkage });
92		\\ @export(__aarch64_cas4_relax, .{ .name = "__aarch64_cas4_relax", .linkage = linkage });
93		\\ @export(__aarch64_cas4_acq, .{ .name = "__aarch64_cas4_acq", .linkage = linkage });
94		\\ @export(__aarch64_cas4_rel, .{ .name = "__aarch64_cas4_rel", .linkage = linkage });
95		\\ @export(__aarch64_cas4_acq_rel, .{ .name = "__aarch64_cas4_acq_rel", .linkage = linkage });
96		\\ @export(__aarch64_cas8_relax, .{ .name = "__aarch64_cas8_relax", .linkage = linkage });
97		\\ @export(__aarch64_cas8_acq, .{ .name = "__aarch64_cas8_acq", .linkage = linkage });
98		\\ @export(__aarch64_cas8_rel, .{ .name = "__aarch64_cas8_rel", .linkage = linkage });
99		\\ @export(__aarch64_cas8_acq_rel, .{ .name = "__aarch64_cas8_acq_rel", .linkage = linkage });
100		\\ @export(__aarch64_cas16_relax, .{ .name = "__aarch64_cas16_relax", .linkage = linkage });
101		\\ @export(__aarch64_cas16_acq, .{ .name = "__aarch64_cas16_acq", .linkage = linkage });
102		\\ @export(__aarch64_cas16_rel, .{ .name = "__aarch64_cas16_rel", .linkage = linkage });
103		\\ @export(__aarch64_cas16_acq_rel, .{ .name = "__aarch64_cas16_acq_rel", .linkage = linkage });
104		\\ @export(__aarch64_swp1_relax, .{ .name = "__aarch64_swp1_relax", .linkage = linkage });
105		\\ @export(__aarch64_swp1_acq, .{ .name = "__aarch64_swp1_acq", .linkage = linkage });
106		\\ @export(__aarch64_swp1_rel, .{ .name = "__aarch64_swp1_rel", .linkage = linkage });
107		\\ @export(__aarch64_swp1_acq_rel, .{ .name = "__aarch64_swp1_acq_rel", .linkage = linkage });
108		\\ @export(__aarch64_swp2_relax, .{ .name = "__aarch64_swp2_relax", .linkage = linkage });
109		\\ @export(__aarch64_swp2_acq, .{ .name = "__aarch64_swp2_acq", .linkage = linkage });
110		\\ @export(__aarch64_swp2_rel, .{ .name = "__aarch64_swp2_rel", .linkage = linkage });
111		\\ @export(__aarch64_swp2_acq_rel, .{ .name = "__aarch64_swp2_acq_rel", .linkage = linkage });
112		\\ @export(__aarch64_swp4_relax, .{ .name = "__aarch64_swp4_relax", .linkage = linkage });
113		\\ @export(__aarch64_swp4_acq, .{ .name = "__aarch64_swp4_acq", .linkage = linkage });
114		\\ @export(__aarch64_swp4_rel, .{ .name = "__aarch64_swp4_rel", .linkage = linkage });
115		\\ @export(__aarch64_swp4_acq_rel, .{ .name = "__aarch64_swp4_acq_rel", .linkage = linkage });
116		\\ @export(__aarch64_swp8_relax, .{ .name = "__aarch64_swp8_relax", .linkage = linkage });
117		\\ @export(__aarch64_swp8_acq, .{ .name = "__aarch64_swp8_acq", .linkage = linkage });
118		\\ @export(__aarch64_swp8_rel, .{ .name = "__aarch64_swp8_rel", .linkage = linkage });
119		\\ @export(__aarch64_swp8_acq_rel, .{ .name = "__aarch64_swp8_acq_rel", .linkage = linkage });
120		\\ @export(__aarch64_ldadd1_relax, .{ .name = "__aarch64_ldadd1_relax", .linkage = linkage });
121		\\ @export(__aarch64_ldadd1_acq, .{ .name = "__aarch64_ldadd1_acq", .linkage = linkage });
122		\\ @export(__aarch64_ldadd1_rel, .{ .name = "__aarch64_ldadd1_rel", .linkage = linkage });
123		\\ @export(__aarch64_ldadd1_acq_rel, .{ .name = "__aarch64_ldadd1_acq_rel", .linkage = linkage });
124		\\ @export(__aarch64_ldadd2_relax, .{ .name = "__aarch64_ldadd2_relax", .linkage = linkage });
125		\\ @export(__aarch64_ldadd2_acq, .{ .name = "__aarch64_ldadd2_acq", .linkage = linkage });
126		\\ @export(__aarch64_ldadd2_rel, .{ .name = "__aarch64_ldadd2_rel", .linkage = linkage });
127		\\ @export(__aarch64_ldadd2_acq_rel, .{ .name = "__aarch64_ldadd2_acq_rel", .linkage = linkage });
128		\\ @export(__aarch64_ldadd4_relax, .{ .name = "__aarch64_ldadd4_relax", .linkage = linkage });
129		\\ @export(__aarch64_ldadd4_acq, .{ .name = "__aarch64_ldadd4_acq", .linkage = linkage });
130		\\ @export(__aarch64_ldadd4_rel, .{ .name = "__aarch64_ldadd4_rel", .linkage = linkage });
131		\\ @export(__aarch64_ldadd4_acq_rel, .{ .name = "__aarch64_ldadd4_acq_rel", .linkage = linkage });
132		\\ @export(__aarch64_ldadd8_relax, .{ .name = "__aarch64_ldadd8_relax", .linkage = linkage });
133		\\ @export(__aarch64_ldadd8_acq, .{ .name = "__aarch64_ldadd8_acq", .linkage = linkage });
134		\\ @export(__aarch64_ldadd8_rel, .{ .name = "__aarch64_ldadd8_rel", .linkage = linkage });
135		\\ @export(__aarch64_ldadd8_acq_rel, .{ .name = "__aarch64_ldadd8_acq_rel", .linkage = linkage });
136		\\ @export(__aarch64_ldclr1_relax, .{ .name = "__aarch64_ldclr1_relax", .linkage = linkage });
137		\\ @export(__aarch64_ldclr1_acq, .{ .name = "__aarch64_ldclr1_acq", .linkage = linkage });
138		\\ @export(__aarch64_ldclr1_rel, .{ .name = "__aarch64_ldclr1_rel", .linkage = linkage });
139		\\ @export(__aarch64_ldclr1_acq_rel, .{ .name = "__aarch64_ldclr1_acq_rel", .linkage = linkage });
140		\\ @export(__aarch64_ldclr2_relax, .{ .name = "__aarch64_ldclr2_relax", .linkage = linkage });
141		\\ @export(__aarch64_ldclr2_acq, .{ .name = "__aarch64_ldclr2_acq", .linkage = linkage });
142		\\ @export(__aarch64_ldclr2_rel, .{ .name = "__aarch64_ldclr2_rel", .linkage = linkage });
143		\\ @export(__aarch64_ldclr2_acq_rel, .{ .name = "__aarch64_ldclr2_acq_rel", .linkage = linkage });
144		\\ @export(__aarch64_ldclr4_relax, .{ .name = "__aarch64_ldclr4_relax", .linkage = linkage });
145		\\ @export(__aarch64_ldclr4_acq, .{ .name = "__aarch64_ldclr4_acq", .linkage = linkage });
146		\\ @export(__aarch64_ldclr4_rel, .{ .name = "__aarch64_ldclr4_rel", .linkage = linkage });
147		\\ @export(__aarch64_ldclr4_acq_rel, .{ .name = "__aarch64_ldclr4_acq_rel", .linkage = linkage });
148		\\ @export(__aarch64_ldclr8_relax, .{ .name = "__aarch64_ldclr8_relax", .linkage = linkage });
149		\\ @export(__aarch64_ldclr8_acq, .{ .name = "__aarch64_ldclr8_acq", .linkage = linkage });
150		\\ @export(__aarch64_ldclr8_rel, .{ .name = "__aarch64_ldclr8_rel", .linkage = linkage });
151		\\ @export(__aarch64_ldclr8_acq_rel, .{ .name = "__aarch64_ldclr8_acq_rel", .linkage = linkage });
152		\\ @export(__aarch64_ldeor1_relax, .{ .name = "__aarch64_ldeor1_relax", .linkage = linkage });
153		\\ @export(__aarch64_ldeor1_acq, .{ .name = "__aarch64_ldeor1_acq", .linkage = linkage });
154		\\ @export(__aarch64_ldeor1_rel, .{ .name = "__aarch64_ldeor1_rel", .linkage = linkage });
155		\\ @export(__aarch64_ldeor1_acq_rel, .{ .name = "__aarch64_ldeor1_acq_rel", .linkage = linkage });
156		\\ @export(__aarch64_ldeor2_relax, .{ .name = "__aarch64_ldeor2_relax", .linkage = linkage });
157		\\ @export(__aarch64_ldeor2_acq, .{ .name = "__aarch64_ldeor2_acq", .linkage = linkage });
158		\\ @export(__aarch64_ldeor2_rel, .{ .name = "__aarch64_ldeor2_rel", .linkage = linkage });
159		\\ @export(__aarch64_ldeor2_acq_rel, .{ .name = "__aarch64_ldeor2_acq_rel", .linkage = linkage });
160		\\ @export(__aarch64_ldeor4_relax, .{ .name = "__aarch64_ldeor4_relax", .linkage = linkage });
161		\\ @export(__aarch64_ldeor4_acq, .{ .name = "__aarch64_ldeor4_acq", .linkage = linkage });
162		\\ @export(__aarch64_ldeor4_rel, .{ .name = "__aarch64_ldeor4_rel", .linkage = linkage });
163		\\ @export(__aarch64_ldeor4_acq_rel, .{ .name = "__aarch64_ldeor4_acq_rel", .linkage = linkage });
164		\\ @export(__aarch64_ldeor8_relax, .{ .name = "__aarch64_ldeor8_relax", .linkage = linkage });
165		\\ @export(__aarch64_ldeor8_acq, .{ .name = "__aarch64_ldeor8_acq", .linkage = linkage });
166		\\ @export(__aarch64_ldeor8_rel, .{ .name = "__aarch64_ldeor8_rel", .linkage = linkage });
167		\\ @export(__aarch64_ldeor8_acq_rel, .{ .name = "__aarch64_ldeor8_acq_rel", .linkage = linkage });
168		\\ @export(__aarch64_ldset1_relax, .{ .name = "__aarch64_ldset1_relax", .linkage = linkage });
169		\\ @export(__aarch64_ldset1_acq, .{ .name = "__aarch64_ldset1_acq", .linkage = linkage });
170		\\ @export(__aarch64_ldset1_rel, .{ .name = "__aarch64_ldset1_rel", .linkage = linkage });
171		\\ @export(__aarch64_ldset1_acq_rel, .{ .name = "__aarch64_ldset1_acq_rel", .linkage = linkage });
172		\\ @export(__aarch64_ldset2_relax, .{ .name = "__aarch64_ldset2_relax", .linkage = linkage });
173		\\ @export(__aarch64_ldset2_acq, .{ .name = "__aarch64_ldset2_acq", .linkage = linkage });
174		\\ @export(__aarch64_ldset2_rel, .{ .name = "__aarch64_ldset2_rel", .linkage = linkage });
175		\\ @export(__aarch64_ldset2_acq_rel, .{ .name = "__aarch64_ldset2_acq_rel", .linkage = linkage });
176		\\ @export(__aarch64_ldset4_relax, .{ .name = "__aarch64_ldset4_relax", .linkage = linkage });
177		\\ @export(__aarch64_ldset4_acq, .{ .name = "__aarch64_ldset4_acq", .linkage = linkage });
178		\\ @export(__aarch64_ldset4_rel, .{ .name = "__aarch64_ldset4_rel", .linkage = linkage });
179		\\ @export(__aarch64_ldset4_acq_rel, .{ .name = "__aarch64_ldset4_acq_rel", .linkage = linkage });
180		\\ @export(__aarch64_ldset8_relax, .{ .name = "__aarch64_ldset8_relax", .linkage = linkage });
181		\\ @export(__aarch64_ldset8_acq, .{ .name = "__aarch64_ldset8_acq", .linkage = linkage });
182		\\ @export(__aarch64_ldset8_rel, .{ .name = "__aarch64_ldset8_rel", .linkage = linkage });
183		\\ @export(__aarch64_ldset8_acq_rel, .{ .name = "__aarch64_ldset8_acq_rel", .linkage = linkage });
184		\\ }
185		\\}
186		\\
187		);
	58	try w.writeAll(footer.items);
	59	try w.writeAll("}\n");
	60	try bw.flush();
188	61	}
189	62
190		fn usageAndExit(file: std.fs.File, arg0: []const u8, code: u8) noreturn {
191		file.writer().print(
192		\\Usage: {s} /path/to/lib/compiler_rt/lse_atomics.zig
193		\\
194		\\Generates outline atomics for compiler-rt.
195		\\
196		, .{arg0}) catch std.process.exit(1);
197		std.process.exit(code);
198		}
199
200		fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, body: [:0]const u8) !void {
201		var fn_buf: [100:0]u8 = undefined;
	63	fn writeFunction(
	64	arena: Allocator,
	65	w: anytype,
	66	name: []const u8,
	67	op: AtomicOp,
	68	n: N,
	69	order: Ordering,
	70	) !void {
	71	const body = switch (op) {
	72	.cas => try generateCas(arena, n, order),
	73	.swp => try generateSwp(arena, n, order),
	74	.ldadd => try generateLd(arena, n, order, .ldadd),
	75	.ldclr => try generateLd(arena, n, order, .ldclr),
	76	.ldeor => try generateLd(arena, n, order, .ldeor),
	77	.ldset => try generateLd(arena, n, order, .ldset),
	78	};
202	79	const fn_sig = if (op != .cas)
203		try std.fmt.bufPrintZ(&fn_buf, "fn {[name]s}(val: u{[n]s}, ptr: *u{[n]s}) callconv(.C) u{[n]s} {{", .{ .name = name, .n = n.toBits() })
	80	try std.fmt.allocPrint(arena, "fn {[name]s}(val: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{
	81	.name = name,
	82	.n = n.toBits(),
	83	})
204	84	else
205		try std.fmt.bufPrintZ(&fn_buf, "fn {[name]s}(expected: u{[n]s}, desired: u{[n]s}, ptr: *u{[n]s}) callconv(.C) u{[n]s} {{", .{ .name = name, .n = n.toBits() });
206		try file.writeAll(fn_sig);
207		try file.writeAll(
	85	try std.fmt.allocPrint(arena, "fn {[name]s}(expected: u{[n]d}, desired: u{[n]d}, ptr: *u{[n]d}) callconv(.C) u{[n]d} {{", .{
	86	.name = name,
	87	.n = n.toBits(),
	88	});
	89	try w.writeAll(fn_sig);
	90	try w.writeAll(
208	91	\\
209	92	\\ @setRuntimeSafety(false);
210		\\ __init_aarch64_have_lse_atomics();
211		\\
212	93	\\ return asm volatile (
213	94	\\
214	95	);
215	96	var iter = std.mem.split(u8, body, "\n");
216	97	while (iter.next()) \|line\| {
217		try file.writeAll(" \\\\");
218		try file.writeAll(line);
219		try file.writeAll("\n");
	98	try w.writeAll(" \\\\");
	99	try w.writeAll(line);
	100	try w.writeAll("\n");
220	101	}
221		var constraint_buf: [500:0]u8 = undefined;
222	102	const constraints = if (op != .cas)
223		try std.fmt.bufPrintZ(&constraint_buf,
224		\\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]s}),
	103	try std.fmt.allocPrint(arena,
	104	\\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}),
225	105	\\ : [val] "{{{[reg]s}0}}" (val),
226	106	\\ [ptr] "{{x1}}" (ptr),
227	107	\\ [__aarch64_have_lse_atomics] "{{w16}}" (__aarch64_have_lse_atomics),
...	...	@@ -229,8 +109,8 @@ fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, bod
229	109	\\
230	110	, .{ .reg = n.register(), .ty = n.toBits() })
231	111	else
232		try std.fmt.bufPrintZ(&constraint_buf,
233		\\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]s}),
	112	try std.fmt.allocPrint(arena,
	113	\\ : [ret] "={{{[reg]s}0}}" (-> u{[ty]d}),
234	114	\\ : [expected] "{{{[reg]s}0}}" (expected),
235	115	\\ [desired] "{{{[reg]s}1}}" (desired),
236	116	\\ [ptr] "{{x2}}" (ptr),
...	...	@@ -239,55 +119,12 @@ fn writeFunction(file: *std.fs.File, name: [:0]const u8, op: AtomicOp, n: N, bod
239	119	\\
240	120	, .{ .reg = n.register(), .ty = n.toBits() });
241	121
242		try file.writeAll(constraints);
243		try file.writeAll(
	122	try w.writeAll(constraints);
	123	try w.writeAll(
244	124	\\ );
245	125	\\
246	126	);
247		try file.writeAll("}\n");
248		}
249
250		fn writeExport(file: *std.fs.File, pat: [:0]const u8, n: [:0]const u8, order: Ordering) !void {
251		var darwin_lse_buf: [100:0]u8 = undefined;
252		var darwin_nolse_buf: [100:0]u8 = undefined;
253		var nodarwin_lse_buf: [100:0]u8 = undefined;
254		var nodarwin_nolse_buf: [100:0]u8 = undefined;
255		var name_buf: [100:0]u8 = undefined;
256		const darwin_lse = try std.fmt.bufPrintZ(&darwin_lse_buf, "{s}{s}{s}DarwinLse", .{ pat, n, order.capName() });
257		const darwin_nolse = try std.fmt.bufPrintZ(&darwin_nolse_buf, "{s}{s}{s}DarwinNolse", .{ pat, n, order.capName() });
258		const nodarwin_lse = try std.fmt.bufPrintZ(&nodarwin_lse_buf, "{s}{s}{s}NondarwinLse", .{ pat, n, order.capName() });
259		const nodarwin_nolse = try std.fmt.bufPrintZ(&nodarwin_nolse_buf, "{s}{s}{s}NondarwinNolse", .{ pat, n, order.capName() });
260		const name = try std.fmt.bufPrintZ(&name_buf, "__aarch64_{s}{s}_{s}", .{ pat, n, @tagName(order) });
261		try file.writeAll("const ");
262		try file.writeAll(name);
263		try file.writeAll(
264		\\ = if (is_darwin)
265		\\ if (has_lse)
266		\\
267		);
268		try file.writeAll(darwin_lse);
269		try file.writeAll(
270		\\
271		\\ else
272		\\
273		);
274		try file.writeAll(darwin_nolse);
275		try file.writeAll(
276		\\
277		\\else if (has_lse)
278		\\
279		);
280		try file.writeAll(nodarwin_lse);
281		try file.writeAll(
282		\\
283		\\else
284		\\
285		);
286		try file.writeAll(nodarwin_nolse);
287		try file.writeAll(
288		\\;
289		\\
290		);
	127	try w.writeAll("}\n");
291	128	}
292	129
293	130	const N = enum(u8) {
...	...	@@ -298,22 +135,23 @@ const N = enum(u8) {
298	135	sixteen = 16,
299	136
300	137	const Defines = struct {
301		s: [:0]const u8,
302		uxt: [:0]const u8,
303		b: [:0]const u8,
	138	s: []const u8,
	139	uxt: []const u8,
	140	b: []const u8,
304	141	};
305		fn defines(self: @This()) Defines {
306		const s = switch (self) {
	142
	143	fn defines(n: N) Defines {
	144	const s = switch (n) {
307	145	.one => "b",
308	146	.two => "h",
309	147	else => "",
310	148	};
311		const uxt = switch (self) {
	149	const uxt = switch (n) {
312	150	.one => "uxtb",
313	151	.two => "uxth",
314	152	.four, .eight, .sixteen => "mov",
315	153	};
316		const b = switch (self) {
	154	const b = switch (n) {
317	155	.one => "0x00000000",
318	156	.two => "0x40000000",
319	157	.four => "0x80000000",
...	...	@@ -327,28 +165,16 @@ const N = enum(u8) {
327	165	};
328	166	}
329	167
330		fn register(self: @This()) [:0]const u8 {
331		return if (@enumToInt(self) < 8) "w" else "x";
	168	fn register(n: N) []const u8 {
	169	return if (@enumToInt(n) < 8) "w" else "x";
332	170	}
333	171
334		fn toBytes(self: @This()) [:0]const u8 {
335		return switch (self) {
336		.one => "1",
337		.two => "2",
338		.four => "4",
339		.eight => "8",
340		.sixteen => "16",
341		};
	172	fn toBytes(n: N) u8 {
	173	return @enumToInt(n);
342	174	}
343	175
344		fn toBits(self: @This()) [:0]const u8 {
345		return switch (self) {
346		.one => "8",
347		.two => "16",
348		.four => "32",
349		.eight => "64",
350		.sixteen => "128",
351		};
	176	fn toBits(n: N) u8 {
	177	return n.toBytes() * 8;
352	178	}
353	179	};
354	180
...	...	@@ -411,31 +237,18 @@ const Ordering = enum {
411	237
412	238	const LdName = enum { ldadd, ldclr, ldeor, ldset };
413	239
414		fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u8 {
	240	fn generateCas(arena: Allocator, n: N, order: Ordering) ![]const u8 {
415	241	const s_def = n.defines();
416	242	const o_def = order.defines();
417		var cas_buf = try alloc.create([200:0]u8);
418		var ldxr_buf = try alloc.create([200:0]u8);
419		var stxr_buf = try alloc.create([200:0]u8);
420		defer alloc.destroy(cas_buf);
421		defer alloc.destroy(ldxr_buf);
422		defer alloc.destroy(stxr_buf);
423		var instr_buf = try alloc.create([1000:0]u8);
424		errdefer alloc.destroy(instr_buf);
425	243
426	244	const reg = n.register();
427	245
428	246	if (@enumToInt(n) < 16) {
429		const cas = if (lse) blk: {
430		break :blk try std.fmt.bufPrintZ(cas_buf,
431		\\cas{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}1, [x2]
432		\\
433		, .{ .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg });
434		} else try std.fmt.bufPrintZ(cas_buf, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m });
435		const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s });
436		const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s });
	247	const cas = try std.fmt.allocPrint(arena, ".inst 0x08a07c41 + {s} + {s}\n", .{ s_def.b, o_def.m });
	248	const ldxr = try std.fmt.allocPrint(arena, "ld{s}xr{s}", .{ o_def.a, s_def.s });
	249	const stxr = try std.fmt.allocPrint(arena, "st{s}xr{s}", .{ o_def.l, s_def.s });
437	250
438		return try std.fmt.bufPrintZ(instr_buf,
	251	return try std.fmt.allocPrint(arena,
439	252	\\ cbz w16, 8f
440	253	\\ {[cas]s}
441	254	\\ cbz wzr, 1f
...	...	@@ -456,15 +269,11 @@ fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u
456	269	.reg = reg,
457	270	});
458	271	} else {
459		const casp = if (lse)
460		try std.fmt.bufPrintZ(cas_buf, "casp{s}{s} x0, x1, x2, x3, [x4]\n", .{ o_def.a, o_def.l })
461		else
462		try std.fmt.bufPrintZ(cas_buf, ".inst 0x48207c82 + {s}\n", .{o_def.m});
463
464		const ldxp = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xp", .{o_def.a});
465		const stxp = try std.fmt.bufPrintZ(stxr_buf, "st{s}xp", .{o_def.l});
	272	const casp = try std.fmt.allocPrint(arena, ".inst 0x48207c82 + {s}\n", .{o_def.m});
	273	const ldxp = try std.fmt.allocPrint(arena, "ld{s}xp", .{o_def.a});
	274	const stxp = try std.fmt.allocPrint(arena, "st{s}xp", .{o_def.l});
466	275
467		return try std.fmt.bufPrintZ(instr_buf,
	276	return try std.fmt.allocPrint(arena,
468	277	\\ cbz w16, 8f
469	278	\\ {[casp]s}
470	279	\\ cbz wzr, 1f
...	...	@@ -487,53 +296,35 @@ fn generateCas(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u
487	296	}
488	297	}
489	298
490		fn generateSwp(alloc: *Allocator, n: N, order: Ordering, lse: bool) ![:0]const u8 {
	299	fn generateSwp(arena: Allocator, n: N, order: Ordering) ![]const u8 {
491	300	const s_def = n.defines();
492	301	const o_def = order.defines();
493
494		var swp_buf = try alloc.create([200:0]u8);
495		var ldxr_buf = try alloc.create([200:0]u8);
496		var stxr_buf = try alloc.create([200:0]u8);
497		defer alloc.destroy(swp_buf);
498		defer alloc.destroy(ldxr_buf);
499		defer alloc.destroy(stxr_buf);
500
501	302	const reg = n.register();
502	303
503		const swp = if (lse) blk: {
504		break :blk try std.fmt.bufPrintZ(swp_buf,
505		\\swp{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}0, [x1]
506		, .{ .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg });
507		} else std.fmt.bufPrintZ(swp_buf, ".inst 0x38208020 + {s} + {s}", .{ .b = s_def.b, .n = o_def.n });
508
509		const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s });
510		const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s });
511
512		var instr_buf = try alloc.create([1000:0]u8);
513		errdefer alloc.destroy(instr_buf);
514		return try std.fmt.bufPrintZ(instr_buf,
	304	return try std.fmt.allocPrint(arena,
515	305	\\ cbz w16, 8f
516		\\ {[swp]s}
	306	\\ .inst 0x38208020 + {[b]s} + {[n]s}
517	307	\\ cbz wzr, 1f
518	308	\\8:
519	309	\\ mov {[reg]s}16, {[reg]s}0
520	310	\\0:
521		\\ {[ldxr]s} {[reg]s}0, [x1]
522		\\ {[stxr]s} w17, {[reg]s}16, [x1]
	311	\\ ld{[a]s}xr{[s]s} {[reg]s}0, [x1]
	312	\\ st{[l]s}xr{[s]s} w17, {[reg]s}16, [x1]
523	313	\\ cbnz w17, 0b
524	314	\\1:
525	315	, .{
526		.swp = swp,
527		.ldxr = ldxr,
528		.stxr = stxr,
	316	.b = s_def.b,
	317	.n = o_def.n,
529	318	.reg = reg,
	319	.s = s_def.s,
	320	.a = o_def.a,
	321	.l = o_def.l,
530	322	});
531	323	}
532	324
533		fn generateLd(alloc: *Allocator, n: N, order: Ordering, ld: LdName, lse: bool) ![:0]const u8 {
	325	fn generateLd(arena: Allocator, n: N, order: Ordering, ld: LdName) ![]const u8 {
534	326	const s_def = n.defines();
535	327	const o_def = order.defines();
536		const ldname = @tagName(ld);
537	328	const op = switch (ld) {
538	329	.ldadd => "add",
539	330	.ldclr => "bic",
...	...	@@ -547,47 +338,27 @@ fn generateLd(alloc: *Allocator, n: N, order: Ordering, ld: LdName, lse: bool) !
547	338	.ldset => "0x3000",
548	339	};
549	340
550		var swp_buf = try alloc.create([200:0]u8);
551		var ldop_buf = try alloc.create([200:0]u8);
552		var ldxr_buf = try alloc.create([200:0]u8);
553		var stxr_buf = try alloc.create([200:0]u8);
554		defer alloc.destroy(swp_buf);
555		defer alloc.destroy(ldop_buf);
556		defer alloc.destroy(ldxr_buf);
557		defer alloc.destroy(stxr_buf);
558
559	341	const reg = n.register();
560	342
561		const ldop = if (lse)
562		std.fmt.bufPrintZ(ldop_buf,
563		\\{[ldnm]s}{[a]s}{[l]s}{[s]s} {[reg]s}0, {[reg]s}0, [x1]
564		, .{ .ldnm = ldname, .a = o_def.a, .l = o_def.l, .s = s_def.s, .reg = reg })
565		else
566		std.fmt.bufPrintZ(ldop_buf,
567		\\.inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}
568		, .{ .op_n = op_n, .b = s_def.b, .n = o_def.n });
569
570		const ldxr = try std.fmt.bufPrintZ(ldxr_buf, "ld{s}xr{s}", .{ o_def.a, s_def.s });
571		const stxr = try std.fmt.bufPrintZ(stxr_buf, "st{s}xr{s}", .{ o_def.l, s_def.s });
572
573		var instr_buf = try alloc.create([1000:0]u8);
574		errdefer alloc.destroy(instr_buf);
575		return try std.fmt.bufPrintZ(instr_buf,
	343	return try std.fmt.allocPrint(arena,
576	344	\\ cbz w16, 8f
577		\\ {[ldop]s}
	345	\\ .inst 0x38200020 + {[op_n]s} + {[b]s} + {[n]s}
578	346	\\ cbz wzr, 1f
579	347	\\8:
580	348	\\ mov {[reg]s}16, {[reg]s}0
581	349	\\0:
582		\\ {[ldxr]s} {[reg]s}0, [x1]
	350	\\ ld{[a]s}xr{[s]s} {[reg]s}0, [x1]
583	351	\\ {[op]s} {[reg]s}17, {[reg]s}0, {[reg]s}16
584		\\ {[stxr]s} w15, {[reg]s}17, [x1]
	352	\\ st{[l]s}xr{[s]s} w15, {[reg]s}17, [x1]
585	353	\\ cbnz w15, 0b
586	354	\\1:
587	355	, .{
588		.ldop = ldop,
589		.ldxr = ldxr,
590		.stxr = stxr,
	356	.op_n = op_n,
	357	.b = s_def.b,
	358	.n = o_def.n,
	359	.s = s_def.s,
	360	.a = o_def.a,
	361	.l = o_def.l,
591	362	.op = op,
592	363	.reg = reg,
593	364	});