Commit f51bec321b1ef9196b143295d47a9ea919cdba28 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-03-12 18:55:16-04:00
committer	GitHub <noreply@github.com> 2020-03-12 18:55:16-04:00
log	f51bec321b1ef9196b143295d47a9ea919cdba28
tree	ec4871b071e28a1b4381399c166b5be7f77e1532
parent	aa49f972d655eb61ca899f76ba37933254f780a2
parent	71d776c3be91f6b4e982b45fbfe03e3696a397f5
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4707 from Vexu/small-atomics

Support atomic operations with bools and non power of two integers

11 files changed, 415 insertions(+), 176 deletions(-)

doc
- langref.html.in
lib
- std
  - atomic
    int.zig
    queue.zig
    stack.zig
  - event
    channel.zig
    lock.zig
    rwlock.zig
src
- codegen.cpp
- ir.cpp
test
- compile_errors.zig
- stage1
  - behavior
    atomics.zig

doc/langref.html.in+10-35

...	...	@@ -6728,17 +6728,8 @@ async fn func(y: *i32) void {
6728	6728	This builtin function atomically dereferences a pointer and returns the value.
6729	6729	</p>
6730	6730	<p>
6731		{#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,
6732		an integer whose bit count meets these requirements:
6733		</p>
6734		<ul>
6735		<li>At least 8</li>
6736		<li>At most the same as usize</li>
6737		<li>Power of 2</li>
6738		</ul> or an enum with a valid integer tag type.
6739		<p>
6740		TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe
6741		we can remove this restriction
	6731	{#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
	6732	an integer or an enum.
6742	6733	</p>
6743	6734	{#header_close#}
6744	6735	{#header_open\|@atomicRmw#}
...	...	@@ -6747,17 +6738,8 @@ async fn func(y: *i32) void {
6747	6738	This builtin function atomically modifies memory and then returns the previous value.
6748	6739	</p>
6749	6740	<p>
6750		{#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#},
6751		or an integer whose bit count meets these requirements:
6752		</p>
6753		<ul>
6754		<li>At least 8</li>
6755		<li>At most the same as usize</li>
6756		<li>Power of 2</li>
6757		</ul>
6758		<p>
6759		TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe
6760		we can remove this restriction
	6741	{#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
	6742	an integer or an enum.
6761	6743	</p>
6762	6744	<p>
6763	6745	Supported operations:
...	...	@@ -6782,17 +6764,8 @@ async fn func(y: *i32) void {
6782	6764	This builtin function atomically stores a value.
6783	6765	</p>
6784	6766	<p>
6785		{#syntax#}T{#endsyntax#} must be a pointer type, a {#syntax#}bool{#endsyntax#}, a float,
6786		an integer whose bit count meets these requirements:
6787		</p>
6788		<ul>
6789		<li>At least 8</li>
6790		<li>At most the same as usize</li>
6791		<li>Power of 2</li>
6792		</ul> or an enum with a valid integer tag type.
6793		<p>
6794		TODO right now bool is not accepted. Also I think we could make non powers of 2 work fine, maybe
6795		we can remove this restriction
	6767	{#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
	6768	an integer or an enum.
6796	6769	</p>
6797	6770	{#header_close#}
6798	6771	{#header_open\|@bitCast#}
...	...	@@ -7108,7 +7081,8 @@ fn cmpxchgStrongButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_v
7108	7081	more efficiently in machine instructions.
7109	7082	</p>
7110	7083	<p>
7111		{#syntax#}AtomicOrder{#endsyntax#} can be found with {#syntax#}@import("builtin").AtomicOrder{#endsyntax#}.
	7084	{#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
	7085	an integer or an enum.
7112	7086	</p>
7113	7087	<p>{#syntax#}@TypeOf(ptr).alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>
7114	7088	{#see_also\|Compile Variables\|cmpxchgWeak#}
...	...	@@ -7136,7 +7110,8 @@ fn cmpxchgWeakButNotAtomic(comptime T: type, ptr: *T, expected_value: T, new_val
7136	7110	However if you need a stronger guarantee, use {#link\|@cmpxchgStrong#}.
7137	7111	</p>
7138	7112	<p>
7139		{#syntax#}AtomicOrder{#endsyntax#} can be found with {#syntax#}@import("builtin").AtomicOrder{#endsyntax#}.
	7113	{#syntax#}T{#endsyntax#} must be a {#syntax#}bool{#endsyntax#}, a float,
	7114	an integer or an enum.
7140	7115	</p>
7141	7116	<p>{#syntax#}@TypeOf(ptr).alignment{#endsyntax#} must be {#syntax#}>= @sizeOf(T).{#endsyntax#}</p>
7142	7117	{#see_also\|Compile Variables\|cmpxchgStrong#}

lib/std/atomic/int.zig+5-8

...	...	@@ -1,6 +1,3 @@
1		const builtin = @import("builtin");
2		const AtomicOrder = builtin.AtomicOrder;
3
4	1	/// Thread-safe, lock-free integer
5	2	pub fn Int(comptime T: type) type {
6	3	return struct {
...	...	@@ -14,16 +11,16 @@ pub fn Int(comptime T: type) type {
14	11
15	12	/// Returns previous value
16	13	pub fn incr(self: *Self) T {
17		return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, 1, AtomicOrder.SeqCst);
	14	return @atomicRmw(T, &self.unprotected_value, .Add, 1, .SeqCst);
18	15	}
19	16
20	17	/// Returns previous value
21	18	pub fn decr(self: *Self) T {
22		return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst);
	19	return @atomicRmw(T, &self.unprotected_value, .Sub, 1, .SeqCst);
23	20	}
24	21
25	22	pub fn get(self: *Self) T {
26		return @atomicLoad(T, &self.unprotected_value, AtomicOrder.SeqCst);
	23	return @atomicLoad(T, &self.unprotected_value, .SeqCst);
27	24	}
28	25
29	26	pub fn set(self: *Self, new_value: T) void {
...	...	@@ -31,11 +28,11 @@ pub fn Int(comptime T: type) type {
31	28	}
32	29
33	30	pub fn xchg(self: *Self, new_value: T) T {
34		return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Xchg, new_value, AtomicOrder.SeqCst);
	31	return @atomicRmw(T, &self.unprotected_value, .Xchg, new_value, .SeqCst);
35	32	}
36	33
37	34	pub fn fetchAdd(self: *Self, op: T) T {
38		return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, op, AtomicOrder.SeqCst);
	35	return @atomicRmw(T, &self.unprotected_value, .Add, op, .SeqCst);
39	36	}
40	37	};
41	38	}

lib/std/atomic/queue.zig+9-11

...	...	@@ -1,7 +1,5 @@
1	1	const std = @import("../std.zig");
2	2	const builtin = @import("builtin");
3		const AtomicOrder = builtin.AtomicOrder;
4		const AtomicRmwOp = builtin.AtomicRmwOp;
5	3	const assert = std.debug.assert;
6	4	const expect = std.testing.expect;
7	5
...	...	@@ -145,7 +143,7 @@ const Context = struct {
145	143	put_sum: isize,
146	144	get_sum: isize,
147	145	get_count: usize,
148		puts_done: u8, // TODO make this a bool
	146	puts_done: bool,
149	147	};
150	148
151	149	// TODO add lazy evaluated build options and then put puts_per_thread behind
...	...	@@ -169,7 +167,7 @@ test "std.atomic.Queue" {
169	167	.queue = &queue,
170	168	.put_sum = 0,
171	169	.get_sum = 0,
172		.puts_done = 0,
	170	.puts_done = false,
173	171	.get_count = 0,
174	172	};
175	173
...	...	@@ -182,7 +180,7 @@ test "std.atomic.Queue" {
182	180	}
183	181	}
184	182	expect(!context.queue.isEmpty());
185		context.puts_done = 1;
	183	context.puts_done = true;
186	184	{
187	185	var i: usize = 0;
188	186	while (i < put_thread_count) : (i += 1) {
...	...	@@ -204,7 +202,7 @@ test "std.atomic.Queue" {
204	202
205	203	for (putters) \|t\|
206	204	t.wait();
207		@atomicStore(u8, &context.puts_done, 1, AtomicOrder.SeqCst);
	205	@atomicStore(bool, &context.puts_done, true, .SeqCst);
208	206	for (getters) \|t\|
209	207	t.wait();
210	208
...	...	@@ -231,25 +229,25 @@ fn startPuts(ctx: *Context) u8 {
231	229	std.time.sleep(1); // let the os scheduler be our fuzz
232	230	const x = @bitCast(i32, r.random.scalar(u32));
233	231	const node = ctx.allocator.create(Queue(i32).Node) catch unreachable;
234		node.* = Queue(i32).Node{
	232	node.* = .{
235	233	.prev = undefined,
236	234	.next = undefined,
237	235	.data = x,
238	236	};
239	237	ctx.queue.put(node);
240		_ = @atomicRmw(isize, &ctx.put_sum, builtin.AtomicRmwOp.Add, x, AtomicOrder.SeqCst);
	238	_ = @atomicRmw(isize, &ctx.put_sum, .Add, x, .SeqCst);
241	239	}
242	240	return 0;
243	241	}
244	242
245	243	fn startGets(ctx: *Context) u8 {
246	244	while (true) {
247		const last = @atomicLoad(u8, &ctx.puts_done, builtin.AtomicOrder.SeqCst) == 1;
	245	const last = @atomicLoad(bool, &ctx.puts_done, .SeqCst);
248	246
249	247	while (ctx.queue.get()) \|node\| {
250	248	std.time.sleep(1); // let the os scheduler be our fuzz
251		_ = @atomicRmw(isize, &ctx.get_sum, builtin.AtomicRmwOp.Add, node.data, builtin.AtomicOrder.SeqCst);
252		_ = @atomicRmw(usize, &ctx.get_count, builtin.AtomicRmwOp.Add, 1, builtin.AtomicOrder.SeqCst);
	249	_ = @atomicRmw(isize, &ctx.get_sum, .Add, node.data, .SeqCst);
	250	_ = @atomicRmw(usize, &ctx.get_count, .Add, 1, .SeqCst);
253	251	}
254	252
255	253	if (last) return 0;

lib/std/atomic/stack.zig+15-16

...	...	@@ -1,6 +1,5 @@
1	1	const assert = std.debug.assert;
2	2	const builtin = @import("builtin");
3		const AtomicOrder = builtin.AtomicOrder;
4	3	const expect = std.testing.expect;
5	4
6	5	/// Many reader, many writer, non-allocating, thread-safe
...	...	@@ -11,7 +10,7 @@ pub fn Stack(comptime T: type) type {
11	10	root: ?*Node,
12	11	lock: @TypeOf(lock_init),
13	12
14		const lock_init = if (builtin.single_threaded) {} else @as(u8, 0);
	13	const lock_init = if (builtin.single_threaded) {} else false;
15	14
16	15	pub const Self = @This();
17	16
...	...	@@ -31,7 +30,7 @@ pub fn Stack(comptime T: type) type {
31	30	/// being the first item in the stack, returns the other item that was there.
32	31	pub fn pushFirst(self: Self, node: Node) ?*Node {
33	32	node.next = null;
34		return @cmpxchgStrong(?*Node, &self.root, null, node, AtomicOrder.SeqCst, AtomicOrder.SeqCst);
	33	return @cmpxchgStrong(?*Node, &self.root, null, node, .SeqCst, .SeqCst);
35	34	}
36	35
37	36	pub fn push(self: Self, node: Node) void {
...	...	@@ -39,8 +38,8 @@ pub fn Stack(comptime T: type) type {
39	38	node.next = self.root;
40	39	self.root = node;
41	40	} else {
42		while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {}
43		defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1);
	41	while (@atomicRmw(bool, &self.lock, .Xchg, true, .SeqCst)) {}
	42	defer assert(@atomicRmw(bool, &self.lock, .Xchg, false, .SeqCst));
44	43
45	44	node.next = self.root;
46	45	self.root = node;
...	...	@@ -53,8 +52,8 @@ pub fn Stack(comptime T: type) type {
53	52	self.root = root.next;
54	53	return root;
55	54	} else {
56		while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {}
57		defer assert(@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1);
	55	while (@atomicRmw(bool, &self.lock, .Xchg, true, .SeqCst)) {}
	56	defer assert(@atomicRmw(bool, &self.lock, .Xchg, false, .SeqCst));
58	57
59	58	const root = self.root orelse return null;
60	59	self.root = root.next;
...	...	@@ -63,7 +62,7 @@ pub fn Stack(comptime T: type) type {
63	62	}
64	63
65	64	pub fn isEmpty(self: *Self) bool {
66		return @atomicLoad(?*Node, &self.root, AtomicOrder.SeqCst) == null;
	65	return @atomicLoad(?*Node, &self.root, .SeqCst) == null;
67	66	}
68	67	};
69	68	}
...	...	@@ -75,7 +74,7 @@ const Context = struct {
75	74	put_sum: isize,
76	75	get_sum: isize,
77	76	get_count: usize,
78		puts_done: u8, // TODO make this a bool
	77	puts_done: bool,
79	78	};
80	79	// TODO add lazy evaluated build options and then put puts_per_thread behind
81	80	// some option such as: "AggressiveMultithreadedFuzzTest". In the AppVeyor
...	...	@@ -98,7 +97,7 @@ test "std.atomic.stack" {
98	97	.stack = &stack,
99	98	.put_sum = 0,
100	99	.get_sum = 0,
101		.puts_done = 0,
	100	.puts_done = false,
102	101	.get_count = 0,
103	102	};
104	103
...	...	@@ -109,7 +108,7 @@ test "std.atomic.stack" {
109	108	expect(startPuts(&context) == 0);
110	109	}
111	110	}
112		context.puts_done = 1;
	111	context.puts_done = true;
113	112	{
114	113	var i: usize = 0;
115	114	while (i < put_thread_count) : (i += 1) {
...	...	@@ -128,7 +127,7 @@ test "std.atomic.stack" {
128	127
129	128	for (putters) \|t\|
130	129	t.wait();
131		@atomicStore(u8, &context.puts_done, 1, AtomicOrder.SeqCst);
	130	@atomicStore(bool, &context.puts_done, true, .SeqCst);
132	131	for (getters) \|t\|
133	132	t.wait();
134	133	}
...	...	@@ -158,19 +157,19 @@ fn startPuts(ctx: *Context) u8 {
158	157	.data = x,
159	158	};
160	159	ctx.stack.push(node);
161		_ = @atomicRmw(isize, &ctx.put_sum, builtin.AtomicRmwOp.Add, x, AtomicOrder.SeqCst);
	160	_ = @atomicRmw(isize, &ctx.put_sum, .Add, x, .SeqCst);
162	161	}
163	162	return 0;
164	163	}
165	164
166	165	fn startGets(ctx: *Context) u8 {
167	166	while (true) {
168		const last = @atomicLoad(u8, &ctx.puts_done, builtin.AtomicOrder.SeqCst) == 1;
	167	const last = @atomicLoad(bool, &ctx.puts_done, .SeqCst);
169	168
170	169	while (ctx.stack.pop()) \|node\| {
171	170	std.time.sleep(1); // let the os scheduler be our fuzz
172		_ = @atomicRmw(isize, &ctx.get_sum, builtin.AtomicRmwOp.Add, node.data, builtin.AtomicOrder.SeqCst);
173		_ = @atomicRmw(usize, &ctx.get_count, builtin.AtomicRmwOp.Add, 1, builtin.AtomicOrder.SeqCst);
	171	_ = @atomicRmw(isize, &ctx.get_sum, .Add, node.data, .SeqCst);
	172	_ = @atomicRmw(usize, &ctx.get_count, .Add, 1, .SeqCst);
174	173	}
175	174
176	175	if (last) return 0;

lib/std/event/channel.zig+10-13

...	...	@@ -14,8 +14,8 @@ pub fn Channel(comptime T: type) type {
14	14	putters: std.atomic.Queue(PutNode),
15	15	get_count: usize,
16	16	put_count: usize,
17		dispatch_lock: u8, // TODO make this a bool
18		need_dispatch: u8, // TODO make this a bool
	17	dispatch_lock: bool,
	18	need_dispatch: bool,
19	19
20	20	// simple fixed size ring buffer
21	21	buffer_nodes: []T,
...	...	@@ -62,8 +62,8 @@ pub fn Channel(comptime T: type) type {
62	62	.buffer_len = 0,
63	63	.buffer_nodes = buffer,
64	64	.buffer_index = 0,
65		.dispatch_lock = 0,
66		.need_dispatch = 0,
	65	.dispatch_lock = false,
	66	.need_dispatch = false,
67	67	.getters = std.atomic.Queue(GetNode).init(),
68	68	.putters = std.atomic.Queue(PutNode).init(),
69	69	.or_null_queue = std.atomic.Queue(*std.atomic.Queue(GetNode).Node).init(),
...	...	@@ -165,15 +165,14 @@ pub fn Channel(comptime T: type) type {
165	165
166	166	fn dispatch(self: *SelfChannel) void {
167	167	// set the "need dispatch" flag
168		@atomicStore(u8, &self.need_dispatch, 1, .SeqCst);
	168	@atomicStore(bool, &self.need_dispatch, true, .SeqCst);
169	169
170	170	lock: while (true) {
171	171	// set the lock flag
172		const prev_lock = @atomicRmw(u8, &self.dispatch_lock, .Xchg, 1, .SeqCst);
173		if (prev_lock != 0) return;
	172	if (@atomicRmw(bool, &self.dispatch_lock, .Xchg, true, .SeqCst)) return;
174	173
175	174	// clear the need_dispatch flag since we're about to do it
176		@atomicStore(u8, &self.need_dispatch, 0, .SeqCst);
	175	@atomicStore(bool, &self.need_dispatch, false, .SeqCst);
177	176
178	177	while (true) {
179	178	one_dispatch: {
...	...	@@ -250,14 +249,12 @@ pub fn Channel(comptime T: type) type {
250	249	}
251	250
252	251	// clear need-dispatch flag
253		const need_dispatch = @atomicRmw(u8, &self.need_dispatch, .Xchg, 0, .SeqCst);
254		if (need_dispatch != 0) continue;
	252	if (@atomicRmw(bool, &self.need_dispatch, .Xchg, false, .SeqCst)) continue;
255	253
256		const my_lock = @atomicRmw(u8, &self.dispatch_lock, .Xchg, 0, .SeqCst);
257		assert(my_lock != 0);
	254	assert(@atomicRmw(bool, &self.dispatch_lock, .Xchg, false, .SeqCst));
258	255
259	256	// we have to check again now that we unlocked
260		if (@atomicLoad(u8, &self.need_dispatch, .SeqCst) != 0) continue :lock;
	257	if (@atomicLoad(bool, &self.need_dispatch, .SeqCst)) continue :lock;
261	258
262	259	return;
263	260	}

lib/std/event/lock.zig+17-19

...	...	@@ -11,9 +11,9 @@ const Loop = std.event.Loop;
11	11	/// Allows only one actor to hold the lock.
12	12	/// TODO: make this API also work in blocking I/O mode.
13	13	pub const Lock = struct {
14		shared_bit: u8, // TODO make this a bool
	14	shared: bool,
15	15	queue: Queue,
16		queue_empty_bit: u8, // TODO make this a bool
	16	queue_empty: bool,
17	17
18	18	const Queue = std.atomic.Queue(anyframe);
19	19
...	...	@@ -31,20 +31,19 @@ pub const Lock = struct {
31	31	}
32	32
33	33	// We need to release the lock.
34		@atomicStore(u8, &self.lock.queue_empty_bit, 1, .SeqCst);
35		@atomicStore(u8, &self.lock.shared_bit, 0, .SeqCst);
	34	@atomicStore(bool, &self.lock.queue_empty, true, .SeqCst);
	35	@atomicStore(bool, &self.lock.shared, false, .SeqCst);
36	36
37	37	// There might be a queue item. If we know the queue is empty, we can be done,
38	38	// because the other actor will try to obtain the lock.
39	39	// But if there's a queue item, we are the actor which must loop and attempt
40	40	// to grab the lock again.
41		if (@atomicLoad(u8, &self.lock.queue_empty_bit, .SeqCst) == 1) {
	41	if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) {
42	42	return;
43	43	}
44	44
45	45	while (true) {
46		const old_bit = @atomicRmw(u8, &self.lock.shared_bit, .Xchg, 1, .SeqCst);
47		if (old_bit != 0) {
	46	if (@atomicRmw(bool, &self.lock.shared, .Xchg, true, .SeqCst)) {
48	47	// We did not obtain the lock. Great, the queue is someone else's problem.
49	48	return;
50	49	}
...	...	@@ -56,11 +55,11 @@ pub const Lock = struct {
56	55	}
57	56
58	57	// Release the lock again.
59		@atomicStore(u8, &self.lock.queue_empty_bit, 1, .SeqCst);
60		@atomicStore(u8, &self.lock.shared_bit, 0, .SeqCst);
	58	@atomicStore(bool, &self.lock.queue_empty, true, .SeqCst);
	59	@atomicStore(bool, &self.lock.shared, false, .SeqCst);
61	60
62	61	// Find out if we can be done.
63		if (@atomicLoad(u8, &self.lock.queue_empty_bit, .SeqCst) == 1) {
	62	if (@atomicLoad(bool, &self.lock.queue_empty, .SeqCst)) {
64	63	return;
65	64	}
66	65	}
...	...	@@ -69,24 +68,24 @@ pub const Lock = struct {
69	68
70	69	pub fn init() Lock {
71	70	return Lock{
72		.shared_bit = 0,
	71	.shared = false,
73	72	.queue = Queue.init(),
74		.queue_empty_bit = 1,
	73	.queue_empty = true,
75	74	};
76	75	}
77	76
78	77	pub fn initLocked() Lock {
79	78	return Lock{
80		.shared_bit = 1,
	79	.shared = true,
81	80	.queue = Queue.init(),
82		.queue_empty_bit = 1,
	81	.queue_empty = true,
83	82	};
84	83	}
85	84
86	85	/// Must be called when not locked. Not thread safe.
87	86	/// All calls to acquire() and release() must complete before calling deinit().
88	87	pub fn deinit(self: *Lock) void {
89		assert(self.shared_bit == 0);
	88	assert(!self.shared);
90	89	while (self.queue.get()) \|node\| resume node.data;
91	90	}
92	91
...	...	@@ -99,12 +98,11 @@ pub const Lock = struct {
99	98
100	99	// At this point, we are in the queue, so we might have already been resumed.
101	100
102		// We set this bit so that later we can rely on the fact, that if queue_empty_bit is 1, some actor
	101	// We set this bit so that later we can rely on the fact, that if queue_empty == true, some actor
103	102	// will attempt to grab the lock.
104		@atomicStore(u8, &self.queue_empty_bit, 0, .SeqCst);
	103	@atomicStore(bool, &self.queue_empty, false, .SeqCst);
105	104
106		const old_bit = @atomicRmw(u8, &self.shared_bit, .Xchg, 1, .SeqCst);
107		if (old_bit == 0) {
	105	if (!@atomicRmw(bool, &self.shared, .Xchg, true, .SeqCst)) {
108	106	if (self.queue.get()) \|node\| {
109	107	// Whether this node is us or someone else, we tail resume it.
110	108	resume node.data;

lib/std/event/rwlock.zig+16-16

...	...	@@ -16,8 +16,8 @@ pub const RwLock = struct {
16	16	shared_state: State,
17	17	writer_queue: Queue,
18	18	reader_queue: Queue,
19		writer_queue_empty_bit: u8, // TODO make this a bool
20		reader_queue_empty_bit: u8, // TODO make this a bool
	19	writer_queue_empty: bool,
	20	reader_queue_empty: bool,
21	21	reader_lock_count: usize,
22	22
23	23	const State = enum(u8) {
...	...	@@ -40,7 +40,7 @@ pub const RwLock = struct {
40	40	return;
41	41	}
42	42
43		@atomicStore(u8, &self.lock.reader_queue_empty_bit, 1, .SeqCst);
	43	@atomicStore(bool, &self.lock.reader_queue_empty, true, .SeqCst);
44	44	if (@cmpxchgStrong(State, &self.lock.shared_state, .ReadLock, .Unlocked, .SeqCst, .SeqCst) != null) {
45	45	// Didn't unlock. Someone else's problem.
46	46	return;
...	...	@@ -62,7 +62,7 @@ pub const RwLock = struct {
62	62	}
63	63
64	64	// We need to release the write lock. Check if any readers are waiting to grab the lock.
65		if (@atomicLoad(u8, &self.lock.reader_queue_empty_bit, .SeqCst) == 0) {
	65	if (!@atomicLoad(bool, &self.lock.reader_queue_empty, .SeqCst)) {
66	66	// Switch to a read lock.
67	67	@atomicStore(State, &self.lock.shared_state, .ReadLock, .SeqCst);
68	68	while (self.lock.reader_queue.get()) \|node\| {
...	...	@@ -71,7 +71,7 @@ pub const RwLock = struct {
71	71	return;
72	72	}
73	73
74		@atomicStore(u8, &self.lock.writer_queue_empty_bit, 1, .SeqCst);
	74	@atomicStore(bool, &self.lock.writer_queue_empty, true, .SeqCst);
75	75	@atomicStore(State, &self.lock.shared_state, .Unlocked, .SeqCst);
76	76
77	77	self.lock.commonPostUnlock();
...	...	@@ -79,12 +79,12 @@ pub const RwLock = struct {
79	79	};
80	80
81	81	pub fn init() RwLock {
82		return RwLock{
	82	return .{
83	83	.shared_state = .Unlocked,
84	84	.writer_queue = Queue.init(),
85		.writer_queue_empty_bit = 1,
	85	.writer_queue_empty = true,
86	86	.reader_queue = Queue.init(),
87		.reader_queue_empty_bit = 1,
	87	.reader_queue_empty = true,
88	88	.reader_lock_count = 0,
89	89	};
90	90	}
...	...	@@ -111,9 +111,9 @@ pub const RwLock = struct {
111	111
112	112	// At this point, we are in the reader_queue, so we might have already been resumed.
113	113
114		// We set this bit so that later we can rely on the fact, that if reader_queue_empty_bit is 1,
	114	// We set this bit so that later we can rely on the fact, that if reader_queue_empty == true,
115	115	// some actor will attempt to grab the lock.
116		@atomicStore(u8, &self.reader_queue_empty_bit, 0, .SeqCst);
	116	@atomicStore(bool, &self.reader_queue_empty, false, .SeqCst);
117	117
118	118	// Here we don't care if we are the one to do the locking or if it was already locked for reading.
119	119	const have_read_lock = if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .ReadLock, .SeqCst, .SeqCst)) \|old_state\| old_state == .ReadLock else true;
...	...	@@ -142,9 +142,9 @@ pub const RwLock = struct {
142	142
143	143	// At this point, we are in the writer_queue, so we might have already been resumed.
144	144
145		// We set this bit so that later we can rely on the fact, that if writer_queue_empty_bit is 1,
	145	// We set this bit so that later we can rely on the fact, that if writer_queue_empty == true,
146	146	// some actor will attempt to grab the lock.
147		@atomicStore(u8, &self.writer_queue_empty_bit, 0, .SeqCst);
	147	@atomicStore(bool, &self.writer_queue_empty, false, .SeqCst);
148	148
149	149	// Here we must be the one to acquire the write lock. It cannot already be locked.
150	150	if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .WriteLock, .SeqCst, .SeqCst) == null) {
...	...	@@ -165,7 +165,7 @@ pub const RwLock = struct {
165	165	// obtain the lock.
166	166	// But if there's a writer_queue item or a reader_queue item,
167	167	// we are the actor which must loop and attempt to grab the lock again.
168		if (@atomicLoad(u8, &self.writer_queue_empty_bit, .SeqCst) == 0) {
	168	if (!@atomicLoad(bool, &self.writer_queue_empty, .SeqCst)) {
169	169	if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .WriteLock, .SeqCst, .SeqCst) != null) {
170	170	// We did not obtain the lock. Great, the queues are someone else's problem.
171	171	return;
...	...	@@ -176,12 +176,12 @@ pub const RwLock = struct {
176	176	return;
177	177	}
178	178	// Release the lock again.
179		@atomicStore(u8, &self.writer_queue_empty_bit, 1, .SeqCst);
	179	@atomicStore(bool, &self.writer_queue_empty, true, .SeqCst);
180	180	@atomicStore(State, &self.shared_state, .Unlocked, .SeqCst);
181	181	continue;
182	182	}
183	183
184		if (@atomicLoad(u8, &self.reader_queue_empty_bit, .SeqCst) == 0) {
	184	if (!@atomicLoad(bool, &self.reader_queue_empty, .SeqCst)) {
185	185	if (@cmpxchgStrong(State, &self.shared_state, .Unlocked, .ReadLock, .SeqCst, .SeqCst) != null) {
186	186	// We did not obtain the lock. Great, the queues are someone else's problem.
187	187	return;
...	...	@@ -195,7 +195,7 @@ pub const RwLock = struct {
195	195	return;
196	196	}
197	197	// Release the lock again.
198		@atomicStore(u8, &self.reader_queue_empty_bit, 1, .SeqCst);
	198	@atomicStore(bool, &self.reader_queue_empty, true, .SeqCst);
199	199	if (@cmpxchgStrong(State, &self.shared_state, .ReadLock, .Unlocked, .SeqCst, .SeqCst) != null) {
200	200	// Didn't unlock. Someone else's problem.
201	201	return;

src/codegen.cpp+89

...	...	@@ -5251,11 +5251,55 @@ static enum ZigLLVM_AtomicRMWBinOp to_ZigLLVMAtomicRMWBinOp(AtomicRmwOp op, bool
5251	5251	zig_unreachable();
5252	5252	}
5253	5253
	5254	static LLVMTypeRef get_atomic_abi_type(CodeGen g, IrInstGen instruction) {
	5255	// If the operand type of an atomic operation is not a power of two sized
	5256	// we need to widen it before using it and then truncate the result.
	5257
	5258	ir_assert(instruction->value->type->id == ZigTypeIdPointer, instruction);
	5259	ZigType *operand_type = instruction->value->type->data.pointer.child_type;
	5260	if (operand_type->id == ZigTypeIdInt \|\| operand_type->id == ZigTypeIdEnum) {
	5261	if (operand_type->id == ZigTypeIdEnum) {
	5262	operand_type = operand_type->data.enumeration.tag_int_type;
	5263	}
	5264	auto bit_count = operand_type->data.integral.bit_count;
	5265	bool is_signed = operand_type->data.integral.is_signed;
	5266
	5267	ir_assert(bit_count != 0, instruction);
	5268	if (bit_count == 1 \|\| !is_power_of_2(bit_count)) {
	5269	return get_llvm_type(g, get_int_type(g, is_signed, operand_type->abi_size * 8));
	5270	} else {
	5271	return nullptr;
	5272	}
	5273	} else if (operand_type->id == ZigTypeIdFloat) {
	5274	return nullptr;
	5275	} else if (operand_type->id == ZigTypeIdBool) {
	5276	return g->builtin_types.entry_u8->llvm_type;
	5277	} else {
	5278	ir_assert(get_codegen_ptr_type_bail(g, operand_type) != nullptr, instruction);
	5279	return nullptr;
	5280	}
	5281	}
	5282
5254	5283	static LLVMValueRef ir_render_cmpxchg(CodeGen g, IrExecutableGen executable, IrInstGenCmpxchg *instruction) {
5255	5284	LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr);
5256	5285	LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value);
5257	5286	LLVMValueRef new_val = ir_llvm_value(g, instruction->new_value);
5258	5287
	5288	ZigType *operand_type = instruction->new_value->value->type;
	5289	LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
	5290	if (actual_abi_type != nullptr) {
	5291	// operand needs widening and truncating
	5292	ptr_val = LLVMBuildBitCast(g->builder, ptr_val,
	5293	LLVMPointerType(actual_abi_type, 0), "");
	5294	if (operand_type->data.integral.is_signed) {
	5295	cmp_val = LLVMBuildSExt(g->builder, cmp_val, actual_abi_type, "");
	5296	new_val = LLVMBuildSExt(g->builder, new_val, actual_abi_type, "");
	5297	} else {
	5298	cmp_val = LLVMBuildZExt(g->builder, cmp_val, actual_abi_type, "");
	5299	new_val = LLVMBuildZExt(g->builder, new_val, actual_abi_type, "");
	5300	}
	5301	}
	5302
5259	5303	LLVMAtomicOrdering success_order = to_LLVMAtomicOrdering(instruction->success_order);
5260	5304	LLVMAtomicOrdering failure_order = to_LLVMAtomicOrdering(instruction->failure_order);
5261	5305
...	...	@@ -5268,6 +5312,9 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen g, IrExecutableGen executable, I
5268	5312
5269	5313	if (!handle_is_ptr(g, optional_type)) {
5270	5314	LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
	5315	if (actual_abi_type != nullptr) {
	5316	payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), "");
	5317	}
5271	5318	LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, "");
5272	5319	return LLVMBuildSelect(g->builder, success_bit, LLVMConstNull(get_llvm_type(g, child_type)), payload_val, "");
5273	5320	}
...	...	@@ -5282,6 +5329,9 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen g, IrExecutableGen executable, I
5282	5329	ir_assert(type_has_bits(g, child_type), &instruction->base);
5283	5330
5284	5331	LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, "");
	5332	if (actual_abi_type != nullptr) {
	5333	payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), "");
	5334	}
5285	5335	LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, "");
5286	5336	gen_assign_raw(g, val_ptr, get_pointer_to_type(g, child_type, false), payload_val);
5287	5337
...	...	@@ -5859,6 +5909,22 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen g, IrExecutableGen executable
5859	5909	LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
5860	5910	LLVMValueRef operand = ir_llvm_value(g, instruction->operand);
5861	5911
	5912	LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
	5913	if (actual_abi_type != nullptr) {
	5914	// operand needs widening and truncating
	5915	LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr,
	5916	LLVMPointerType(actual_abi_type, 0), "");
	5917	LLVMValueRef casted_operand;
	5918	if (operand_type->data.integral.is_signed) {
	5919	casted_operand = LLVMBuildSExt(g->builder, operand, actual_abi_type, "");
	5920	} else {
	5921	casted_operand = LLVMBuildZExt(g->builder, operand, actual_abi_type, "");
	5922	}
	5923	LLVMValueRef uncasted_result = ZigLLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering,
	5924	g->is_single_threaded);
	5925	return LLVMBuildTrunc(g->builder, uncasted_result, get_llvm_type(g, operand_type), "");
	5926	}
	5927
5862	5928	if (get_codegen_ptr_type_bail(g, operand_type) == nullptr) {
5863	5929	return ZigLLVMBuildAtomicRMW(g->builder, op, ptr, operand, ordering, g->is_single_threaded);
5864	5930	}
...	...	@@ -5877,6 +5943,17 @@ static LLVMValueRef ir_render_atomic_load(CodeGen g, IrExecutableGen executabl
5877	5943	{
5878	5944	LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
5879	5945	LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
	5946
	5947	ZigType *operand_type = instruction->ptr->value->type->data.pointer.child_type;
	5948	LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
	5949	if (actual_abi_type != nullptr) {
	5950	// operand needs widening and truncating
	5951	ptr = LLVMBuildBitCast(g->builder, ptr,
	5952	LLVMPointerType(actual_abi_type, 0), "");
	5953	LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");
	5954	LLVMSetOrdering(load_inst, ordering);
	5955	return LLVMBuildTrunc(g->builder, load_inst, get_llvm_type(g, operand_type), "");
	5956	}
5880	5957	LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, "");
5881	5958	LLVMSetOrdering(load_inst, ordering);
5882	5959	return load_inst;
...	...	@@ -5888,6 +5965,18 @@ static LLVMValueRef ir_render_atomic_store(CodeGen g, IrExecutableGen executab
5888	5965	LLVMAtomicOrdering ordering = to_LLVMAtomicOrdering(instruction->ordering);
5889	5966	LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr);
5890	5967	LLVMValueRef value = ir_llvm_value(g, instruction->value);
	5968
	5969	LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr);
	5970	if (actual_abi_type != nullptr) {
	5971	// operand needs widening
	5972	ptr = LLVMBuildBitCast(g->builder, ptr,
	5973	LLVMPointerType(actual_abi_type, 0), "");
	5974	if (instruction->value->value->type->data.integral.is_signed) {
	5975	value = LLVMBuildSExt(g->builder, value, actual_abi_type, "");
	5976	} else {
	5977	value = LLVMBuildZExt(g->builder, value, actual_abi_type, "");
	5978	}
	5979	}
5891	5980	LLVMValueRef store_inst = gen_store(g, value, ptr, instruction->ptr->value->type);
5892	5981	LLVMSetOrdering(store_inst, ordering);
5893	5982	return nullptr;

src/ir.cpp+163-43

...	...	@@ -25208,12 +25208,50 @@ static IrInstGen ir_analyze_instruction_cmpxchg(IrAnalyze ira, IrInstSrcCmpxch
25208	25208	return ira->codegen->invalid_inst_gen;
25209	25209	}
25210	25210
	25211	ZigType *result_type = get_optional_type(ira->codegen, operand_type);
	25212
	25213	// special case zero bit types
	25214	switch (type_has_one_possible_value(ira->codegen, operand_type)) {
	25215	case OnePossibleValueInvalid:
	25216	return ira->codegen->invalid_inst_gen;
	25217	case OnePossibleValueYes: {
	25218	IrInstGen *result = ir_const(ira, &instruction->base.base, result_type);
	25219	set_optional_value_to_null(result->value);
	25220	return result;
	25221	}
	25222	case OnePossibleValueNo:
	25223	break;
	25224	}
	25225
25211	25226	if (instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut != ConstPtrMutRuntimeVar &&
25212	25227	instr_is_comptime(casted_cmp_value) && instr_is_comptime(casted_new_value)) {
25213		zig_panic("TODO compile-time execution of cmpxchg");
	25228	ZigValue *ptr_val = ir_resolve_const(ira, casted_ptr, UndefBad);
	25229	if (ptr_val == nullptr)
	25230	return ira->codegen->invalid_inst_gen;
	25231
	25232	ZigValue *stored_val = const_ptr_pointee(ira, ira->codegen, ptr_val, instruction->base.base.source_node);
	25233	if (stored_val == nullptr)
	25234	return ira->codegen->invalid_inst_gen;
	25235
	25236	ZigValue *expected_val = ir_resolve_const(ira, casted_cmp_value, UndefBad);
	25237	if (expected_val == nullptr)
	25238	return ira->codegen->invalid_inst_gen;
	25239
	25240	ZigValue *new_val = ir_resolve_const(ira, casted_new_value, UndefBad);
	25241	if (new_val == nullptr)
	25242	return ira->codegen->invalid_inst_gen;
	25243
	25244	bool eql = const_values_equal(ira->codegen, stored_val, expected_val);
	25245	IrInstGen *result = ir_const(ira, &instruction->base.base, result_type);
	25246	if (eql) {
	25247	copy_const_val(ira->codegen, stored_val, new_val);
	25248	set_optional_value_to_null(result->value);
	25249	} else {
	25250	set_optional_payload(result->value, stored_val);
	25251	}
	25252	return result;
25214	25253	}
25215	25254
25216		ZigType *result_type = get_optional_type(ira->codegen, operand_type);
25217	25255	IrInstGen *result_loc;
25218	25256	if (handle_is_ptr(ira->codegen, result_type)) {
25219	25257	result_loc = ir_resolve_result(ira, &instruction->base.base, instruction->result_loc,
...	...	@@ -28324,43 +28362,20 @@ static ZigType ir_resolve_atomic_operand_type(IrAnalyze ira, IrInstGen *op) {
28324	28362	if (type_is_invalid(operand_type))
28325	28363	return ira->codegen->builtin_types.entry_invalid;
28326	28364
28327		if (operand_type->id == ZigTypeIdInt) {
28328		if (operand_type->data.integral.bit_count < 8) {
28329		ir_add_error(ira, &op->base,
28330		buf_sprintf("expected integer type 8 bits or larger, found %" PRIu32 "-bit integer type",
28331		operand_type->data.integral.bit_count));
28332		return ira->codegen->builtin_types.entry_invalid;
	28365	if (operand_type->id == ZigTypeIdInt \|\| operand_type->id == ZigTypeIdEnum) {
	28366	ZigType *int_type;
	28367	if (operand_type->id == ZigTypeIdEnum) {
	28368	int_type = operand_type->data.enumeration.tag_int_type;
	28369	} else {
	28370	int_type = operand_type;
28333	28371	}
	28372	auto bit_count = int_type->data.integral.bit_count;
28334	28373	uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);
28335		if (operand_type->data.integral.bit_count > max_atomic_bits) {
	28374
	28375	if (bit_count > max_atomic_bits) {
28336	28376	ir_add_error(ira, &op->base,
28337	28377	buf_sprintf("expected %" PRIu32 "-bit integer type or smaller, found %" PRIu32 "-bit integer type",
28338		max_atomic_bits, operand_type->data.integral.bit_count));
28339		return ira->codegen->builtin_types.entry_invalid;
28340		}
28341		if (!is_power_of_2(operand_type->data.integral.bit_count)) {
28342		ir_add_error(ira, &op->base,
28343		buf_sprintf("%" PRIu32 "-bit integer type is not a power of 2", operand_type->data.integral.bit_count));
28344		return ira->codegen->builtin_types.entry_invalid;
28345		}
28346		} else if (operand_type->id == ZigTypeIdEnum) {
28347		ZigType *int_type = operand_type->data.enumeration.tag_int_type;
28348		if (int_type->data.integral.bit_count < 8) {
28349		ir_add_error(ira, &op->base,
28350		buf_sprintf("expected enum tag type 8 bits or larger, found %" PRIu32 "-bit tag type",
28351		int_type->data.integral.bit_count));
28352		return ira->codegen->builtin_types.entry_invalid;
28353		}
28354		uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch);
28355		if (int_type->data.integral.bit_count > max_atomic_bits) {
28356		ir_add_error(ira, &op->base,
28357		buf_sprintf("expected %" PRIu32 "-bit enum tag type or smaller, found %" PRIu32 "-bit tag type",
28358		max_atomic_bits, int_type->data.integral.bit_count));
28359		return ira->codegen->builtin_types.entry_invalid;
28360		}
28361		if (!is_power_of_2(int_type->data.integral.bit_count)) {
28362		ir_add_error(ira, &op->base,
28363		buf_sprintf("%" PRIu32 "-bit enum tag type is not a power of 2", int_type->data.integral.bit_count));
	28378	max_atomic_bits, bit_count));
28364	28379	return ira->codegen->builtin_types.entry_invalid;
28365	28380	}
28366	28381	} else if (operand_type->id == ZigTypeIdFloat) {
...	...	@@ -28371,6 +28386,8 @@ static ZigType ir_resolve_atomic_operand_type(IrAnalyze ira, IrInstGen *op) {
28371	28386	max_atomic_bits, (uint32_t) operand_type->data.floating.bit_count));
28372	28387	return ira->codegen->builtin_types.entry_invalid;
28373	28388	}
	28389	} else if (operand_type->id == ZigTypeIdBool) {
	28390	// will be treated as u8
28374	28391	} else {
28375	28392	Error err;
28376	28393	ZigType *operand_ptr_type;
...	...	@@ -28409,11 +28426,15 @@ static IrInstGen ir_analyze_instruction_atomic_rmw(IrAnalyze ira, IrInstSrcAto
28409	28426
28410	28427	if (operand_type->id == ZigTypeIdEnum && op != AtomicRmwOp_xchg) {
28411	28428	ir_add_error(ira, &instruction->op->base,
28412		buf_sprintf("@atomicRmw on enum only works with .Xchg"));
	28429	buf_sprintf("@atomicRmw with enum only allowed with .Xchg"));
	28430	return ira->codegen->invalid_inst_gen;
	28431	} else if (operand_type->id == ZigTypeIdBool && op != AtomicRmwOp_xchg) {
	28432	ir_add_error(ira, &instruction->op->base,
	28433	buf_sprintf("@atomicRmw with bool only allowed with .Xchg"));
28413	28434	return ira->codegen->invalid_inst_gen;
28414	28435	} else if (operand_type->id == ZigTypeIdFloat && op > AtomicRmwOp_sub) {
28415	28436	ir_add_error(ira, &instruction->op->base,
28416		buf_sprintf("@atomicRmw with float only works with .Xchg, .Add and .Sub"));
	28437	buf_sprintf("@atomicRmw with float only allowed with .Xchg, .Add and .Sub"));
28417	28438	return ira->codegen->invalid_inst_gen;
28418	28439	}
28419	28440
...	...	@@ -28434,14 +28455,103 @@ static IrInstGen ir_analyze_instruction_atomic_rmw(IrAnalyze ira, IrInstSrcAto
28434	28455	return ira->codegen->invalid_inst_gen;
28435	28456	}
28436	28457
28437		if (instr_is_comptime(casted_operand) && instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut == ConstPtrMutComptimeVar)
28438		{
28439		ir_add_error(ira, &instruction->base.base,
28440		buf_sprintf("compiler bug: TODO compile-time execution of @atomicRmw"));
28441		return ira->codegen->invalid_inst_gen;
	28458	// special case zero bit types
	28459	switch (type_has_one_possible_value(ira->codegen, operand_type)) {
	28460	case OnePossibleValueInvalid:
	28461	return ira->codegen->invalid_inst_gen;
	28462	case OnePossibleValueYes:
	28463	return ir_const_move(ira, &instruction->base.base, get_the_one_possible_value(ira->codegen, operand_type));
	28464	case OnePossibleValueNo:
	28465	break;
	28466	}
	28467
	28468	IrInst *source_inst = &instruction->base.base;
	28469	if (instr_is_comptime(casted_operand) && instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut == ConstPtrMutComptimeVar) {
	28470	ZigValue *ptr_val = ir_resolve_const(ira, casted_ptr, UndefBad);
	28471	if (ptr_val == nullptr)
	28472	return ira->codegen->invalid_inst_gen;
	28473
	28474	ZigValue *op1_val = const_ptr_pointee(ira, ira->codegen, ptr_val, instruction->base.base.source_node);
	28475	if (op1_val == nullptr)
	28476	return ira->codegen->invalid_inst_gen;
	28477
	28478	ZigValue *op2_val = ir_resolve_const(ira, casted_operand, UndefBad);
	28479	if (op2_val == nullptr)
	28480	return ira->codegen->invalid_inst_gen;
	28481
	28482	IrInstGen *result = ir_const(ira, source_inst, operand_type);
	28483	copy_const_val(ira->codegen, result->value, op1_val);
	28484	if (op == AtomicRmwOp_xchg) {
	28485	copy_const_val(ira->codegen, op1_val, op2_val);
	28486	return result;
	28487	}
	28488
	28489	if (operand_type->id == ZigTypeIdPointer \|\| operand_type->id == ZigTypeIdOptional) {
	28490	ir_add_error(ira, &instruction->ordering->base,
	28491	buf_sprintf("TODO comptime @atomicRmw with pointers other than .Xchg"));
	28492	return ira->codegen->invalid_inst_gen;
	28493	}
	28494
	28495	ErrorMsg *msg;
	28496	if (op == AtomicRmwOp_min \|\| op == AtomicRmwOp_max) {
	28497	IrBinOp bin_op;
	28498	if (op == AtomicRmwOp_min)
	28499	// store op2 if op2 < op1
	28500	bin_op = IrBinOpCmpGreaterThan;
	28501	else
	28502	// store op2 if op2 > op1
	28503	bin_op = IrBinOpCmpLessThan;
	28504
	28505	IrInstGen *dummy_value = ir_const(ira, source_inst, operand_type);
	28506	msg = ir_eval_bin_op_cmp_scalar(ira, source_inst, op1_val, bin_op, op2_val, dummy_value->value);
	28507	if (msg != nullptr) {
	28508	return ira->codegen->invalid_inst_gen;
	28509	}
	28510	if (dummy_value->value->data.x_bool)
	28511	copy_const_val(ira->codegen, op1_val, op2_val);
	28512	} else {
	28513	IrBinOp bin_op;
	28514	switch (op) {
	28515	case AtomicRmwOp_xchg:
	28516	case AtomicRmwOp_max:
	28517	case AtomicRmwOp_min:
	28518	zig_unreachable();
	28519	case AtomicRmwOp_add:
	28520	if (operand_type->id == ZigTypeIdFloat)
	28521	bin_op = IrBinOpAdd;
	28522	else
	28523	bin_op = IrBinOpAddWrap;
	28524	break;
	28525	case AtomicRmwOp_sub:
	28526	if (operand_type->id == ZigTypeIdFloat)
	28527	bin_op = IrBinOpSub;
	28528	else
	28529	bin_op = IrBinOpSubWrap;
	28530	break;
	28531	case AtomicRmwOp_and:
	28532	case AtomicRmwOp_nand:
	28533	bin_op = IrBinOpBinAnd;
	28534	break;
	28535	case AtomicRmwOp_or:
	28536	bin_op = IrBinOpBinOr;
	28537	break;
	28538	case AtomicRmwOp_xor:
	28539	bin_op = IrBinOpBinXor;
	28540	break;
	28541	}
	28542	msg = ir_eval_math_op_scalar(ira, source_inst, operand_type, op1_val, bin_op, op2_val, op1_val);
	28543	if (msg != nullptr) {
	28544	return ira->codegen->invalid_inst_gen;
	28545	}
	28546	if (op == AtomicRmwOp_nand) {
	28547	bigint_not(&op1_val->data.x_bigint, &op1_val->data.x_bigint,
	28548	operand_type->data.integral.bit_count, operand_type->data.integral.is_signed);
	28549	}
	28550	}
	28551	return result;
28442	28552	}
28443	28553
28444		return ir_build_atomic_rmw_gen(ira, &instruction->base.base, casted_ptr, casted_operand, op,
	28554	return ir_build_atomic_rmw_gen(ira, source_inst, casted_ptr, casted_operand, op,
28445	28555	ordering, operand_type);
28446	28556	}
28447	28557
...	...	@@ -28513,6 +28623,16 @@ static IrInstGen ir_analyze_instruction_atomic_store(IrAnalyze ira, IrInstSrcA
28513	28623	return ira->codegen->invalid_inst_gen;
28514	28624	}
28515	28625
	28626	// special case zero bit types
	28627	switch (type_has_one_possible_value(ira->codegen, operand_type)) {
	28628	case OnePossibleValueInvalid:
	28629	return ira->codegen->invalid_inst_gen;
	28630	case OnePossibleValueYes:
	28631	return ir_const_void(ira, &instruction->base.base);
	28632	case OnePossibleValueNo:
	28633	break;
	28634	}
	28635
28516	28636	if (instr_is_comptime(casted_value) && instr_is_comptime(casted_ptr)) {
28517	28637	IrInstGen *result = ir_analyze_store_ptr(ira, &instruction->base.base, casted_ptr, value, false);
28518	28638	result->value->type = ira->codegen->builtin_types.entry_void;

test/compile_errors.zig+11-2

...	...	@@ -49,6 +49,15 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
49	49	"tmp.zig:5:9: error: resume in noasync scope",
50	50	});
51	51
	52	cases.add("atomicrmw with bool op not .Xchg",
	53	\\export fn entry() void {
	54	\\ var x = false;
	55	\\ _ = @atomicRmw(bool, &x, .Add, true, .SeqCst);
	56	\\}
	57	, &[_][]const u8{
	58	"tmp.zig:3:30: error: @atomicRmw with bool only allowed with .Xchg",
	59	});
	60
52	61	cases.addTest("@TypeOf with no arguments",
53	62	\\export fn entry() void {
54	63	\\ _ = @TypeOf();
...	...	@@ -357,7 +366,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
357	366	\\ _ = @atomicRmw(f32, &x, .And, 2, .SeqCst);
358	367	\\}
359	368	, &[_][]const u8{
360		"tmp.zig:3:29: error: @atomicRmw with float only works with .Xchg, .Add and .Sub",
	369	"tmp.zig:3:29: error: @atomicRmw with float only allowed with .Xchg, .Add and .Sub",
361	370	});
362	371
363	372	cases.add("intToPtr with misaligned address",
...	...	@@ -574,7 +583,7 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
574	583	\\ _ = @atomicRmw(E, &x, .Add, .b, .SeqCst);
575	584	\\}
576	585	, &[_][]const u8{
577		"tmp.zig:9:27: error: @atomicRmw on enum only works with .Xchg",
	586	"tmp.zig:9:27: error: @atomicRmw with enum only allowed with .Xchg",
578	587	});
579	588
580	589	cases.add("disallow coercion from non-null-terminated pointer to null-terminated pointer",

test/stage1/behavior/atomics.zig+70-13

...	...	@@ -2,29 +2,32 @@ const std = @import("std");
2	2	const expect = std.testing.expect;
3	3	const expectEqual = std.testing.expectEqual;
4	4	const builtin = @import("builtin");
5		const AtomicRmwOp = builtin.AtomicRmwOp;
6		const AtomicOrder = builtin.AtomicOrder;
7	5
8	6	test "cmpxchg" {
	7	testCmpxchg();
	8	comptime testCmpxchg();
	9	}
	10
	11	fn testCmpxchg() void {
9	12	var x: i32 = 1234;
10		if (@cmpxchgWeak(i32, &x, 99, 5678, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) \|x1\| {
	13	if (@cmpxchgWeak(i32, &x, 99, 5678, .SeqCst, .SeqCst)) \|x1\| {
11	14	expect(x1 == 1234);
12	15	} else {
13	16	@panic("cmpxchg should have failed");
14	17	}
15	18
16		while (@cmpxchgWeak(i32, &x, 1234, 5678, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) \|x1\| {
	19	while (@cmpxchgWeak(i32, &x, 1234, 5678, .SeqCst, .SeqCst)) \|x1\| {
17	20	expect(x1 == 1234);
18	21	}
19	22	expect(x == 5678);
20	23
21		expect(@cmpxchgStrong(i32, &x, 5678, 42, AtomicOrder.SeqCst, AtomicOrder.SeqCst) == null);
	24	expect(@cmpxchgStrong(i32, &x, 5678, 42, .SeqCst, .SeqCst) == null);
22	25	expect(x == 42);
23	26	}
24	27
25	28	test "fence" {
26	29	var x: i32 = 1234;
27		@fence(AtomicOrder.SeqCst);
	30	@fence(.SeqCst);
28	31	x = 5678;
29	32	}
30	33
...	...	@@ -36,18 +39,18 @@ test "atomicrmw and atomicload" {
36	39	}
37	40
38	41	fn testAtomicRmw(ptr: *u8) void {
39		const prev_value = @atomicRmw(u8, ptr, AtomicRmwOp.Xchg, 42, AtomicOrder.SeqCst);
	42	const prev_value = @atomicRmw(u8, ptr, .Xchg, 42, .SeqCst);
40	43	expect(prev_value == 200);
41	44	comptime {
42	45	var x: i32 = 1234;
43	46	const y: i32 = 12345;
44		expect(@atomicLoad(i32, &x, AtomicOrder.SeqCst) == 1234);
45		expect(@atomicLoad(i32, &y, AtomicOrder.SeqCst) == 12345);
	47	expect(@atomicLoad(i32, &x, .SeqCst) == 1234);
	48	expect(@atomicLoad(i32, &y, .SeqCst) == 12345);
46	49	}
47	50	}
48	51
49	52	fn testAtomicLoad(ptr: *u8) void {
50		const x = @atomicLoad(u8, ptr, AtomicOrder.SeqCst);
	53	const x = @atomicLoad(u8, ptr, .SeqCst);
51	54	expect(x == 42);
52	55	}
53	56
...	...	@@ -56,18 +59,18 @@ test "cmpxchg with ptr" {
56	59	var data2: i32 = 5678;
57	60	var data3: i32 = 9101;
58	61	var x: *i32 = &data1;
59		if (@cmpxchgWeak(*i32, &x, &data2, &data3, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) \|x1\| {
	62	if (@cmpxchgWeak(*i32, &x, &data2, &data3, .SeqCst, .SeqCst)) \|x1\| {
60	63	expect(x1 == &data1);
61	64	} else {
62	65	@panic("cmpxchg should have failed");
63	66	}
64	67
65		while (@cmpxchgWeak(*i32, &x, &data1, &data3, AtomicOrder.SeqCst, AtomicOrder.SeqCst)) \|x1\| {
	68	while (@cmpxchgWeak(*i32, &x, &data1, &data3, .SeqCst, .SeqCst)) \|x1\| {
66	69	expect(x1 == &data1);
67	70	}
68	71	expect(x == &data3);
69	72
70		expect(@cmpxchgStrong(*i32, &x, &data3, &data2, AtomicOrder.SeqCst, AtomicOrder.SeqCst) == null);
	73	expect(@cmpxchgStrong(*i32, &x, &data3, &data2, .SeqCst, .SeqCst) == null);
71	74	expect(x == &data2);
72	75	}
73	76
...	...	@@ -146,9 +149,11 @@ fn testAtomicStore() void {
146	149	}
147	150
148	151	test "atomicrmw with floats" {
	152	// TODO https://github.com/ziglang/zig/issues/4457
149	153	if (builtin.arch == .aarch64 or builtin.arch == .arm or builtin.arch == .riscv64)
150	154	return error.SkipZigTest;
151	155	testAtomicRmwFloat();
	156	comptime testAtomicRmwFloat();
152	157	}
153	158
154	159	fn testAtomicRmwFloat() void {
...	...	@@ -161,3 +166,55 @@ fn testAtomicRmwFloat() void {
161	166	_ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst);
162	167	expect(x == 4);
163	168	}
	169
	170	test "atomicrmw with ints" {
	171	testAtomicRmwInt();
	172	comptime testAtomicRmwInt();
	173	}
	174
	175	fn testAtomicRmwInt() void {
	176	var x: u8 = 1;
	177	var res = @atomicRmw(u8, &x, .Xchg, 3, .SeqCst);
	178	expect(x == 3 and res == 1);
	179	_ = @atomicRmw(u8, &x, .Add, 3, .SeqCst);
	180	expect(x == 6);
	181	_ = @atomicRmw(u8, &x, .Sub, 1, .SeqCst);
	182	expect(x == 5);
	183	_ = @atomicRmw(u8, &x, .And, 4, .SeqCst);
	184	expect(x == 4);
	185	_ = @atomicRmw(u8, &x, .Nand, 4, .SeqCst);
	186	expect(x == 0xfb);
	187	_ = @atomicRmw(u8, &x, .Or, 6, .SeqCst);
	188	expect(x == 0xff);
	189	_ = @atomicRmw(u8, &x, .Xor, 2, .SeqCst);
	190	expect(x == 0xfd);
	191
	192	// TODO https://github.com/ziglang/zig/issues/4724
	193	if (builtin.arch == .mipsel) return;
	194	_ = @atomicRmw(u8, &x, .Max, 1, .SeqCst);
	195	expect(x == 0xfd);
	196	_ = @atomicRmw(u8, &x, .Min, 1, .SeqCst);
	197	expect(x == 1);
	198	}
	199
	200
	201	test "atomics with different types" {
	202	testAtomicsWithType(bool, true, false);
	203	inline for (.{ u1, i5, u15 }) \|T\| {
	204	var x: T = 0;
	205	testAtomicsWithType(T, 0, 1);
	206	}
	207	testAtomicsWithType(u0, 0, 0);
	208	testAtomicsWithType(i0, 0, 0);
	209	}
	210
	211	fn testAtomicsWithType(comptime T: type, a: T, b: T) void {
	212	var x: T = b;
	213	@atomicStore(T, &x, a, .SeqCst);
	214	expect(x == a);
	215	expect(@atomicLoad(T, &x, .SeqCst) == a);
	216	expect(@atomicRmw(T, &x, .Xchg, b, .SeqCst) == a);
	217	expect(@cmpxchgStrong(T, &x, b, a, .SeqCst, .SeqCst) == null);
	218	if (@sizeOf(T) != 0)
	219	expect(@cmpxchgStrong(T, &x, b, a, .SeqCst, .SeqCst).? == a);
	220	}