authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-11-22 18:49:18-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-11-22 19:08:55-07:00
log70931dbdea96d92feb60406c827e39e566317863
treeee569fab186e848d73de1e4dbc272ff5f9f6b7c1
parentedb2f72988cd180c5d87b03481fa1c20b3325968

rework std.atomic

* move std.atomic.Atomic to std.atomic.Value * fix incorrect argument order passed to testing.expectEqual * make the functions be a thin wrapper over the atomic builtins and stick to the naming conventions. * remove pointless functions loadUnchecked and storeUnchecked. Instead, name the field `raw` instead of `value` (which is redundant with the type name). * simplify the tests by not passing every possible combination. Many cases were iterating over every possible combinations but then not even using the for loop element value! * remove the redundant compile errors which are already implemented by the language itself. * remove dead x86 inline assembly. this should be implemented in the language if at all.

15 files changed, 448 insertions(+), 712 deletions(-)

CMakeLists.txt-1
...@@ -209,7 +209,6 @@ set(ZIG_STAGE2_SOURCES...@@ -209,7 +209,6 @@ set(ZIG_STAGE2_SOURCES
209 "${CMAKE_SOURCE_DIR}/lib/std/array_list.zig"209 "${CMAKE_SOURCE_DIR}/lib/std/array_list.zig"
210 "${CMAKE_SOURCE_DIR}/lib/std/ascii.zig"210 "${CMAKE_SOURCE_DIR}/lib/std/ascii.zig"
211 "${CMAKE_SOURCE_DIR}/lib/std/atomic.zig"211 "${CMAKE_SOURCE_DIR}/lib/std/atomic.zig"
212 "${CMAKE_SOURCE_DIR}/lib/std/atomic/Atomic.zig"
213 "${CMAKE_SOURCE_DIR}/lib/std/base64.zig"212 "${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
214 "${CMAKE_SOURCE_DIR}/lib/std/BitStack.zig"213 "${CMAKE_SOURCE_DIR}/lib/std/BitStack.zig"
215 "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"214 "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
lib/std/Thread.zig+7-8
...@@ -8,7 +8,6 @@ const math = std.math;...@@ -8,7 +8,6 @@ const math = std.math;
8const os = std.os;8const os = std.os;
9const assert = std.debug.assert;9const assert = std.debug.assert;
10const target = builtin.target;10const target = builtin.target;
11const Atomic = std.atomic.Atomic;
1211
13pub const Futex = @import("Thread/Futex.zig");12pub const Futex = @import("Thread/Futex.zig");
14pub const ResetEvent = @import("Thread/ResetEvent.zig");13pub const ResetEvent = @import("Thread/ResetEvent.zig");
...@@ -388,7 +387,7 @@ pub fn yield() YieldError!void {...@@ -388,7 +387,7 @@ pub fn yield() YieldError!void {
388}387}
389388
390/// State to synchronize detachment of spawner thread to spawned thread389/// State to synchronize detachment of spawner thread to spawned thread
391const Completion = Atomic(enum(u8) {390const Completion = std.atomic.Value(enum(u8) {
392 running,391 running,
393 detached,392 detached,
394 completed,393 completed,
...@@ -746,7 +745,7 @@ const WasiThreadImpl = struct {...@@ -746,7 +745,7 @@ const WasiThreadImpl = struct {
746745
747 const WasiThread = struct {746 const WasiThread = struct {
748 /// Thread ID747 /// Thread ID
749 tid: Atomic(i32) = Atomic(i32).init(0),748 tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(0),
750 /// Contains all memory which was allocated to bootstrap this thread, including:749 /// Contains all memory which was allocated to bootstrap this thread, including:
751 /// - Guard page750 /// - Guard page
752 /// - Stack751 /// - Stack
...@@ -784,7 +783,7 @@ const WasiThreadImpl = struct {...@@ -784,7 +783,7 @@ const WasiThreadImpl = struct {
784 original_stack_pointer: [*]u8,783 original_stack_pointer: [*]u8,
785 };784 };
786785
787 const State = Atomic(enum(u8) { running, completed, detached });786 const State = std.atomic.Value(enum(u8) { running, completed, detached });
788787
789 fn getCurrentId() Id {788 fn getCurrentId() Id {
790 return tls_thread_id;789 return tls_thread_id;
...@@ -1048,7 +1047,7 @@ const LinuxThreadImpl = struct {...@@ -1048,7 +1047,7 @@ const LinuxThreadImpl = struct {
10481047
1049 const ThreadCompletion = struct {1048 const ThreadCompletion = struct {
1050 completion: Completion = Completion.init(.running),1049 completion: Completion = Completion.init(.running),
1051 child_tid: Atomic(i32) = Atomic(i32).init(1),1050 child_tid: std.atomic.Value(i32) = std.atomic.Value(i32).init(1),
1052 parent_tid: i32 = undefined,1051 parent_tid: i32 = undefined,
1053 mapped: []align(std.mem.page_size) u8,1052 mapped: []align(std.mem.page_size) u8,
10541053
...@@ -1304,7 +1303,7 @@ const LinuxThreadImpl = struct {...@@ -1304,7 +1303,7 @@ const LinuxThreadImpl = struct {
1304 @intFromPtr(instance),1303 @intFromPtr(instance),
1305 &instance.thread.parent_tid,1304 &instance.thread.parent_tid,
1306 tls_ptr,1305 tls_ptr,
1307 &instance.thread.child_tid.value,1306 &instance.thread.child_tid.raw,
1308 ))) {1307 ))) {
1309 .SUCCESS => return Impl{ .thread = &instance.thread },1308 .SUCCESS => return Impl{ .thread = &instance.thread },
1310 .AGAIN => return error.ThreadQuotaExceeded,1309 .AGAIN => return error.ThreadQuotaExceeded,
...@@ -1346,7 +1345,7 @@ const LinuxThreadImpl = struct {...@@ -1346,7 +1345,7 @@ const LinuxThreadImpl = struct {
1346 }1345 }
13471346
1348 switch (linux.getErrno(linux.futex_wait(1347 switch (linux.getErrno(linux.futex_wait(
1349 &self.thread.child_tid.value,1348 &self.thread.child_tid.raw,
1350 linux.FUTEX.WAIT,1349 linux.FUTEX.WAIT,
1351 tid,1350 tid,
1352 null,1351 null,
...@@ -1387,7 +1386,7 @@ test "setName, getName" {...@@ -1387,7 +1386,7 @@ test "setName, getName" {
1387 test_done_event: ResetEvent = .{},1386 test_done_event: ResetEvent = .{},
1388 thread_done_event: ResetEvent = .{},1387 thread_done_event: ResetEvent = .{},
13891388
1390 done: std.atomic.Atomic(bool) = std.atomic.Atomic(bool).init(false),1389 done: std.atomic.Value(bool) = std.atomic.Value(bool).init(false),
1391 thread: Thread = undefined,1390 thread: Thread = undefined,
13921391
1393 pub fn run(ctx: *@This()) !void {1392 pub fn run(ctx: *@This()) !void {
lib/std/Thread/Condition.zig+6-7
...@@ -50,7 +50,6 @@ const Mutex = std.Thread.Mutex;...@@ -50,7 +50,6 @@ const Mutex = std.Thread.Mutex;
50const os = std.os;50const os = std.os;
51const assert = std.debug.assert;51const assert = std.debug.assert;
52const testing = std.testing;52const testing = std.testing;
53const Atomic = std.atomic.Atomic;
54const Futex = std.Thread.Futex;53const Futex = std.Thread.Futex;
5554
56impl: Impl = .{},55impl: Impl = .{},
...@@ -193,8 +192,8 @@ const WindowsImpl = struct {...@@ -193,8 +192,8 @@ const WindowsImpl = struct {
193};192};
194193
195const FutexImpl = struct {194const FutexImpl = struct {
196 state: Atomic(u32) = Atomic(u32).init(0),195 state: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
197 epoch: Atomic(u32) = Atomic(u32).init(0),196 epoch: std.atomic.Value(u32) = std.atomic.Value(u32).init(0),
198197
199 const one_waiter = 1;198 const one_waiter = 1;
200 const waiter_mask = 0xffff;199 const waiter_mask = 0xffff;
...@@ -232,12 +231,12 @@ const FutexImpl = struct {...@@ -232,12 +231,12 @@ const FutexImpl = struct {
232 // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.231 // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
233 while (state & signal_mask != 0) {232 while (state & signal_mask != 0) {
234 const new_state = state - one_waiter - one_signal;233 const new_state = state - one_waiter - one_signal;
235 state = self.state.tryCompareAndSwap(state, new_state, .Acquire, .Monotonic) orelse return;234 state = self.state.cmpxchgWeak(state, new_state, .Acquire, .Monotonic) orelse return;
236 }235 }
237236
238 // Remove the waiter we added and officially return timed out.237 // Remove the waiter we added and officially return timed out.
239 const new_state = state - one_waiter;238 const new_state = state - one_waiter;
240 state = self.state.tryCompareAndSwap(state, new_state, .Monotonic, .Monotonic) orelse return err;239 state = self.state.cmpxchgWeak(state, new_state, .Monotonic, .Monotonic) orelse return err;
241 }240 }
242 },241 },
243 };242 };
...@@ -249,7 +248,7 @@ const FutexImpl = struct {...@@ -249,7 +248,7 @@ const FutexImpl = struct {
249 // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.248 // Acquire barrier ensures code before the wake() which added the signal happens before we decrement it and return.
250 while (state & signal_mask != 0) {249 while (state & signal_mask != 0) {
251 const new_state = state - one_waiter - one_signal;250 const new_state = state - one_waiter - one_signal;
252 state = self.state.tryCompareAndSwap(state, new_state, .Acquire, .Monotonic) orelse return;251 state = self.state.cmpxchgWeak(state, new_state, .Acquire, .Monotonic) orelse return;
253 }252 }
254 }253 }
255 }254 }
...@@ -276,7 +275,7 @@ const FutexImpl = struct {...@@ -276,7 +275,7 @@ const FutexImpl = struct {
276 // Reserve the amount of waiters to wake by incrementing the signals count.275 // Reserve the amount of waiters to wake by incrementing the signals count.
277 // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.276 // Release barrier ensures code before the wake() happens before the signal it posted and consumed by the wait() threads.
278 const new_state = state + (one_signal * to_wake);277 const new_state = state + (one_signal * to_wake);
279 state = self.state.tryCompareAndSwap(state, new_state, .Release, .Monotonic) orelse {278 state = self.state.cmpxchgWeak(state, new_state, .Release, .Monotonic) orelse {
280 // Wake up the waiting threads we reserved above by changing the epoch value.279 // Wake up the waiting threads we reserved above by changing the epoch value.
281 // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.280 // NOTE: a waiting thread could miss a wake up if *exactly* ((1<<32)-1) wake()s happen between it observing the epoch and sleeping on it.
282 // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.281 // This is very unlikely due to how many precise amount of Futex.wake() calls that would be between the waiting thread's potential preemption.
lib/std/Thread/Futex.zig+37-37
...@@ -10,7 +10,7 @@ const Futex = @This();...@@ -10,7 +10,7 @@ const Futex = @This();
10const os = std.os;10const os = std.os;
11const assert = std.debug.assert;11const assert = std.debug.assert;
12const testing = std.testing;12const testing = std.testing;
13const Atomic = std.atomic.Atomic;13const atomic = std.atomic;
1414
15/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:15/// Checks if `ptr` still contains the value `expect` and, if so, blocks the caller until either:
16/// - The value at `ptr` is no longer equal to `expect`.16/// - The value at `ptr` is no longer equal to `expect`.
...@@ -19,7 +19,7 @@ const Atomic = std.atomic.Atomic;...@@ -19,7 +19,7 @@ const Atomic = std.atomic.Atomic;
19///19///
20/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically20/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
21/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.21/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
22pub fn wait(ptr: *const Atomic(u32), expect: u32) void {22pub fn wait(ptr: *const atomic.Value(u32), expect: u32) void {
23 @setCold(true);23 @setCold(true);
2424
25 Impl.wait(ptr, expect, null) catch |err| switch (err) {25 Impl.wait(ptr, expect, null) catch |err| switch (err) {
...@@ -35,7 +35,7 @@ pub fn wait(ptr: *const Atomic(u32), expect: u32) void {...@@ -35,7 +35,7 @@ pub fn wait(ptr: *const Atomic(u32), expect: u32) void {
35///35///
36/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically36/// The checking of `ptr` and `expect`, along with blocking the caller, is done atomically
37/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.37/// and totally ordered (sequentially consistent) with respect to other wait()/wake() calls on the same `ptr`.
38pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {38pub fn timedWait(ptr: *const atomic.Value(u32), expect: u32, timeout_ns: u64) error{Timeout}!void {
39 @setCold(true);39 @setCold(true);
4040
41 // Avoid calling into the OS for no-op timeouts.41 // Avoid calling into the OS for no-op timeouts.
...@@ -48,7 +48,7 @@ pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Ti...@@ -48,7 +48,7 @@ pub fn timedWait(ptr: *const Atomic(u32), expect: u32, timeout_ns: u64) error{Ti
48}48}
4949
50/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.50/// Unblocks at most `max_waiters` callers blocked in a `wait()` call on `ptr`.
51pub fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {51pub fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
52 @setCold(true);52 @setCold(true);
5353
54 // Avoid calling into the OS if there's nothing to wake up.54 // Avoid calling into the OS if there's nothing to wake up.
...@@ -83,11 +83,11 @@ else...@@ -83,11 +83,11 @@ else
83/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.83/// We can't do @compileError() in the `Impl` switch statement above as its eagerly evaluated.
84/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.84/// So instead, we @compileError() on the methods themselves for platforms which don't support futex.
85const UnsupportedImpl = struct {85const UnsupportedImpl = struct {
86 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {86 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
87 return unsupported(.{ ptr, expect, timeout });87 return unsupported(.{ ptr, expect, timeout });
88 }88 }
8989
90 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {90 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
91 return unsupported(.{ ptr, max_waiters });91 return unsupported(.{ ptr, max_waiters });
92 }92 }
9393
...@@ -98,8 +98,8 @@ const UnsupportedImpl = struct {...@@ -98,8 +98,8 @@ const UnsupportedImpl = struct {
98};98};
9999
100const SingleThreadedImpl = struct {100const SingleThreadedImpl = struct {
101 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {101 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
102 if (ptr.loadUnchecked() != expect) {102 if (ptr.raw != expect) {
103 return;103 return;
104 }104 }
105105
...@@ -113,7 +113,7 @@ const SingleThreadedImpl = struct {...@@ -113,7 +113,7 @@ const SingleThreadedImpl = struct {
113 return error.Timeout;113 return error.Timeout;
114 }114 }
115115
116 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {116 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
117 // There are no other threads to possibly wake up117 // There are no other threads to possibly wake up
118 _ = ptr;118 _ = ptr;
119 _ = max_waiters;119 _ = max_waiters;
...@@ -123,7 +123,7 @@ const SingleThreadedImpl = struct {...@@ -123,7 +123,7 @@ const SingleThreadedImpl = struct {
123// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll123// We use WaitOnAddress through NtDll instead of API-MS-Win-Core-Synch-l1-2-0.dll
124// as it's generally already a linked target and is autoloaded into all processes anyway.124// as it's generally already a linked target and is autoloaded into all processes anyway.
125const WindowsImpl = struct {125const WindowsImpl = struct {
126 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {126 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
127 var timeout_value: os.windows.LARGE_INTEGER = undefined;127 var timeout_value: os.windows.LARGE_INTEGER = undefined;
128 var timeout_ptr: ?*const os.windows.LARGE_INTEGER = null;128 var timeout_ptr: ?*const os.windows.LARGE_INTEGER = null;
129129
...@@ -152,7 +152,7 @@ const WindowsImpl = struct {...@@ -152,7 +152,7 @@ const WindowsImpl = struct {
152 }152 }
153 }153 }
154154
155 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {155 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
156 const address: ?*const anyopaque = ptr;156 const address: ?*const anyopaque = ptr;
157 assert(max_waiters != 0);157 assert(max_waiters != 0);
158158
...@@ -164,7 +164,7 @@ const WindowsImpl = struct {...@@ -164,7 +164,7 @@ const WindowsImpl = struct {
164};164};
165165
166const DarwinImpl = struct {166const DarwinImpl = struct {
167 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {167 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
168 // Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:168 // Darwin XNU 7195.50.7.100.1 introduced __ulock_wait2 and migrated code paths (notably pthread_cond_t) towards it:
169 // https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6169 // https://github.com/apple/darwin-xnu/commit/d4061fb0260b3ed486147341b72468f836ed6c8f#diff-08f993cc40af475663274687b7c326cc6c3031e0db3ac8de7b24624610616be6
170 //170 //
...@@ -220,7 +220,7 @@ const DarwinImpl = struct {...@@ -220,7 +220,7 @@ const DarwinImpl = struct {
220 }220 }
221 }221 }
222222
223 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {223 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
224 var flags: u32 = os.darwin.UL_COMPARE_AND_WAIT | os.darwin.ULF_NO_ERRNO;224 var flags: u32 = os.darwin.UL_COMPARE_AND_WAIT | os.darwin.ULF_NO_ERRNO;
225 if (max_waiters > 1) {225 if (max_waiters > 1) {
226 flags |= os.darwin.ULF_WAKE_ALL;226 flags |= os.darwin.ULF_WAKE_ALL;
...@@ -244,7 +244,7 @@ const DarwinImpl = struct {...@@ -244,7 +244,7 @@ const DarwinImpl = struct {
244244
245// https://man7.org/linux/man-pages/man2/futex.2.html245// https://man7.org/linux/man-pages/man2/futex.2.html
246const LinuxImpl = struct {246const LinuxImpl = struct {
247 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {247 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
248 var ts: os.timespec = undefined;248 var ts: os.timespec = undefined;
249 if (timeout) |timeout_ns| {249 if (timeout) |timeout_ns| {
250 ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));250 ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));
...@@ -252,7 +252,7 @@ const LinuxImpl = struct {...@@ -252,7 +252,7 @@ const LinuxImpl = struct {
252 }252 }
253253
254 const rc = os.linux.futex_wait(254 const rc = os.linux.futex_wait(
255 @as(*const i32, @ptrCast(&ptr.value)),255 @as(*const i32, @ptrCast(&ptr.raw)),
256 os.linux.FUTEX.PRIVATE_FLAG | os.linux.FUTEX.WAIT,256 os.linux.FUTEX.PRIVATE_FLAG | os.linux.FUTEX.WAIT,
257 @as(i32, @bitCast(expect)),257 @as(i32, @bitCast(expect)),
258 if (timeout != null) &ts else null,258 if (timeout != null) &ts else null,
...@@ -272,9 +272,9 @@ const LinuxImpl = struct {...@@ -272,9 +272,9 @@ const LinuxImpl = struct {
272 }272 }
273 }273 }
274274
275 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {275 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
276 const rc = os.linux.futex_wake(276 const rc = os.linux.futex_wake(
277 @as(*const i32, @ptrCast(&ptr.value)),277 @as(*const i32, @ptrCast(&ptr.raw)),
278 os.linux.FUTEX.PRIVATE_FLAG | os.linux.FUTEX.WAKE,278 os.linux.FUTEX.PRIVATE_FLAG | os.linux.FUTEX.WAKE,
279 std.math.cast(i32, max_waiters) orelse std.math.maxInt(i32),279 std.math.cast(i32, max_waiters) orelse std.math.maxInt(i32),
280 );280 );
...@@ -290,7 +290,7 @@ const LinuxImpl = struct {...@@ -290,7 +290,7 @@ const LinuxImpl = struct {
290290
291// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1291// https://www.freebsd.org/cgi/man.cgi?query=_umtx_op&sektion=2&n=1
292const FreebsdImpl = struct {292const FreebsdImpl = struct {
293 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {293 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
294 var tm_size: usize = 0;294 var tm_size: usize = 0;
295 var tm: os.freebsd._umtx_time = undefined;295 var tm: os.freebsd._umtx_time = undefined;
296 var tm_ptr: ?*const os.freebsd._umtx_time = null;296 var tm_ptr: ?*const os.freebsd._umtx_time = null;
...@@ -326,7 +326,7 @@ const FreebsdImpl = struct {...@@ -326,7 +326,7 @@ const FreebsdImpl = struct {
326 }326 }
327 }327 }
328328
329 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {329 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
330 const rc = os.freebsd._umtx_op(330 const rc = os.freebsd._umtx_op(
331 @intFromPtr(&ptr.value),331 @intFromPtr(&ptr.value),
332 @intFromEnum(os.freebsd.UMTX_OP.WAKE_PRIVATE),332 @intFromEnum(os.freebsd.UMTX_OP.WAKE_PRIVATE),
...@@ -346,7 +346,7 @@ const FreebsdImpl = struct {...@@ -346,7 +346,7 @@ const FreebsdImpl = struct {
346346
347// https://man.openbsd.org/futex.2347// https://man.openbsd.org/futex.2
348const OpenbsdImpl = struct {348const OpenbsdImpl = struct {
349 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {349 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
350 var ts: os.timespec = undefined;350 var ts: os.timespec = undefined;
351 if (timeout) |timeout_ns| {351 if (timeout) |timeout_ns| {
352 ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));352 ts.tv_sec = @as(@TypeOf(ts.tv_sec), @intCast(timeout_ns / std.time.ns_per_s));
...@@ -377,7 +377,7 @@ const OpenbsdImpl = struct {...@@ -377,7 +377,7 @@ const OpenbsdImpl = struct {
377 }377 }
378 }378 }
379379
380 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {380 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
381 const rc = os.openbsd.futex(381 const rc = os.openbsd.futex(
382 @as(*const volatile u32, @ptrCast(&ptr.value)),382 @as(*const volatile u32, @ptrCast(&ptr.value)),
383 os.openbsd.FUTEX_WAKE | os.openbsd.FUTEX_PRIVATE_FLAG,383 os.openbsd.FUTEX_WAKE | os.openbsd.FUTEX_PRIVATE_FLAG,
...@@ -393,7 +393,7 @@ const OpenbsdImpl = struct {...@@ -393,7 +393,7 @@ const OpenbsdImpl = struct {
393393
394// https://man.dragonflybsd.org/?command=umtx&section=2394// https://man.dragonflybsd.org/?command=umtx&section=2
395const DragonflyImpl = struct {395const DragonflyImpl = struct {
396 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {396 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
397 // Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.397 // Dragonfly uses a scheme where 0 timeout means wait until signaled or spurious wake.
398 // It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.398 // It's reporting of timeout's is also unrealiable so we use an external timing source (Timer) instead.
399 var timeout_us: c_int = 0;399 var timeout_us: c_int = 0;
...@@ -435,7 +435,7 @@ const DragonflyImpl = struct {...@@ -435,7 +435,7 @@ const DragonflyImpl = struct {
435 }435 }
436 }436 }
437437
438 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {438 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
439 // A count of zero means wake all waiters.439 // A count of zero means wake all waiters.
440 assert(max_waiters != 0);440 assert(max_waiters != 0);
441 const to_wake = std.math.cast(c_int, max_waiters) orelse 0;441 const to_wake = std.math.cast(c_int, max_waiters) orelse 0;
...@@ -449,7 +449,7 @@ const DragonflyImpl = struct {...@@ -449,7 +449,7 @@ const DragonflyImpl = struct {
449};449};
450450
451const WasmImpl = struct {451const WasmImpl = struct {
452 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {452 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
453 if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {453 if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {
454 @compileError("WASI target missing cpu feature 'atomics'");454 @compileError("WASI target missing cpu feature 'atomics'");
455 }455 }
...@@ -473,7 +473,7 @@ const WasmImpl = struct {...@@ -473,7 +473,7 @@ const WasmImpl = struct {
473 }473 }
474 }474 }
475475
476 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {476 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
477 if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {477 if (!comptime std.Target.wasm.featureSetHas(builtin.target.cpu.features, .atomics)) {
478 @compileError("WASI target missing cpu feature 'atomics'");478 @compileError("WASI target missing cpu feature 'atomics'");
479 }479 }
...@@ -732,8 +732,8 @@ const PosixImpl = struct {...@@ -732,8 +732,8 @@ const PosixImpl = struct {
732 };732 };
733733
734 const Bucket = struct {734 const Bucket = struct {
735 mutex: std.c.pthread_mutex_t align(std.atomic.cache_line) = .{},735 mutex: std.c.pthread_mutex_t align(atomic.cache_line) = .{},
736 pending: Atomic(usize) = Atomic(usize).init(0),736 pending: atomic.Value(usize) = atomic.Value(usize).init(0),
737 treap: Treap = .{},737 treap: Treap = .{},
738738
739 // Global array of buckets that addresses map to.739 // Global array of buckets that addresses map to.
...@@ -757,9 +757,9 @@ const PosixImpl = struct {...@@ -757,9 +757,9 @@ const PosixImpl = struct {
757 };757 };
758758
759 const Address = struct {759 const Address = struct {
760 fn from(ptr: *const Atomic(u32)) usize {760 fn from(ptr: *const atomic.Value(u32)) usize {
761 // Get the alignment of the pointer.761 // Get the alignment of the pointer.
762 const alignment = @alignOf(Atomic(u32));762 const alignment = @alignOf(atomic.Value(u32));
763 comptime assert(std.math.isPowerOfTwo(alignment));763 comptime assert(std.math.isPowerOfTwo(alignment));
764764
765 // Make sure the pointer is aligned,765 // Make sure the pointer is aligned,
...@@ -770,7 +770,7 @@ const PosixImpl = struct {...@@ -770,7 +770,7 @@ const PosixImpl = struct {
770 }770 }
771 };771 };
772772
773 fn wait(ptr: *const Atomic(u32), expect: u32, timeout: ?u64) error{Timeout}!void {773 fn wait(ptr: *const atomic.Value(u32), expect: u32, timeout: ?u64) error{Timeout}!void {
774 const address = Address.from(ptr);774 const address = Address.from(ptr);
775 const bucket = Bucket.from(address);775 const bucket = Bucket.from(address);
776776
...@@ -831,7 +831,7 @@ const PosixImpl = struct {...@@ -831,7 +831,7 @@ const PosixImpl = struct {
831 };831 };
832 }832 }
833833
834 fn wake(ptr: *const Atomic(u32), max_waiters: u32) void {834 fn wake(ptr: *const atomic.Value(u32), max_waiters: u32) void {
835 const address = Address.from(ptr);835 const address = Address.from(ptr);
836 const bucket = Bucket.from(address);836 const bucket = Bucket.from(address);
837837
...@@ -882,7 +882,7 @@ const PosixImpl = struct {...@@ -882,7 +882,7 @@ const PosixImpl = struct {
882};882};
883883
884test "Futex - smoke test" {884test "Futex - smoke test" {
885 var value = Atomic(u32).init(0);885 var value = atomic.Value(u32).init(0);
886886
887 // Try waits with invalid values.887 // Try waits with invalid values.
888 Futex.wait(&value, 0xdeadbeef);888 Futex.wait(&value, 0xdeadbeef);
...@@ -908,7 +908,7 @@ test "Futex - signaling" {...@@ -908,7 +908,7 @@ test "Futex - signaling" {
908 const num_iterations = 4;908 const num_iterations = 4;
909909
910 const Paddle = struct {910 const Paddle = struct {
911 value: Atomic(u32) = Atomic(u32).init(0),911 value: atomic.Value(u32) = atomic.Value(u32).init(0),
912 current: u32 = 0,912 current: u32 = 0,
913913
914 fn hit(self: *@This()) void {914 fn hit(self: *@This()) void {
...@@ -962,8 +962,8 @@ test "Futex - broadcasting" {...@@ -962,8 +962,8 @@ test "Futex - broadcasting" {
962 const num_iterations = 4;962 const num_iterations = 4;
963963
964 const Barrier = struct {964 const Barrier = struct {
965 count: Atomic(u32) = Atomic(u32).init(num_threads),965 count: atomic.Value(u32) = atomic.Value(u32).init(num_threads),
966 futex: Atomic(u32) = Atomic(u32).init(0),966 futex: atomic.Value(u32) = atomic.Value(u32).init(0),
967967
968 fn wait(self: *@This()) !void {968 fn wait(self: *@This()) !void {
969 // Decrement the counter.969 // Decrement the counter.
...@@ -1036,7 +1036,7 @@ pub const Deadline = struct {...@@ -1036,7 +1036,7 @@ pub const Deadline = struct {
1036 /// - `Futex.wake()` is called on the `ptr`.1036 /// - `Futex.wake()` is called on the `ptr`.
1037 /// - A spurious wake occurs.1037 /// - A spurious wake occurs.
1038 /// - The deadline expires; In which case `error.Timeout` is returned.1038 /// - The deadline expires; In which case `error.Timeout` is returned.
1039 pub fn wait(self: *Deadline, ptr: *const Atomic(u32), expect: u32) error{Timeout}!void {1039 pub fn wait(self: *Deadline, ptr: *const atomic.Value(u32), expect: u32) error{Timeout}!void {
1040 @setCold(true);1040 @setCold(true);
10411041
1042 // Check if we actually have a timeout to wait until.1042 // Check if we actually have a timeout to wait until.
...@@ -1056,7 +1056,7 @@ pub const Deadline = struct {...@@ -1056,7 +1056,7 @@ pub const Deadline = struct {
10561056
1057test "Futex - Deadline" {1057test "Futex - Deadline" {
1058 var deadline = Deadline.init(100 * std.time.ns_per_ms);1058 var deadline = Deadline.init(100 * std.time.ns_per_ms);
1059 var futex_word = Atomic(u32).init(0);1059 var futex_word = atomic.Value(u32).init(0);
10601060
1061 while (true) {1061 while (true) {
1062 deadline.wait(&futex_word, 0) catch break;1062 deadline.wait(&futex_word, 0) catch break;
lib/std/Thread/Mutex.zig+9-18
...@@ -26,7 +26,6 @@ const Mutex = @This();...@@ -26,7 +26,6 @@ const Mutex = @This();
26const os = std.os;26const os = std.os;
27const assert = std.debug.assert;27const assert = std.debug.assert;
28const testing = std.testing;28const testing = std.testing;
29const Atomic = std.atomic.Atomic;
30const Thread = std.Thread;29const Thread = std.Thread;
31const Futex = Thread.Futex;30const Futex = Thread.Futex;
3231
...@@ -67,7 +66,7 @@ else...@@ -67,7 +66,7 @@ else
67 FutexImpl;66 FutexImpl;
6867
69const DebugImpl = struct {68const DebugImpl = struct {
70 locking_thread: Atomic(Thread.Id) = Atomic(Thread.Id).init(0), // 0 means it's not locked.69 locking_thread: std.atomic.Value(Thread.Id) = std.atomic.Value(Thread.Id).init(0), // 0 means it's not locked.
71 impl: ReleaseImpl = .{},70 impl: ReleaseImpl = .{},
7271
73 inline fn tryLock(self: *@This()) bool {72 inline fn tryLock(self: *@This()) bool {
...@@ -151,37 +150,29 @@ const DarwinImpl = struct {...@@ -151,37 +150,29 @@ const DarwinImpl = struct {
151};150};
152151
153const FutexImpl = struct {152const FutexImpl = struct {
154 state: Atomic(u32) = Atomic(u32).init(unlocked),153 state: std.atomic.Value(u32) = std.atomic.Value(u32).init(unlocked),
155154
156 const unlocked = 0b00;155 const unlocked: u32 = 0b00;
157 const locked = 0b01;156 const locked: u32 = 0b01;
158 const contended = 0b11; // must contain the `locked` bit for x86 optimization below157 const contended: u32 = 0b11; // must contain the `locked` bit for x86 optimization below
159
160 fn tryLock(self: *@This()) bool {
161 // Lock with compareAndSwap instead of tryCompareAndSwap to avoid reporting spurious CAS failure.
162 return self.lockFast("compareAndSwap");
163 }
164158
165 fn lock(self: *@This()) void {159 fn lock(self: *@This()) void {
166 // Lock with tryCompareAndSwap instead of compareAndSwap due to being more inline-able on LL/SC archs like ARM.160 if (!self.tryLock())
167 if (!self.lockFast("tryCompareAndSwap")) {
168 self.lockSlow();161 self.lockSlow();
169 }
170 }162 }
171163
172 inline fn lockFast(self: *@This(), comptime cas_fn_name: []const u8) bool {164 fn tryLock(self: *@This()) bool {
173 // On x86, use `lock bts` instead of `lock cmpxchg` as:165 // On x86, use `lock bts` instead of `lock cmpxchg` as:
174 // - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048166 // - they both seem to mark the cache-line as modified regardless: https://stackoverflow.com/a/63350048
175 // - `lock bts` is smaller instruction-wise which makes it better for inlining167 // - `lock bts` is smaller instruction-wise which makes it better for inlining
176 if (comptime builtin.target.cpu.arch.isX86()) {168 if (comptime builtin.target.cpu.arch.isX86()) {
177 const locked_bit = @ctz(@as(u32, locked));169 const locked_bit = @ctz(locked);
178 return self.state.bitSet(locked_bit, .Acquire) == 0;170 return self.state.bitSet(locked_bit, .Acquire) == 0;
179 }171 }
180172
181 // Acquire barrier ensures grabbing the lock happens before the critical section173 // Acquire barrier ensures grabbing the lock happens before the critical section
182 // and that the previous lock holder's critical section happens before we grab the lock.174 // and that the previous lock holder's critical section happens before we grab the lock.
183 const casFn = @field(@TypeOf(self.state), cas_fn_name);175 return self.state.cmpxchgWeak(unlocked, locked, .Acquire, .Monotonic) == null;
184 return casFn(&self.state, unlocked, locked, .Acquire, .Monotonic) == null;
185 }176 }
186177
187 fn lockSlow(self: *@This()) void {178 fn lockSlow(self: *@This()) void {
lib/std/Thread/ResetEvent.zig+3-4
...@@ -9,7 +9,6 @@ const ResetEvent = @This();...@@ -9,7 +9,6 @@ const ResetEvent = @This();
9const os = std.os;9const os = std.os;
10const assert = std.debug.assert;10const assert = std.debug.assert;
11const testing = std.testing;11const testing = std.testing;
12const Atomic = std.atomic.Atomic;
13const Futex = std.Thread.Futex;12const Futex = std.Thread.Futex;
1413
15impl: Impl = .{},14impl: Impl = .{},
...@@ -89,7 +88,7 @@ const SingleThreadedImpl = struct {...@@ -89,7 +88,7 @@ const SingleThreadedImpl = struct {
89};88};
9089
91const FutexImpl = struct {90const FutexImpl = struct {
92 state: Atomic(u32) = Atomic(u32).init(unset),91 state: std.atomic.Value(u32) = std.atomic.Value(u32).init(unset),
9392
94 const unset = 0;93 const unset = 0;
95 const waiting = 1;94 const waiting = 1;
...@@ -115,7 +114,7 @@ const FutexImpl = struct {...@@ -115,7 +114,7 @@ const FutexImpl = struct {
115 // We avoid using any strict barriers until the end when we know the ResetEvent is set.114 // We avoid using any strict barriers until the end when we know the ResetEvent is set.
116 var state = self.state.load(.Monotonic);115 var state = self.state.load(.Monotonic);
117 if (state == unset) {116 if (state == unset) {
118 state = self.state.compareAndSwap(state, waiting, .Monotonic, .Monotonic) orelse waiting;117 state = self.state.cmpxchgStrong(state, waiting, .Monotonic, .Monotonic) orelse waiting;
119 }118 }
120119
121 // Wait until the ResetEvent is set since the state is waiting.120 // Wait until the ResetEvent is set since the state is waiting.
...@@ -252,7 +251,7 @@ test "ResetEvent - broadcast" {...@@ -252,7 +251,7 @@ test "ResetEvent - broadcast" {
252 const num_threads = 10;251 const num_threads = 10;
253 const Barrier = struct {252 const Barrier = struct {
254 event: ResetEvent = .{},253 event: ResetEvent = .{},
255 counter: Atomic(usize) = Atomic(usize).init(num_threads),254 counter: std.atomic.Value(usize) = std.atomic.Value(usize).init(num_threads),
256255
257 fn wait(self: *@This()) void {256 fn wait(self: *@This()) void {
258 if (self.counter.fetchSub(1, .AcqRel) == 1) {257 if (self.counter.fetchSub(1, .AcqRel) == 1) {
lib/std/Thread/RwLock.zig+1-1
...@@ -307,7 +307,7 @@ test "RwLock - concurrent access" {...@@ -307,7 +307,7 @@ test "RwLock - concurrent access" {
307307
308 rwl: RwLock = .{},308 rwl: RwLock = .{},
309 writes: usize = 0,309 writes: usize = 0,
310 reads: std.atomic.Atomic(usize) = std.atomic.Atomic(usize).init(0),310 reads: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
311311
312 term1: usize = 0,312 term1: usize = 0,
313 term2: usize = 0,313 term2: usize = 0,
lib/std/Thread/WaitGroup.zig+1-2
...@@ -1,12 +1,11 @@...@@ -1,12 +1,11 @@
1const std = @import("std");1const std = @import("std");
2const Atomic = std.atomic.Atomic;
3const assert = std.debug.assert;2const assert = std.debug.assert;
4const WaitGroup = @This();3const WaitGroup = @This();
54
6const is_waiting: usize = 1 << 0;5const is_waiting: usize = 1 << 0;
7const one_pending: usize = 1 << 1;6const one_pending: usize = 1 << 1;
87
9state: Atomic(usize) = Atomic(usize).init(0),8state: std.atomic.Value(usize) = std.atomic.Value(usize).init(0),
10event: std.Thread.ResetEvent = .{},9event: std.Thread.ResetEvent = .{},
1110
12pub fn start(self: *WaitGroup) void {11pub fn start(self: *WaitGroup) void {
lib/std/atomic.zig+376-6
...@@ -1,7 +1,376 @@...@@ -1,7 +1,376 @@
1const std = @import("std.zig");1/// This is a thin wrapper around a primitive value to prevent accidental data races.
2const builtin = @import("builtin");2pub fn Value(comptime T: type) type {
3 return extern struct {
4 /// Care must be taken to avoid data races when interacting with this field directly.
5 raw: T,
6
7 const Self = @This();
8
9 pub fn init(value: T) Self {
10 return .{ .raw = value };
11 }
12
13 /// Perform an atomic fence which uses the atomic value as a hint for
14 /// the modification order. Use this when you want to imply a fence on
15 /// an atomic variable without necessarily performing a memory access.
16 pub inline fn fence(self: *Self, comptime order: AtomicOrder) void {
17 // LLVM's ThreadSanitizer doesn't support the normal fences so we specialize for it.
18 if (builtin.sanitize_thread) {
19 const tsan = struct {
20 extern "c" fn __tsan_acquire(addr: *anyopaque) void;
21 extern "c" fn __tsan_release(addr: *anyopaque) void;
22 };
23
24 const addr: *anyopaque = self;
25 return switch (order) {
26 .Unordered, .Monotonic => @compileError(@tagName(order) ++ " only applies to atomic loads and stores"),
27 .Acquire => tsan.__tsan_acquire(addr),
28 .Release => tsan.__tsan_release(addr),
29 .AcqRel, .SeqCst => {
30 tsan.__tsan_acquire(addr);
31 tsan.__tsan_release(addr);
32 },
33 };
34 }
35
36 return @fence(order);
37 }
38
39 pub inline fn load(self: *const Self, comptime order: AtomicOrder) T {
40 return @atomicLoad(T, &self.raw, order);
41 }
42
43 pub inline fn store(self: *Self, value: T, comptime order: AtomicOrder) void {
44 @atomicStore(T, &self.raw, value, order);
45 }
46
47 pub inline fn swap(self: *Self, operand: T, comptime order: AtomicOrder) T {
48 return @atomicRmw(T, &self.raw, .Xchg, operand, order);
49 }
50
51 pub inline fn cmpxchgWeak(
52 self: *Self,
53 expected_value: T,
54 new_value: T,
55 comptime success_order: AtomicOrder,
56 comptime fail_order: AtomicOrder,
57 ) ?T {
58 return @cmpxchgWeak(T, &self.raw, expected_value, new_value, success_order, fail_order);
59 }
60
61 pub inline fn cmpxchgStrong(
62 self: *Self,
63 expected_value: T,
64 new_value: T,
65 comptime success_order: AtomicOrder,
66 comptime fail_order: AtomicOrder,
67 ) ?T {
68 return @cmpxchgStrong(T, &self.raw, expected_value, new_value, success_order, fail_order);
69 }
70
71 pub inline fn fetchAdd(self: *Self, operand: T, comptime order: AtomicOrder) T {
72 return @atomicRmw(T, &self.raw, .Add, operand, order);
73 }
74
75 pub inline fn fetchSub(self: *Self, operand: T, comptime order: AtomicOrder) T {
76 return @atomicRmw(T, &self.raw, .Sub, operand, order);
77 }
78
79 pub inline fn fetchMin(self: *Self, operand: T, comptime order: AtomicOrder) T {
80 return @atomicRmw(T, &self.raw, .Min, operand, order);
81 }
82
83 pub inline fn fetchMax(self: *Self, operand: T, comptime order: AtomicOrder) T {
84 return @atomicRmw(T, &self.raw, .Max, operand, order);
85 }
86
87 pub inline fn fetchAnd(self: *Self, operand: T, comptime order: AtomicOrder) T {
88 return @atomicRmw(T, &self.raw, .And, operand, order);
89 }
90
91 pub inline fn fetchNand(self: *Self, operand: T, comptime order: AtomicOrder) T {
92 return @atomicRmw(T, &self.raw, .Nand, operand, order);
93 }
94
95 pub inline fn fetchXor(self: *Self, operand: T, comptime order: AtomicOrder) T {
96 return @atomicRmw(T, &self.raw, .Xor, operand, order);
97 }
98
99 pub inline fn fetchOr(self: *Self, operand: T, comptime order: AtomicOrder) T {
100 return @atomicRmw(T, &self.raw, .Or, operand, order);
101 }
102
103 pub inline fn rmw(
104 self: *Self,
105 comptime op: std.builtin.AtomicRmwOp,
106 operand: T,
107 comptime order: AtomicOrder,
108 ) T {
109 return @atomicRmw(T, &self.raw, op, operand, order);
110 }
111
112 const Bit = std.math.Log2Int(T);
113
114 /// Marked `inline` so that if `bit` is comptime-known, the instruction
115 /// can be lowered to a more efficient machine code instruction if
116 /// possible.
117 pub inline fn bitSet(self: *Self, bit: Bit, comptime order: AtomicOrder) u1 {
118 const mask = @as(T, 1) << bit;
119 const value = self.fetchOr(mask, order);
120 return @intFromBool(value & mask != 0);
121 }
122
123 /// Marked `inline` so that if `bit` is comptime-known, the instruction
124 /// can be lowered to a more efficient machine code instruction if
125 /// possible.
126 pub inline fn bitReset(self: *Self, bit: Bit, comptime order: AtomicOrder) u1 {
127 const mask = @as(T, 1) << bit;
128 const value = self.fetchAnd(~mask, order);
129 return @intFromBool(value & mask != 0);
130 }
131
132 /// Marked `inline` so that if `bit` is comptime-known, the instruction
133 /// can be lowered to a more efficient machine code instruction if
134 /// possible.
135 pub inline fn bitToggle(self: *Self, bit: Bit, comptime order: AtomicOrder) u1 {
136 const mask = @as(T, 1) << bit;
137 const value = self.fetchXor(mask, order);
138 return @intFromBool(value & mask != 0);
139 }
140 };
141}
142
143test Value {
144 const RefCount = struct {
145 count: Value(usize),
146 dropFn: *const fn (*RefCount) void,
147
148 const RefCount = @This();
149
150 fn ref(rc: *RefCount) void {
151 // No ordering necessary; just updating a counter.
152 _ = rc.count.fetchAdd(1, .Monotonic);
153 }
154
155 fn unref(rc: *RefCount) void {
156 // Release ensures code before unref() happens-before the
157 // count is decremented as dropFn could be called by then.
158 if (rc.count.fetchSub(1, .Release) == 1) {
159 // Acquire ensures count decrement and code before
160 // previous unrefs()s happens-before we call dropFn
161 // below.
162 // Another alternative is to use .AcqRel on the
163 // fetchSub count decrement but it's extra barrier in
164 // possibly hot path.
165 rc.count.fence(.Acquire);
166 (rc.dropFn)(rc);
167 }
168 }
169
170 fn noop(rc: *RefCount) void {
171 _ = rc;
172 }
173 };
174
175 var ref_count: RefCount = .{
176 .count = Value(usize).init(0),
177 .dropFn = RefCount.noop,
178 };
179 ref_count.ref();
180 ref_count.unref();
181}
182
183test "Value.swap" {
184 var x = Value(usize).init(5);
185 try testing.expectEqual(@as(usize, 5), x.swap(10, .SeqCst));
186 try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
187
188 const E = enum(usize) { a, b, c };
189 var y = Value(E).init(.c);
190 try testing.expectEqual(E.c, y.swap(.a, .SeqCst));
191 try testing.expectEqual(E.a, y.load(.SeqCst));
192
193 var z = Value(f32).init(5.0);
194 try testing.expectEqual(@as(f32, 5.0), z.swap(10.0, .SeqCst));
195 try testing.expectEqual(@as(f32, 10.0), z.load(.SeqCst));
196
197 var a = Value(bool).init(false);
198 try testing.expectEqual(false, a.swap(true, .SeqCst));
199 try testing.expectEqual(true, a.load(.SeqCst));
200
201 var b = Value(?*u8).init(null);
202 try testing.expectEqual(@as(?*u8, null), b.swap(@as(?*u8, @ptrFromInt(@alignOf(u8))), .SeqCst));
203 try testing.expectEqual(@as(?*u8, @ptrFromInt(@alignOf(u8))), b.load(.SeqCst));
204}
205
206test "Value.store" {
207 var x = Value(usize).init(5);
208 x.store(10, .SeqCst);
209 try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
210}
211
212test "Value.cmpxchgWeak" {
213 var x = Value(usize).init(0);
214
215 try testing.expectEqual(@as(?usize, 0), x.cmpxchgWeak(1, 0, .SeqCst, .SeqCst));
216 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
3217
4pub const Atomic = @import("atomic/Atomic.zig").Atomic;218 while (x.cmpxchgWeak(0, 1, .SeqCst, .SeqCst)) |_| {}
219 try testing.expectEqual(@as(usize, 1), x.load(.SeqCst));
220
221 while (x.cmpxchgWeak(1, 0, .SeqCst, .SeqCst)) |_| {}
222 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
223}
224
225test "Value.cmpxchgStrong" {
226 var x = Value(usize).init(0);
227 try testing.expectEqual(@as(?usize, 0), x.cmpxchgStrong(1, 0, .SeqCst, .SeqCst));
228 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
229 try testing.expectEqual(@as(?usize, null), x.cmpxchgStrong(0, 1, .SeqCst, .SeqCst));
230 try testing.expectEqual(@as(usize, 1), x.load(.SeqCst));
231 try testing.expectEqual(@as(?usize, null), x.cmpxchgStrong(1, 0, .SeqCst, .SeqCst));
232 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
233}
234
235test "Value.fetchAdd" {
236 var x = Value(usize).init(5);
237 try testing.expectEqual(@as(usize, 5), x.fetchAdd(5, .SeqCst));
238 try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
239 try testing.expectEqual(@as(usize, 10), x.fetchAdd(std.math.maxInt(usize), .SeqCst));
240 try testing.expectEqual(@as(usize, 9), x.load(.SeqCst));
241}
242
243test "Value.fetchSub" {
244 var x = Value(usize).init(5);
245 try testing.expectEqual(@as(usize, 5), x.fetchSub(5, .SeqCst));
246 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
247 try testing.expectEqual(@as(usize, 0), x.fetchSub(1, .SeqCst));
248 try testing.expectEqual(@as(usize, std.math.maxInt(usize)), x.load(.SeqCst));
249}
250
251test "Value.fetchMin" {
252 var x = Value(usize).init(5);
253 try testing.expectEqual(@as(usize, 5), x.fetchMin(0, .SeqCst));
254 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
255 try testing.expectEqual(@as(usize, 0), x.fetchMin(10, .SeqCst));
256 try testing.expectEqual(@as(usize, 0), x.load(.SeqCst));
257}
258
259test "Value.fetchMax" {
260 var x = Value(usize).init(5);
261 try testing.expectEqual(@as(usize, 5), x.fetchMax(10, .SeqCst));
262 try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
263 try testing.expectEqual(@as(usize, 10), x.fetchMax(5, .SeqCst));
264 try testing.expectEqual(@as(usize, 10), x.load(.SeqCst));
265}
266
267test "Value.fetchAnd" {
268 var x = Value(usize).init(0b11);
269 try testing.expectEqual(@as(usize, 0b11), x.fetchAnd(0b10, .SeqCst));
270 try testing.expectEqual(@as(usize, 0b10), x.load(.SeqCst));
271 try testing.expectEqual(@as(usize, 0b10), x.fetchAnd(0b00, .SeqCst));
272 try testing.expectEqual(@as(usize, 0b00), x.load(.SeqCst));
273}
274
275test "Value.fetchNand" {
276 var x = Value(usize).init(0b11);
277 try testing.expectEqual(@as(usize, 0b11), x.fetchNand(0b10, .SeqCst));
278 try testing.expectEqual(~@as(usize, 0b10), x.load(.SeqCst));
279 try testing.expectEqual(~@as(usize, 0b10), x.fetchNand(0b00, .SeqCst));
280 try testing.expectEqual(~@as(usize, 0b00), x.load(.SeqCst));
281}
282
283test "Value.fetchOr" {
284 var x = Value(usize).init(0b11);
285 try testing.expectEqual(@as(usize, 0b11), x.fetchOr(0b100, .SeqCst));
286 try testing.expectEqual(@as(usize, 0b111), x.load(.SeqCst));
287 try testing.expectEqual(@as(usize, 0b111), x.fetchOr(0b010, .SeqCst));
288 try testing.expectEqual(@as(usize, 0b111), x.load(.SeqCst));
289}
290
291test "Value.fetchXor" {
292 var x = Value(usize).init(0b11);
293 try testing.expectEqual(@as(usize, 0b11), x.fetchXor(0b10, .SeqCst));
294 try testing.expectEqual(@as(usize, 0b01), x.load(.SeqCst));
295 try testing.expectEqual(@as(usize, 0b01), x.fetchXor(0b01, .SeqCst));
296 try testing.expectEqual(@as(usize, 0b00), x.load(.SeqCst));
297}
298
299test "Value.bitSet" {
300 var x = Value(usize).init(0);
301
302 for (0..@bitSizeOf(usize)) |bit_index| {
303 const bit = @as(std.math.Log2Int(usize), @intCast(bit_index));
304 const mask = @as(usize, 1) << bit;
305
306 // setting the bit should change the bit
307 try testing.expect(x.load(.SeqCst) & mask == 0);
308 try testing.expectEqual(@as(u1, 0), x.bitSet(bit, .SeqCst));
309 try testing.expect(x.load(.SeqCst) & mask != 0);
310
311 // setting it again shouldn't change the bit
312 try testing.expectEqual(@as(u1, 1), x.bitSet(bit, .SeqCst));
313 try testing.expect(x.load(.SeqCst) & mask != 0);
314
315 // all the previous bits should have not changed (still be set)
316 for (0..bit_index) |prev_bit_index| {
317 const prev_bit = @as(std.math.Log2Int(usize), @intCast(prev_bit_index));
318 const prev_mask = @as(usize, 1) << prev_bit;
319 try testing.expect(x.load(.SeqCst) & prev_mask != 0);
320 }
321 }
322}
323
324test "Value.bitReset" {
325 var x = Value(usize).init(0);
326
327 for (0..@bitSizeOf(usize)) |bit_index| {
328 const bit = @as(std.math.Log2Int(usize), @intCast(bit_index));
329 const mask = @as(usize, 1) << bit;
330 x.raw |= mask;
331
332 // unsetting the bit should change the bit
333 try testing.expect(x.load(.SeqCst) & mask != 0);
334 try testing.expectEqual(@as(u1, 1), x.bitReset(bit, .SeqCst));
335 try testing.expect(x.load(.SeqCst) & mask == 0);
336
337 // unsetting it again shouldn't change the bit
338 try testing.expectEqual(@as(u1, 0), x.bitReset(bit, .SeqCst));
339 try testing.expect(x.load(.SeqCst) & mask == 0);
340
341 // all the previous bits should have not changed (still be reset)
342 for (0..bit_index) |prev_bit_index| {
343 const prev_bit = @as(std.math.Log2Int(usize), @intCast(prev_bit_index));
344 const prev_mask = @as(usize, 1) << prev_bit;
345 try testing.expect(x.load(.SeqCst) & prev_mask == 0);
346 }
347 }
348}
349
350test "Value.bitToggle" {
351 var x = Value(usize).init(0);
352
353 for (0..@bitSizeOf(usize)) |bit_index| {
354 const bit = @as(std.math.Log2Int(usize), @intCast(bit_index));
355 const mask = @as(usize, 1) << bit;
356
357 // toggling the bit should change the bit
358 try testing.expect(x.load(.SeqCst) & mask == 0);
359 try testing.expectEqual(@as(u1, 0), x.bitToggle(bit, .SeqCst));
360 try testing.expect(x.load(.SeqCst) & mask != 0);
361
362 // toggling it again *should* change the bit
363 try testing.expectEqual(@as(u1, 1), x.bitToggle(bit, .SeqCst));
364 try testing.expect(x.load(.SeqCst) & mask == 0);
365
366 // all the previous bits should have not changed (still be toggled back)
367 for (0..bit_index) |prev_bit_index| {
368 const prev_bit = @as(std.math.Log2Int(usize), @intCast(prev_bit_index));
369 const prev_mask = @as(usize, 1) << prev_bit;
370 try testing.expect(x.load(.SeqCst) & prev_mask == 0);
371 }
372 }
373}
5374
6/// Signals to the processor that the caller is inside a busy-wait spin-loop.375/// Signals to the processor that the caller is inside a busy-wait spin-loop.
7pub inline fn spinLoopHint() void {376pub inline fn spinLoopHint() void {
...@@ -83,6 +452,7 @@ pub const cache_line = switch (builtin.cpu.arch) {...@@ -83,6 +452,7 @@ pub const cache_line = switch (builtin.cpu.arch) {
83 else => 64,452 else => 64,
84};453};
85454
86test {455const std = @import("std.zig");
87 _ = Atomic;456const builtin = @import("builtin");
88}457const AtomicOrder = std.builtin.AtomicOrder;
458const testing = std.testing;
lib/std/atomic/Atomic.zig deleted-619
...@@ -1,619 +0,0 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3
4const testing = std.testing;
5const AtomicOrder = std.builtin.AtomicOrder;
6
7pub fn Atomic(comptime T: type) type {
8 return extern struct {
9 value: T,
10
11 const Self = @This();
12
13 pub fn init(value: T) Self {
14 return .{ .value = value };
15 }
16
17 /// Perform an atomic fence which uses the atomic value as a hint for the modification order.
18 /// Use this when you want to imply a fence on an atomic variable without necessarily performing a memory access.
19 ///
20 /// Example:
21 /// ```
22 /// const RefCount = struct {
23 /// count: Atomic(usize),
24 /// dropFn: *const fn (*RefCount) void,
25 ///
26 /// fn ref(self: *RefCount) void {
27 /// _ = self.count.fetchAdd(1, .Monotonic); // no ordering necessary, just updating a counter
28 /// }
29 ///
30 /// fn unref(self: *RefCount) void {
31 /// // Release ensures code before unref() happens-before the count is decremented as dropFn could be called by then.
32 /// if (self.count.fetchSub(1, .Release)) {
33 /// // Acquire ensures count decrement and code before previous unrefs()s happens-before we call dropFn below.
34 /// // NOTE: another alternative is to use .AcqRel on the fetchSub count decrement but it's extra barrier in possibly hot path.
35 /// self.count.fence(.Acquire);
36 /// (self.dropFn)(self);
37 /// }
38 /// }
39 /// };
40 /// ```
41 pub inline fn fence(self: *Self, comptime ordering: AtomicOrder) void {
42 // LLVM's ThreadSanitizer doesn't support the normal fences so we specialize for it.
43 if (builtin.sanitize_thread) {
44 const tsan = struct {
45 extern "c" fn __tsan_acquire(addr: *anyopaque) void;
46 extern "c" fn __tsan_release(addr: *anyopaque) void;
47 };
48
49 const addr: *anyopaque = self;
50 return switch (ordering) {
51 .Unordered, .Monotonic => @compileError(@tagName(ordering) ++ " only applies to atomic loads and stores"),
52 .Acquire => tsan.__tsan_acquire(addr),
53 .Release => tsan.__tsan_release(addr),
54 .AcqRel, .SeqCst => {
55 tsan.__tsan_acquire(addr);
56 tsan.__tsan_release(addr);
57 },
58 };
59 }
60
61 return @fence(ordering);
62 }
63
64 test fence {
65 inline for (.{ .Acquire, .Release, .AcqRel, .SeqCst }) |ordering| {
66 var x = Atomic(usize).init(0);
67 x.fence(ordering);
68 }
69 }
70
71 /// Non-atomically load from the atomic value without synchronization.
72 /// Care must be taken to avoid data-races when interacting with other atomic operations.
73 pub inline fn loadUnchecked(self: Self) T {
74 return self.value;
75 }
76
77 /// Non-atomically store to the atomic value without synchronization.
78 /// Care must be taken to avoid data-races when interacting with other atomic operations.
79 pub inline fn storeUnchecked(self: *Self, value: T) void {
80 self.value = value;
81 }
82
83 pub inline fn load(self: *const Self, comptime ordering: AtomicOrder) T {
84 return switch (ordering) {
85 .AcqRel => @compileError(@tagName(ordering) ++ " implies " ++ @tagName(AtomicOrder.Release) ++ " which is only allowed on atomic stores"),
86 .Release => @compileError(@tagName(ordering) ++ " is only allowed on atomic stores"),
87 else => @atomicLoad(T, &self.value, ordering),
88 };
89 }
90
91 pub inline fn store(self: *Self, value: T, comptime ordering: AtomicOrder) void {
92 switch (ordering) {
93 .AcqRel => @compileError(@tagName(ordering) ++ " implies " ++ @tagName(AtomicOrder.Acquire) ++ " which is only allowed on atomic loads"),
94 .Acquire => @compileError(@tagName(ordering) ++ " is only allowed on atomic loads"),
95 else => @atomicStore(T, &self.value, value, ordering),
96 }
97 }
98
99 pub inline fn swap(self: *Self, value: T, comptime ordering: AtomicOrder) T {
100 return self.rmw(.Xchg, value, ordering);
101 }
102
103 pub inline fn compareAndSwap(
104 self: *Self,
105 compare: T,
106 exchange: T,
107 comptime success: AtomicOrder,
108 comptime failure: AtomicOrder,
109 ) ?T {
110 return self.cmpxchg(true, compare, exchange, success, failure);
111 }
112
113 pub inline fn tryCompareAndSwap(
114 self: *Self,
115 compare: T,
116 exchange: T,
117 comptime success: AtomicOrder,
118 comptime failure: AtomicOrder,
119 ) ?T {
120 return self.cmpxchg(false, compare, exchange, success, failure);
121 }
122
123 inline fn cmpxchg(
124 self: *Self,
125 comptime is_strong: bool,
126 compare: T,
127 exchange: T,
128 comptime success: AtomicOrder,
129 comptime failure: AtomicOrder,
130 ) ?T {
131 if (success == .Unordered or failure == .Unordered) {
132 @compileError(@tagName(AtomicOrder.Unordered) ++ " is only allowed on atomic loads and stores");
133 }
134
135 const success_is_stronger = switch (failure) {
136 .SeqCst => success == .SeqCst,
137 .AcqRel => @compileError(@tagName(failure) ++ " implies " ++ @tagName(AtomicOrder.Release) ++ " which is only allowed on success"),
138 .Acquire => success == .SeqCst or success == .AcqRel or success == .Acquire,
139 .Release => @compileError(@tagName(failure) ++ " is only allowed on success"),
140 .Monotonic => true,
141 .Unordered => unreachable,
142 };
143
144 if (!success_is_stronger) {
145 @compileError(@tagName(success) ++ " must be stronger than " ++ @tagName(failure));
146 }
147
148 return switch (is_strong) {
149 true => @cmpxchgStrong(T, &self.value, compare, exchange, success, failure),
150 false => @cmpxchgWeak(T, &self.value, compare, exchange, success, failure),
151 };
152 }
153
154 inline fn rmw(
155 self: *Self,
156 comptime op: std.builtin.AtomicRmwOp,
157 value: T,
158 comptime ordering: AtomicOrder,
159 ) T {
160 return @atomicRmw(T, &self.value, op, value, ordering);
161 }
162
163 pub inline fn fetchAdd(self: *Self, value: T, comptime ordering: AtomicOrder) T {
164 return self.rmw(.Add, value, ordering);
165 }
166
167 pub inline fn fetchSub(self: *Self, value: T, comptime ordering: AtomicOrder) T {
168 return self.rmw(.Sub, value, ordering);
169 }
170
171 pub inline fn fetchMin(self: *Self, value: T, comptime ordering: AtomicOrder) T {
172 return self.rmw(.Min, value, ordering);
173 }
174
175 pub inline fn fetchMax(self: *Self, value: T, comptime ordering: AtomicOrder) T {
176 return self.rmw(.Max, value, ordering);
177 }
178
179 pub inline fn fetchAnd(self: *Self, value: T, comptime ordering: AtomicOrder) T {
180 return self.rmw(.And, value, ordering);
181 }
182
183 pub inline fn fetchNand(self: *Self, value: T, comptime ordering: AtomicOrder) T {
184 return self.rmw(.Nand, value, ordering);
185 }
186
187 pub inline fn fetchOr(self: *Self, value: T, comptime ordering: AtomicOrder) T {
188 return self.rmw(.Or, value, ordering);
189 }
190
191 pub inline fn fetchXor(self: *Self, value: T, comptime ordering: AtomicOrder) T {
192 return self.rmw(.Xor, value, ordering);
193 }
194
195 const Bit = std.math.Log2Int(T);
196 const BitRmwOp = enum {
197 Set,
198 Reset,
199 Toggle,
200 };
201
202 pub inline fn bitSet(self: *Self, bit: Bit, comptime ordering: AtomicOrder) u1 {
203 return bitRmw(self, .Set, bit, ordering);
204 }
205
206 pub inline fn bitReset(self: *Self, bit: Bit, comptime ordering: AtomicOrder) u1 {
207 return bitRmw(self, .Reset, bit, ordering);
208 }
209
210 pub inline fn bitToggle(self: *Self, bit: Bit, comptime ordering: AtomicOrder) u1 {
211 return bitRmw(self, .Toggle, bit, ordering);
212 }
213
214 inline fn bitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: AtomicOrder) u1 {
215 // x86 supports dedicated bitwise instructions
216 if (comptime builtin.target.cpu.arch.isX86() and @sizeOf(T) >= 2 and @sizeOf(T) <= 8) {
217 // TODO: this causes std lib test failures when enabled
218 if (false) {
219 return x86BitRmw(self, op, bit, ordering);
220 }
221 }
222
223 const mask = @as(T, 1) << bit;
224 const value = switch (op) {
225 .Set => self.fetchOr(mask, ordering),
226 .Reset => self.fetchAnd(~mask, ordering),
227 .Toggle => self.fetchXor(mask, ordering),
228 };
229
230 return @intFromBool(value & mask != 0);
231 }
232
233 inline fn x86BitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: AtomicOrder) u1 {
234 const old_bit: u8 = switch (@sizeOf(T)) {
235 2 => switch (op) {
236 .Set => asm volatile ("lock btsw %[bit], %[ptr]"
237 // LLVM doesn't support u1 flag register return values
238 : [result] "={@ccc}" (-> u8),
239 : [ptr] "*m" (&self.value),
240 [bit] "X" (@as(T, bit)),
241 : "cc", "memory"
242 ),
243 .Reset => asm volatile ("lock btrw %[bit], %[ptr]"
244 // LLVM doesn't support u1 flag register return values
245 : [result] "={@ccc}" (-> u8),
246 : [ptr] "*m" (&self.value),
247 [bit] "X" (@as(T, bit)),
248 : "cc", "memory"
249 ),
250 .Toggle => asm volatile ("lock btcw %[bit], %[ptr]"
251 // LLVM doesn't support u1 flag register return values
252 : [result] "={@ccc}" (-> u8),
253 : [ptr] "*m" (&self.value),
254 [bit] "X" (@as(T, bit)),
255 : "cc", "memory"
256 ),
257 },
258 4 => switch (op) {
259 .Set => asm volatile ("lock btsl %[bit], %[ptr]"
260 // LLVM doesn't support u1 flag register return values
261 : [result] "={@ccc}" (-> u8),
262 : [ptr] "*m" (&self.value),
263 [bit] "X" (@as(T, bit)),
264 : "cc", "memory"
265 ),
266 .Reset => asm volatile ("lock btrl %[bit], %[ptr]"
267 // LLVM doesn't support u1 flag register return values
268 : [result] "={@ccc}" (-> u8),
269 : [ptr] "*m" (&self.value),
270 [bit] "X" (@as(T, bit)),
271 : "cc", "memory"
272 ),
273 .Toggle => asm volatile ("lock btcl %[bit], %[ptr]"
274 // LLVM doesn't support u1 flag register return values
275 : [result] "={@ccc}" (-> u8),
276 : [ptr] "*m" (&self.value),
277 [bit] "X" (@as(T, bit)),
278 : "cc", "memory"
279 ),
280 },
281 8 => switch (op) {
282 .Set => asm volatile ("lock btsq %[bit], %[ptr]"
283 // LLVM doesn't support u1 flag register return values
284 : [result] "={@ccc}" (-> u8),
285 : [ptr] "*m" (&self.value),
286 [bit] "X" (@as(T, bit)),
287 : "cc", "memory"
288 ),
289 .Reset => asm volatile ("lock btrq %[bit], %[ptr]"
290 // LLVM doesn't support u1 flag register return values
291 : [result] "={@ccc}" (-> u8),
292 : [ptr] "*m" (&self.value),
293 [bit] "X" (@as(T, bit)),
294 : "cc", "memory"
295 ),
296 .Toggle => asm volatile ("lock btcq %[bit], %[ptr]"
297 // LLVM doesn't support u1 flag register return values
298 : [result] "={@ccc}" (-> u8),
299 : [ptr] "*m" (&self.value),
300 [bit] "X" (@as(T, bit)),
301 : "cc", "memory"
302 ),
303 },
304 else => @compileError("Invalid atomic type " ++ @typeName(T)),
305 };
306
307 // TODO: emit appropriate tsan fence if compiling with tsan
308 _ = ordering;
309
310 return @intCast(old_bit);
311 }
312 };
313}
314
315fn atomicIntTypes() []const type {
316 comptime var bytes = 1;
317 comptime var types: []const type = &[_]type{};
318 inline while (bytes <= @sizeOf(usize)) : (bytes *= 2) {
319 types = types ++ &[_]type{std.meta.Int(.unsigned, bytes * 8)};
320 }
321 return types;
322}
323
324test "Atomic.loadUnchecked" {
325 inline for (atomicIntTypes()) |Int| {
326 var x = Atomic(Int).init(5);
327 try testing.expectEqual(x.loadUnchecked(), 5);
328 }
329}
330
331test "Atomic.storeUnchecked" {
332 inline for (atomicIntTypes()) |Int| {
333 _ = Int;
334 var x = Atomic(usize).init(5);
335 x.storeUnchecked(10);
336 try testing.expectEqual(x.loadUnchecked(), 10);
337 }
338}
339
340test "Atomic.load" {
341 inline for (atomicIntTypes()) |Int| {
342 inline for (.{ .Unordered, .Monotonic, .Acquire, .SeqCst }) |ordering| {
343 var x = Atomic(Int).init(5);
344 try testing.expectEqual(x.load(ordering), 5);
345 }
346 }
347}
348
349test "Atomic.store" {
350 inline for (atomicIntTypes()) |Int| {
351 inline for (.{ .Unordered, .Monotonic, .Release, .SeqCst }) |ordering| {
352 _ = Int;
353 var x = Atomic(usize).init(5);
354 x.store(10, ordering);
355 try testing.expectEqual(x.load(.SeqCst), 10);
356 }
357 }
358}
359
360const atomic_rmw_orderings = [_]AtomicOrder{
361 .Monotonic,
362 .Acquire,
363 .Release,
364 .AcqRel,
365 .SeqCst,
366};
367
368test "Atomic.swap" {
369 inline for (atomic_rmw_orderings) |ordering| {
370 var x = Atomic(usize).init(5);
371 try testing.expectEqual(x.swap(10, ordering), 5);
372 try testing.expectEqual(x.load(.SeqCst), 10);
373
374 var y = Atomic(enum(usize) { a, b, c }).init(.c);
375 try testing.expectEqual(y.swap(.a, ordering), .c);
376 try testing.expectEqual(y.load(.SeqCst), .a);
377
378 var z = Atomic(f32).init(5.0);
379 try testing.expectEqual(z.swap(10.0, ordering), 5.0);
380 try testing.expectEqual(z.load(.SeqCst), 10.0);
381
382 var a = Atomic(bool).init(false);
383 try testing.expectEqual(a.swap(true, ordering), false);
384 try testing.expectEqual(a.load(.SeqCst), true);
385
386 var b = Atomic(?*u8).init(null);
387 try testing.expectEqual(b.swap(@as(?*u8, @ptrFromInt(@alignOf(u8))), ordering), null);
388 try testing.expectEqual(b.load(.SeqCst), @as(?*u8, @ptrFromInt(@alignOf(u8))));
389 }
390}
391
392const atomic_cmpxchg_orderings = [_][2]AtomicOrder{
393 .{ .Monotonic, .Monotonic },
394 .{ .Acquire, .Monotonic },
395 .{ .Acquire, .Acquire },
396 .{ .Release, .Monotonic },
397 // Although accepted by LLVM, acquire failure implies AcqRel success
398 // .{ .Release, .Acquire },
399 .{ .AcqRel, .Monotonic },
400 .{ .AcqRel, .Acquire },
401 .{ .SeqCst, .Monotonic },
402 .{ .SeqCst, .Acquire },
403 .{ .SeqCst, .SeqCst },
404};
405
406test "Atomic.compareAndSwap" {
407 inline for (atomicIntTypes()) |Int| {
408 inline for (atomic_cmpxchg_orderings) |ordering| {
409 var x = Atomic(Int).init(0);
410 try testing.expectEqual(x.compareAndSwap(1, 0, ordering[0], ordering[1]), 0);
411 try testing.expectEqual(x.load(.SeqCst), 0);
412 try testing.expectEqual(x.compareAndSwap(0, 1, ordering[0], ordering[1]), null);
413 try testing.expectEqual(x.load(.SeqCst), 1);
414 try testing.expectEqual(x.compareAndSwap(1, 0, ordering[0], ordering[1]), null);
415 try testing.expectEqual(x.load(.SeqCst), 0);
416 }
417 }
418}
419
420test "Atomic.tryCompareAndSwap" {
421 inline for (atomicIntTypes()) |Int| {
422 inline for (atomic_cmpxchg_orderings) |ordering| {
423 var x = Atomic(Int).init(0);
424
425 try testing.expectEqual(x.tryCompareAndSwap(1, 0, ordering[0], ordering[1]), 0);
426 try testing.expectEqual(x.load(.SeqCst), 0);
427
428 while (x.tryCompareAndSwap(0, 1, ordering[0], ordering[1])) |_| {}
429 try testing.expectEqual(x.load(.SeqCst), 1);
430
431 while (x.tryCompareAndSwap(1, 0, ordering[0], ordering[1])) |_| {}
432 try testing.expectEqual(x.load(.SeqCst), 0);
433 }
434 }
435}
436
437test "Atomic.fetchAdd" {
438 inline for (atomicIntTypes()) |Int| {
439 inline for (atomic_rmw_orderings) |ordering| {
440 var x = Atomic(Int).init(5);
441 try testing.expectEqual(x.fetchAdd(5, ordering), 5);
442 try testing.expectEqual(x.load(.SeqCst), 10);
443 try testing.expectEqual(x.fetchAdd(std.math.maxInt(Int), ordering), 10);
444 try testing.expectEqual(x.load(.SeqCst), 9);
445 }
446 }
447}
448
449test "Atomic.fetchSub" {
450 inline for (atomicIntTypes()) |Int| {
451 inline for (atomic_rmw_orderings) |ordering| {
452 var x = Atomic(Int).init(5);
453 try testing.expectEqual(x.fetchSub(5, ordering), 5);
454 try testing.expectEqual(x.load(.SeqCst), 0);
455 try testing.expectEqual(x.fetchSub(1, ordering), 0);
456 try testing.expectEqual(x.load(.SeqCst), std.math.maxInt(Int));
457 }
458 }
459}
460
461test "Atomic.fetchMin" {
462 inline for (atomicIntTypes()) |Int| {
463 inline for (atomic_rmw_orderings) |ordering| {
464 var x = Atomic(Int).init(5);
465 try testing.expectEqual(x.fetchMin(0, ordering), 5);
466 try testing.expectEqual(x.load(.SeqCst), 0);
467 try testing.expectEqual(x.fetchMin(10, ordering), 0);
468 try testing.expectEqual(x.load(.SeqCst), 0);
469 }
470 }
471}
472
473test "Atomic.fetchMax" {
474 inline for (atomicIntTypes()) |Int| {
475 inline for (atomic_rmw_orderings) |ordering| {
476 var x = Atomic(Int).init(5);
477 try testing.expectEqual(x.fetchMax(10, ordering), 5);
478 try testing.expectEqual(x.load(.SeqCst), 10);
479 try testing.expectEqual(x.fetchMax(5, ordering), 10);
480 try testing.expectEqual(x.load(.SeqCst), 10);
481 }
482 }
483}
484
485test "Atomic.fetchAnd" {
486 inline for (atomicIntTypes()) |Int| {
487 inline for (atomic_rmw_orderings) |ordering| {
488 var x = Atomic(Int).init(0b11);
489 try testing.expectEqual(x.fetchAnd(0b10, ordering), 0b11);
490 try testing.expectEqual(x.load(.SeqCst), 0b10);
491 try testing.expectEqual(x.fetchAnd(0b00, ordering), 0b10);
492 try testing.expectEqual(x.load(.SeqCst), 0b00);
493 }
494 }
495}
496
497test "Atomic.fetchNand" {
498 inline for (atomicIntTypes()) |Int| {
499 inline for (atomic_rmw_orderings) |ordering| {
500 var x = Atomic(Int).init(0b11);
501 try testing.expectEqual(x.fetchNand(0b10, ordering), 0b11);
502 try testing.expectEqual(x.load(.SeqCst), ~@as(Int, 0b10));
503 try testing.expectEqual(x.fetchNand(0b00, ordering), ~@as(Int, 0b10));
504 try testing.expectEqual(x.load(.SeqCst), ~@as(Int, 0b00));
505 }
506 }
507}
508
509test "Atomic.fetchOr" {
510 inline for (atomicIntTypes()) |Int| {
511 inline for (atomic_rmw_orderings) |ordering| {
512 var x = Atomic(Int).init(0b11);
513 try testing.expectEqual(x.fetchOr(0b100, ordering), 0b11);
514 try testing.expectEqual(x.load(.SeqCst), 0b111);
515 try testing.expectEqual(x.fetchOr(0b010, ordering), 0b111);
516 try testing.expectEqual(x.load(.SeqCst), 0b111);
517 }
518 }
519}
520
521test "Atomic.fetchXor" {
522 inline for (atomicIntTypes()) |Int| {
523 inline for (atomic_rmw_orderings) |ordering| {
524 var x = Atomic(Int).init(0b11);
525 try testing.expectEqual(x.fetchXor(0b10, ordering), 0b11);
526 try testing.expectEqual(x.load(.SeqCst), 0b01);
527 try testing.expectEqual(x.fetchXor(0b01, ordering), 0b01);
528 try testing.expectEqual(x.load(.SeqCst), 0b00);
529 }
530 }
531}
532
533test "Atomic.bitSet" {
534 inline for (atomicIntTypes()) |Int| {
535 inline for (atomic_rmw_orderings) |ordering| {
536 var x = Atomic(Int).init(0);
537
538 for (0..@bitSizeOf(Int)) |bit_index| {
539 const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
540 const mask = @as(Int, 1) << bit;
541
542 // setting the bit should change the bit
543 try testing.expect(x.load(.SeqCst) & mask == 0);
544 try testing.expectEqual(x.bitSet(bit, ordering), 0);
545 try testing.expect(x.load(.SeqCst) & mask != 0);
546
547 // setting it again shouldn't change the bit
548 try testing.expectEqual(x.bitSet(bit, ordering), 1);
549 try testing.expect(x.load(.SeqCst) & mask != 0);
550
551 // all the previous bits should have not changed (still be set)
552 for (0..bit_index) |prev_bit_index| {
553 const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
554 const prev_mask = @as(Int, 1) << prev_bit;
555 try testing.expect(x.load(.SeqCst) & prev_mask != 0);
556 }
557 }
558 }
559 }
560}
561
562test "Atomic.bitReset" {
563 inline for (atomicIntTypes()) |Int| {
564 inline for (atomic_rmw_orderings) |ordering| {
565 var x = Atomic(Int).init(0);
566
567 for (0..@bitSizeOf(Int)) |bit_index| {
568 const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
569 const mask = @as(Int, 1) << bit;
570 x.storeUnchecked(x.loadUnchecked() | mask);
571
572 // unsetting the bit should change the bit
573 try testing.expect(x.load(.SeqCst) & mask != 0);
574 try testing.expectEqual(x.bitReset(bit, ordering), 1);
575 try testing.expect(x.load(.SeqCst) & mask == 0);
576
577 // unsetting it again shouldn't change the bit
578 try testing.expectEqual(x.bitReset(bit, ordering), 0);
579 try testing.expect(x.load(.SeqCst) & mask == 0);
580
581 // all the previous bits should have not changed (still be reset)
582 for (0..bit_index) |prev_bit_index| {
583 const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
584 const prev_mask = @as(Int, 1) << prev_bit;
585 try testing.expect(x.load(.SeqCst) & prev_mask == 0);
586 }
587 }
588 }
589 }
590}
591
592test "Atomic.bitToggle" {
593 inline for (atomicIntTypes()) |Int| {
594 inline for (atomic_rmw_orderings) |ordering| {
595 var x = Atomic(Int).init(0);
596
597 for (0..@bitSizeOf(Int)) |bit_index| {
598 const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
599 const mask = @as(Int, 1) << bit;
600
601 // toggling the bit should change the bit
602 try testing.expect(x.load(.SeqCst) & mask == 0);
603 try testing.expectEqual(x.bitToggle(bit, ordering), 0);
604 try testing.expect(x.load(.SeqCst) & mask != 0);
605
606 // toggling it again *should* change the bit
607 try testing.expectEqual(x.bitToggle(bit, ordering), 1);
608 try testing.expect(x.load(.SeqCst) & mask == 0);
609
610 // all the previous bits should have not changed (still be toggled back)
611 for (0..bit_index) |prev_bit_index| {
612 const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
613 const prev_mask = @as(Int, 1) << prev_bit;
614 try testing.expect(x.load(.SeqCst) & prev_mask == 0);
615 }
616 }
617 }
618 }
619}
lib/std/child_process.zig+1-1
...@@ -1285,7 +1285,7 @@ fn windowsMakePipeIn(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const w...@@ -1285,7 +1285,7 @@ fn windowsMakePipeIn(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const w
1285 wr.* = wr_h;1285 wr.* = wr_h;
1286}1286}
12871287
1288var pipe_name_counter = std.atomic.Atomic(u32).init(1);1288var pipe_name_counter = std.atomic.Value(u32).init(1);
12891289
1290fn windowsMakeAsyncPipe(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {1290fn windowsMakeAsyncPipe(rd: *?windows.HANDLE, wr: *?windows.HANDLE, sattr: *const windows.SECURITY_ATTRIBUTES) !void {
1291 var tmp_bufw: [128]u16 = undefined;1291 var tmp_bufw: [128]u16 = undefined;
lib/std/debug.zig+2-2
...@@ -375,7 +375,7 @@ pub fn panicExtra(...@@ -375,7 +375,7 @@ pub fn panicExtra(
375375
376/// Non-zero whenever the program triggered a panic.376/// Non-zero whenever the program triggered a panic.
377/// The counter is incremented/decremented atomically.377/// The counter is incremented/decremented atomically.
378var panicking = std.atomic.Atomic(u8).init(0);378var panicking = std.atomic.Value(u8).init(0);
379379
380// Locked to avoid interleaving panic messages from multiple threads.380// Locked to avoid interleaving panic messages from multiple threads.
381var panic_mutex = std.Thread.Mutex{};381var panic_mutex = std.Thread.Mutex{};
...@@ -448,7 +448,7 @@ fn waitForOtherThreadToFinishPanicking() void {...@@ -448,7 +448,7 @@ fn waitForOtherThreadToFinishPanicking() void {
448 if (builtin.single_threaded) unreachable;448 if (builtin.single_threaded) unreachable;
449449
450 // Sleep forever without hammering the CPU450 // Sleep forever without hammering the CPU
451 var futex = std.atomic.Atomic(u32).init(0);451 var futex = std.atomic.Value(u32).init(0);
452 while (true) std.Thread.Futex.wait(&futex, 0);452 while (true) std.Thread.Futex.wait(&futex, 0);
453 unreachable;453 unreachable;
454 }454 }
lib/std/event/loop.zig+2-3
...@@ -7,7 +7,6 @@ const os = std.os;...@@ -7,7 +7,6 @@ const os = std.os;
7const windows = os.windows;7const windows = os.windows;
8const maxInt = std.math.maxInt;8const maxInt = std.math.maxInt;
9const Thread = std.Thread;9const Thread = std.Thread;
10const Atomic = std.atomic.Atomic;
1110
12const is_windows = builtin.os.tag == .windows;11const is_windows = builtin.os.tag == .windows;
1312
...@@ -854,7 +853,7 @@ pub const Loop = struct {...@@ -854,7 +853,7 @@ pub const Loop = struct {
854 waiters: Waiters,853 waiters: Waiters,
855 thread: std.Thread,854 thread: std.Thread,
856 event: std.Thread.ResetEvent,855 event: std.Thread.ResetEvent,
857 is_running: Atomic(bool),856 is_running: std.atomic.Value(bool),
858857
859 /// Initialize the delay queue by spawning the timer thread858 /// Initialize the delay queue by spawning the timer thread
860 /// and starting any timer resources.859 /// and starting any timer resources.
...@@ -866,7 +865,7 @@ pub const Loop = struct {...@@ -866,7 +865,7 @@ pub const Loop = struct {
866 },865 },
867 .thread = undefined,866 .thread = undefined,
868 .event = .{},867 .event = .{},
869 .is_running = Atomic(bool).init(true),868 .is_running = std.atomic.Value(bool).init(true),
870 };869 };
871870
872 // Must be after init so that it can read the other state, such as `is_running`.871 // Must be after init so that it can read the other state, such as `is_running`.
lib/std/os.zig+1-1
...@@ -6461,7 +6461,7 @@ pub const CopyFileRangeError = error{...@@ -6461,7 +6461,7 @@ pub const CopyFileRangeError = error{
6461 CorruptedData,6461 CorruptedData,
6462} || PReadError || PWriteError || UnexpectedError;6462} || PReadError || PWriteError || UnexpectedError;
64636463
6464var has_copy_file_range_syscall = std.atomic.Atomic(bool).init(true);6464var has_copy_file_range_syscall = std.atomic.Value(bool).init(true);
64656465
6466/// Transfer data between file descriptors at specified offsets.6466/// Transfer data between file descriptors at specified offsets.
6467/// Returns the number of bytes written, which can less than requested.6467/// Returns the number of bytes written, which can less than requested.
src/crash_report.zig+2-2
...@@ -322,7 +322,7 @@ const PanicSwitch = struct {...@@ -322,7 +322,7 @@ const PanicSwitch = struct {
322 /// Updated atomically before taking the panic_mutex.322 /// Updated atomically before taking the panic_mutex.
323 /// In recoverable cases, the program will not abort323 /// In recoverable cases, the program will not abort
324 /// until all panicking threads have dumped their traces.324 /// until all panicking threads have dumped their traces.
325 var panicking = std.atomic.Atomic(u8).init(0);325 var panicking = std.atomic.Value(u8).init(0);
326326
327 // Locked to avoid interleaving panic messages from multiple threads.327 // Locked to avoid interleaving panic messages from multiple threads.
328 var panic_mutex = std.Thread.Mutex{};328 var panic_mutex = std.Thread.Mutex{};
...@@ -477,7 +477,7 @@ const PanicSwitch = struct {...@@ -477,7 +477,7 @@ const PanicSwitch = struct {
477 // and call abort()477 // and call abort()
478478
479 // Sleep forever without hammering the CPU479 // Sleep forever without hammering the CPU
480 var futex = std.atomic.Atomic(u32).init(0);480 var futex = std.atomic.Value(u32).init(0);
481 while (true) std.Thread.Futex.wait(&futex, 0);481 while (true) std.Thread.Futex.wait(&futex, 0);
482482
483 // This should be unreachable, recurse into recoverAbort.483 // This should be unreachable, recurse into recoverAbort.