authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-03-31 14:36:20-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-02 16:30:59-07:00
log663611773ca5e6438fab82c4697bac6cc9b8e538
tree089fa894fb8560027085825af916415d26b61a5a
parent0f083f24ff9ed532576cb6e4a476bfdb57427c35

EventLoop: implement detached async

data races on deinit tho

2 files changed, 133 insertions(+), 40 deletions(-)

lib/std/Io.zig+31-26
......@@ -626,7 +626,7 @@ pub const VTable = struct {
626626 /// Thread-safe.
627627 cancelRequested: *const fn (?*anyopaque) bool,
628628
629 mutexLock: *const fn (?*anyopaque, prev_state: Mutex.State, mutex: *Mutex) error{Canceled}!void,
629 mutexLock: *const fn (?*anyopaque, prev_state: Mutex.State, mutex: *Mutex) Cancelable!void,
630630 mutexUnlock: *const fn (?*anyopaque, prev_state: Mutex.State, mutex: *Mutex) void,
631631
632632 conditionWait: *const fn (?*anyopaque, cond: *Condition, mutex: *Mutex, timeout_ns: ?u64) Condition.WaitError!void,
......@@ -645,11 +645,11 @@ pub const VTable = struct {
645645pub const OpenFlags = fs.File.OpenFlags;
646646pub const CreateFlags = fs.File.CreateFlags;
647647
648pub const FileOpenError = fs.File.OpenError || error{Canceled};
649pub const FileReadError = fs.File.ReadError || error{Canceled};
650pub const FilePReadError = fs.File.PReadError || error{Canceled};
651pub const FileWriteError = fs.File.WriteError || error{Canceled};
652pub const FilePWriteError = fs.File.PWriteError || error{Canceled};
648pub const FileOpenError = fs.File.OpenError || Cancelable;
649pub const FileReadError = fs.File.ReadError || Cancelable;
650pub const FilePReadError = fs.File.PReadError || Cancelable;
651pub const FileWriteError = fs.File.WriteError || Cancelable;
652pub const FilePWriteError = fs.File.PWriteError || Cancelable;
653653
654654pub const Timestamp = enum(i96) {
655655 _,
......@@ -666,7 +666,7 @@ pub const Deadline = union(enum) {
666666 nanoseconds: i96,
667667 timestamp: Timestamp,
668668};
669pub const ClockGetTimeError = std.posix.ClockGetTimeError || error{Canceled};
669pub const ClockGetTimeError = std.posix.ClockGetTimeError || Cancelable;
670670pub const SleepError = error{ UnsupportedClock, Unexpected, Canceled };
671671
672672pub const AnyFuture = opaque {};
......@@ -734,7 +734,7 @@ pub const Mutex = if (true) struct {
734734 return prev_state.isUnlocked();
735735 }
736736
737 pub fn lock(mutex: *Mutex, io: std.Io) error{Canceled}!void {
737 pub fn lock(mutex: *Mutex, io: std.Io) Cancelable!void {
738738 const prev_state: State = @enumFromInt(@atomicRmw(
739739 usize,
740740 @as(*usize, @ptrCast(&mutex.state)),
......@@ -783,7 +783,7 @@ pub const Mutex = if (true) struct {
783783 }
784784
785785 /// Avoids the vtable for uncontended locks.
786 pub fn lock(m: *Mutex, io: Io) error{Canceled}!void {
786 pub fn lock(m: *Mutex, io: Io) Cancelable!void {
787787 if (!m.tryLock()) {
788788 @branchHint(.unlikely);
789789 try io.vtable.mutexLock(io.userdata, {}, m);
......@@ -809,10 +809,10 @@ pub const Condition = struct {
809809 all,
810810 };
811811
812 pub fn wait(cond: *Condition, io: Io, mutex: *Mutex) void {
812 pub fn wait(cond: *Condition, io: Io, mutex: *Mutex) Cancelable!void {
813813 io.vtable.conditionWait(io.userdata, cond, mutex, null) catch |err| switch (err) {
814814 error.Timeout => unreachable, // no timeout provided so we shouldn't have timed-out
815 error.Canceled => return, // handled as spurious wakeup
815 error.Canceled => return error.Canceled,
816816 };
817817 }
818818
......@@ -829,6 +829,11 @@ pub const Condition = struct {
829829 }
830830};
831831
832pub const Cancelable = error{
833 /// Caller has requested the async operation to stop.
834 Canceled,
835};
836
832837pub const TypeErasedQueue = struct {
833838 mutex: Mutex,
834839
......@@ -852,7 +857,7 @@ pub const TypeErasedQueue = struct {
852857
853858 pub fn init(buffer: []u8) TypeErasedQueue {
854859 return .{
855 .mutex = .{},
860 .mutex = .init,
856861 .buffer = buffer,
857862 .put_index = 0,
858863 .get_index = 0,
......@@ -861,10 +866,10 @@ pub const TypeErasedQueue = struct {
861866 };
862867 }
863868
864 pub fn put(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize) usize {
869 pub fn put(q: *TypeErasedQueue, io: Io, elements: []const u8, min: usize) Cancelable!usize {
865870 assert(elements.len >= min);
866871
867 q.mutex.lock(io);
872 try q.mutex.lock(io);
868873 defer q.mutex.unlock(io);
869874
870875 // Getters have first priority on the data, and only when the getters
......@@ -911,15 +916,15 @@ pub const TypeErasedQueue = struct {
911916 .data = .{ .remaining = remaining, .condition = .{} },
912917 };
913918 q.putters.append(&node);
914 node.data.condition.wait(io, &q.mutex);
919 try node.data.condition.wait(io, &q.mutex);
915920 remaining = node.data.remaining;
916921 }
917922 }
918923
919 pub fn get(q: *@This(), io: Io, buffer: []u8, min: usize) usize {
924 pub fn get(q: *@This(), io: Io, buffer: []u8, min: usize) Cancelable!usize {
920925 assert(buffer.len >= min);
921926
922 q.mutex.lock(io);
927 try q.mutex.lock(io);
923928 defer q.mutex.unlock(io);
924929
925930 // The ring buffer gets first priority, then data should come from any
......@@ -976,7 +981,7 @@ pub const TypeErasedQueue = struct {
976981 .data = .{ .remaining = remaining, .condition = .{} },
977982 };
978983 q.getters.append(&node);
979 node.data.condition.wait(io, &q.mutex);
984 try node.data.condition.wait(io, &q.mutex);
980985 remaining = node.data.remaining;
981986 }
982987 }
......@@ -1030,8 +1035,8 @@ pub fn Queue(Elem: type) type {
10301035 /// Returns how many elements have been added to the queue.
10311036 ///
10321037 /// Asserts that `elements.len >= min`.
1033 pub fn put(q: *@This(), io: Io, elements: []const Elem, min: usize) usize {
1034 return @divExact(q.type_erased.put(io, @ptrCast(elements), min * @sizeOf(Elem)), @sizeOf(Elem));
1038 pub fn put(q: *@This(), io: Io, elements: []const Elem, min: usize) Cancelable!usize {
1039 return @divExact(try q.type_erased.put(io, @ptrCast(elements), min * @sizeOf(Elem)), @sizeOf(Elem));
10351040 }
10361041
10371042 /// Receives elements from the beginning of the queue. The function
......@@ -1041,17 +1046,17 @@ pub fn Queue(Elem: type) type {
10411046 /// Returns how many elements of `buffer` have been populated.
10421047 ///
10431048 /// Asserts that `buffer.len >= min`.
1044 pub fn get(q: *@This(), io: Io, buffer: []Elem, min: usize) usize {
1045 return @divExact(q.type_erased.get(io, @ptrCast(buffer), min * @sizeOf(Elem)), @sizeOf(Elem));
1049 pub fn get(q: *@This(), io: Io, buffer: []Elem, min: usize) Cancelable!usize {
1050 return @divExact(try q.type_erased.get(io, @ptrCast(buffer), min * @sizeOf(Elem)), @sizeOf(Elem));
10461051 }
10471052
1048 pub fn putOne(q: *@This(), io: Io, item: Elem) void {
1049 assert(q.put(io, &.{item}, 1) == 1);
1053 pub fn putOne(q: *@This(), io: Io, item: Elem) Cancelable!void {
1054 assert(try q.put(io, &.{item}, 1) == 1);
10501055 }
10511056
1052 pub fn getOne(q: *@This(), io: Io) Elem {
1057 pub fn getOne(q: *@This(), io: Io) Cancelable!Elem {
10531058 var buf: [1]Elem = undefined;
1054 assert(q.get(io, &buf, 1) == 1);
1059 assert(try q.get(io, &buf, 1) == 1);
10551060 return buf[0];
10561061 }
10571062 };
lib/std/Io/EventLoop.zig+102-14
......@@ -27,6 +27,7 @@ const Thread = struct {
2727 current_context: *Context,
2828 ready_queue: ?*Fiber,
2929 free_queue: ?*Fiber,
30 detached_queue: ?*Fiber,
3031 io_uring: IoUring,
3132 idle_search_index: u32,
3233 steal_ready_search_index: u32,
......@@ -208,6 +209,7 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void {
208209 .current_context = &main_fiber.context,
209210 .ready_queue = null,
210211 .free_queue = null,
212 .detached_queue = null,
211213 .io_uring = try IoUring.init(io_uring_entries, 0),
212214 .idle_search_index = 1,
213215 .steal_ready_search_index = 1,
......@@ -218,7 +220,16 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void {
218220}
219221
220222pub fn deinit(el: *EventLoop) void {
223 // Wait for detached fibers.
221224 const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);
225 for (el.threads.allocated[0..active_threads]) |*thread| {
226 while (thread.detached_queue) |detached_fiber| {
227 if (@atomicLoad(?*Fiber, &detached_fiber.awaiter, .acquire) != Fiber.finished)
228 el.yield(null, .{ .register_awaiter = &detached_fiber.awaiter });
229 detached_fiber.recycle();
230 }
231 }
232
222233 for (el.threads.allocated[0..active_threads]) |*thread| {
223234 const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
224235 assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
......@@ -336,6 +347,7 @@ fn schedule(el: *EventLoop, thread: *Thread, ready_queue: Fiber.Queue) void {
336347 .current_context = &new_thread.idle_context,
337348 .ready_queue = ready_queue.head,
338349 .free_queue = null,
350 .detached_queue = null,
339351 .io_uring = IoUring.init(io_uring_entries, 0) catch |err| {
340352 @atomicStore(u32, &el.threads.reserved, new_thread_index, .release);
341353 // no more access to `thread` after giving up reservation
......@@ -470,6 +482,7 @@ const SwitchMessage = struct {
470482 const PendingTask = union(enum) {
471483 nothing,
472484 reschedule,
485 recycle: *Fiber,
473486 register_awaiter: *?*Fiber,
474487 lock_mutex: struct {
475488 prev_state: Io.Mutex.State,
......@@ -488,6 +501,9 @@ const SwitchMessage = struct {
488501 assert(prev_fiber.queue_next == null);
489502 el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
490503 },
504 .recycle => |fiber| {
505 fiber.recycle();
506 },
491507 .register_awaiter => |awaiter| {
492508 const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
493509 assert(prev_fiber.queue_next == null);
......@@ -612,6 +628,18 @@ fn fiberEntry() callconv(.naked) void {
612628 }
613629}
614630
631fn fiberEntryDetached() callconv(.naked) void {
632 switch (builtin.cpu.arch) {
633 .x86_64 => asm volatile (
634 \\ leaq 8(%%rsp), %%rdi
635 \\ jmp %[DetachedClosure_call:P]
636 :
637 : [DetachedClosure_call] "X" (&DetachedClosure.call),
638 ),
639 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
640 }
641}
642
615643const AsyncClosure = struct {
616644 event_loop: *EventLoop,
617645 fiber: *Fiber,
......@@ -632,6 +660,31 @@ const AsyncClosure = struct {
632660 }
633661};
634662
663const DetachedClosure = struct {
664 event_loop: *EventLoop,
665 fiber: *Fiber,
666 start: *const fn (context: *const anyopaque) void,
667
668 fn contextPointer(closure: *DetachedClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
669 return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(DetachedClosure));
670 }
671
672 fn call(closure: *DetachedClosure, message: *const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(DetachedClosure))) noreturn {
673 message.handle(closure.event_loop);
674 std.log.debug("{*} performing async detached", .{closure.fiber});
675 closure.start(closure.contextPointer());
676 const current_thread: *Thread = .current();
677 current_thread.detached_queue = closure.fiber.queue_next;
678 const awaiter = @atomicRmw(?*Fiber, &closure.fiber.awaiter, .Xchg, Fiber.finished, .acq_rel);
679 if (awaiter) |a| {
680 closure.event_loop.yield(a, .nothing);
681 } else {
682 closure.event_loop.yield(null, .{ .recycle = closure.fiber });
683 }
684 unreachable; // switched to dead fiber
685 }
686};
687
635688fn @"async"(
636689 userdata: ?*anyopaque,
637690 result: []u8,
......@@ -682,6 +735,53 @@ fn @"async"(
682735 return @ptrCast(fiber);
683736}
684737
738fn go(
739 userdata: ?*anyopaque,
740 context: []const u8,
741 context_alignment: std.mem.Alignment,
742 start: *const fn (context: *const anyopaque) void,
743) void {
744 assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
745 assert(context.len <= Fiber.max_context_size); // TODO
746
747 const event_loop: *EventLoop = @alignCast(@ptrCast(userdata));
748 const fiber = Fiber.allocate(event_loop) catch {
749 start(context.ptr);
750 return;
751 };
752 std.log.debug("allocated {*}", .{fiber});
753
754 const current_thread: *Thread = .current();
755 const closure: *DetachedClosure = @ptrFromInt(Fiber.max_context_align.max(.of(DetachedClosure)).backward(
756 @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
757 ) - @sizeOf(DetachedClosure));
758 fiber.* = .{
759 .required_align = {},
760 .context = switch (builtin.cpu.arch) {
761 .x86_64 => .{
762 .rsp = @intFromPtr(closure) - @sizeOf(usize),
763 .rbp = 0,
764 .rip = @intFromPtr(&fiberEntryDetached),
765 },
766 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
767 },
768 .awaiter = null,
769 .queue_next = current_thread.detached_queue,
770 .cancel_thread = null,
771 .awaiting_completions = .initEmpty(),
772 };
773 current_thread.detached_queue = fiber;
774 closure.* = .{
775 .event_loop = event_loop,
776 .fiber = fiber,
777 .start = start,
778 };
779 @memcpy(closure.contextPointer(), context);
780
781 event_loop.schedule(current_thread, .{ .head = fiber, .tail = fiber });
782}
783
784
685785fn @"await"(
686786 userdata: ?*anyopaque,
687787 any_future: *std.Io.AnyFuture,
......@@ -690,24 +790,12 @@ fn @"await"(
690790) void {
691791 const event_loop: *EventLoop = @alignCast(@ptrCast(userdata));
692792 const future_fiber: *Fiber = @alignCast(@ptrCast(any_future));
693 if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished) event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter });
793 if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished)
794 event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter });
694795 @memcpy(result, future_fiber.resultBytes(result_alignment));
695796 future_fiber.recycle();
696797}
697798
698fn go(
699 userdata: ?*anyopaque,
700 context: []const u8,
701 context_alignment: std.mem.Alignment,
702 start: *const fn (context: *const anyopaque) void,
703) void {
704 _ = userdata;
705 _ = context;
706 _ = context_alignment;
707 _ = start;
708 @panic("TODO");
709}
710
711799fn cancel(
712800 userdata: ?*anyopaque,
713801 any_future: *std.Io.AnyFuture,