authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2025-03-27 12:03:35-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-10-02 16:30:59-07:00
log19e7613a2d9c487c7f6cb9bfe0815c7d33ac68df
tree5edfba9745289d40ec4bf0bff41f290b815ca0ef
parenta9723598d7f5b21144c7ad21c450573524a63736

EventLoop: rewrite context switching

1 files changed, 72 insertions(+), 73 deletions(-)

lib/std/Io/EventLoop.zig+72-73
...@@ -1,4 +1,5 @@...@@ -1,4 +1,5 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const builtin = @import("builtin");
2const assert = std.debug.assert;3const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
4const Io = std.Io;5const Io = std.Io;
...@@ -16,7 +17,7 @@ const max_result_len = 64;...@@ -16,7 +17,7 @@ const max_result_len = 64;
16const min_stack_size = 4 * 1024 * 1024;17const min_stack_size = 4 * 1024 * 1024;
1718
18const Fiber = struct {19const Fiber = struct {
19 regs: Regs,20 context: Context,
20 awaiter: ?*Fiber,21 awaiter: ?*Fiber,
21 queue_node: std.DoublyLinkedList(void).Node,22 queue_node: std.DoublyLinkedList(void).Node,
2223
...@@ -66,34 +67,28 @@ fn allocateFiber(el: *EventLoop, result_len: usize) error{OutOfMemory}!*Fiber {...@@ -66,34 +67,28 @@ fn allocateFiber(el: *EventLoop, result_len: usize) error{OutOfMemory}!*Fiber {
66}67}
6768
68fn yield(el: *EventLoop, optional_fiber: ?*Fiber, register_awaiter: ?*?*Fiber) void {69fn yield(el: *EventLoop, optional_fiber: ?*Fiber, register_awaiter: ?*?*Fiber) void {
70 const ready_fiber: *Fiber = optional_fiber orelse if (ready_node: {
71 el.mutex.lock();
72 defer el.mutex.unlock();
73 break :ready_node el.queue.pop();
74 }) |ready_node|
75 @fieldParentPtr("queue_node", ready_node)
76 else if (register_awaiter) |_| // time to switch to an idle fiber?
77 @panic("no other fiber to switch to in order to be able to register this fiber as an awaiter")
78 else // nothing to do
79 return;
69 const message: SwitchMessage = .{80 const message: SwitchMessage = .{
70 .ready_fiber = optional_fiber orelse if (ready_node: {81 .prev_context = &current_fiber.context,
71 el.mutex.lock();82 .ready_context = &ready_fiber.context,
72 defer el.mutex.unlock();
73 break :ready_node el.queue.pop();
74 }) |ready_node|
75 @fieldParentPtr("queue_node", ready_node)
76 else if (register_awaiter) |_|
77 @panic("no other fiber to switch to in order to be able to register this fiber as an awaiter") // time to switch to an idle fiber?
78 else
79 return, // nothing to do
80 .register_awaiter = register_awaiter,83 .register_awaiter = register_awaiter,
81 };84 };
82 std.log.debug("switching from {*} to {*}", .{ current_fiber, message.ready_fiber });85 std.log.debug("switching from {*} to {*}", .{
83 SwitchMessage.handle(@ptrFromInt(contextSwitch(&current_fiber.regs, &message.ready_fiber.regs, @intFromPtr(&message))), el);86 @as(*Fiber, @fieldParentPtr("context", message.prev_context)),
87 @as(*Fiber, @fieldParentPtr("context", message.ready_context)),
88 });
89 contextSwitch(&message).handle(el);
84}90}
8591
86const SwitchMessage = struct {
87 ready_fiber: *Fiber,
88 register_awaiter: ?*?*Fiber,
89
90 fn handle(message: *const SwitchMessage, el: *EventLoop) void {
91 const prev_fiber = current_fiber;
92 current_fiber = message.ready_fiber;
93 if (message.register_awaiter) |awaiter| if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) el.schedule(prev_fiber);
94 }
95};
96
97fn schedule(el: *EventLoop, fiber: *Fiber) void {92fn schedule(el: *EventLoop, fiber: *Fiber) void {
98 el.mutex.lock();93 el.mutex.lock();
99 defer el.mutex.unlock();94 defer el.mutex.unlock();
...@@ -109,47 +104,62 @@ fn recycle(el: *EventLoop, fiber: *Fiber) void {...@@ -109,47 +104,62 @@ fn recycle(el: *EventLoop, fiber: *Fiber) void {
109 el.free.append(&fiber.queue_node);104 el.free.append(&fiber.queue_node);
110}105}
111106
112const Regs = extern struct {107const SwitchMessage = extern struct {
108 prev_context: *Context,
109 ready_context: *Context,
110 register_awaiter: ?*?*Fiber,
111
112 fn handle(message: *const SwitchMessage, el: *EventLoop) void {
113 const prev_fiber: *Fiber = @fieldParentPtr("context", message.prev_context);
114 current_fiber = @fieldParentPtr("context", message.ready_context);
115 if (message.register_awaiter) |awaiter| if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) el.schedule(prev_fiber);
116 }
117};
118
119const Context = extern struct {
113 rsp: usize,120 rsp: usize,
114 r15: usize,
115 r14: usize,
116 r13: usize,
117 r12: usize,
118 rbx: usize,
119 rbp: usize,121 rbp: usize,
122 rip: usize,
120};123};
121124
122const contextSwitch: *const fn (old: *Regs, new: *Regs, message: usize) callconv(.c) usize = @ptrCast(&contextSwitch_naked);125inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage {
123126 return switch (builtin.cpu.arch) {
124noinline fn contextSwitch_naked() callconv(.naked) void {127 .x86_64 => asm volatile (
125 asm volatile (128 \\ movq 0(%%rsi), %%rax
126 \\movq %%rsp, 0x00(%%rdi)129 \\ movq 8(%%rsi), %%rcx
127 \\movq %%r15, 0x08(%%rdi)130 \\ leaq 0f(%%rip), %%rdx
128 \\movq %%r14, 0x10(%%rdi)131 \\ movq %%rsp, 0(%%rax)
129 \\movq %%r13, 0x18(%%rdi)132 \\ movq %%rbp, 8(%%rax)
130 \\movq %%r12, 0x20(%%rdi)133 \\ movq %%rdx, 16(%%rax)
131 \\movq %%rbx, 0x28(%%rdi)134 \\ movq 0(%%rcx), %%rsp
132 \\movq %%rbp, 0x30(%%rdi)135 \\ movq 8(%%rcx), %%rbp
133 \\136 \\ jmpq *16(%%rcx)
134 \\movq 0x00(%%rsi), %%rsp137 \\0:
135 \\movq 0x08(%%rsi), %%r15138 : [received_message] "={rsi}" (-> *const SwitchMessage),
136 \\movq 0x10(%%rsi), %%r14139 : [message_to_send] "{rsi}" (message),
137 \\movq 0x18(%%rsi), %%r13140 : "rax", "rcx", "rdx", "rbx", "rdi", //
138 \\movq 0x20(%%rsi), %%r12141 "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15", //
139 \\movq 0x28(%%rsi), %%rbx142 "mm0", "mm1", "mm2", "mm3", "mm4", "mm5", "mm6", "mm7", //
140 \\movq 0x30(%%rsi), %%rbp143 "zmm0", "zmm1", "zmm2", "zmm3", "zmm4", "zmm5", "zmm6", "zmm7", //
141 \\144 "zmm8", "zmm9", "zmm10", "zmm11", "zmm12", "zmm13", "zmm14", "zmm15", //
142 \\movq %%rdx, %%rax145 "zmm16", "zmm17", "zmm18", "zmm19", "zmm20", "zmm21", "zmm22", "zmm23", //
143 \\ret146 "zmm24", "zmm25", "zmm26", "zmm27", "zmm28", "zmm29", "zmm30", "zmm31", //
144 );147 "fpsr", "fpcr", "mxcsr", "rflags", "dirflag", "memory"
148 ),
149 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
150 };
145}151}
146152
147fn popRet() callconv(.naked) void {153fn fiberEntry() callconv(.naked) void {
148 asm volatile (154 switch (builtin.cpu.arch) {
149 \\pop %%rdi155 .x86_64 => asm volatile (
150 \\movq %%rax, %%rsi156 \\ leaq 8(%%rsp), %%rdi
151 \\ret157 \\ jmp %[AsyncClosure_call:P]
152 );158 :
159 : [AsyncClosure_call] "X" (&AsyncClosure.call),
160 ),
161 else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
162 }
153}163}
154164
155pub fn @"async"(165pub fn @"async"(
...@@ -179,21 +189,10 @@ pub fn @"async"(...@@ -179,21 +189,10 @@ pub fn @"async"(
179 .start = start,189 .start = start,
180 };190 };
181 const stack_end: [*]align(16) usize = @alignCast(@ptrCast(closure));191 const stack_end: [*]align(16) usize = @alignCast(@ptrCast(closure));
182 const stack_top = (stack_end - 4)[0..4];192 fiber.context = .{
183 stack_top.* = .{193 .rsp = @intFromPtr(stack_end - 1),
184 @intFromPtr(&popRet),
185 @intFromPtr(closure),
186 @intFromPtr(&AsyncClosure.call),
187 0,
188 };
189 fiber.regs = .{
190 .rsp = @intFromPtr(stack_top),
191 .r15 = 0,
192 .r14 = 0,
193 .r13 = 0,
194 .r12 = 0,
195 .rbx = 0,
196 .rbp = 0,194 .rbp = 0,
195 .rip = @intFromPtr(&fiberEntry),
197 };196 };
198197
199 event_loop.schedule(fiber);198 event_loop.schedule(fiber);