| author |  Andrew Kelley <andrew@ziglang.org> 2025-10-15 14:47:41-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2025-10-29 06:20:49-07:00 |
| log | 1e81c3a9259873fac197ed5dc6e2133f0a097243 |
| tree | aff7722e47948044694d90a752caa1c3654e689c |
| parent | 81b1bfbfbbf4f71bc79191ca36ac62c00c8ac92c |
std.Io: rename EventLoop to IoUring
`std.Io.Evented` is introduced to select an appropriate Io
implementation depending on OS3 files changed, 1622 insertions(+), 1619 deletions(-)
lib
lib/std/Io.zig+4-1
| ... | @@ -557,7 +557,10 @@ test { | ... | @@ -557,7 +557,10 @@ test { |
| 557 | 557 | ||
| 558 | const Io = @This(); | 558 | const Io = @This(); |
| 559 | 559 | ||
| 560 | pub const EventLoop = @import("Io/EventLoop.zig"); | 560 | pub const Evented = switch (builtin.os.tag) { |
| 561 | .linux => @import("Io/IoUring.zig"), | ||
| 562 | else => void, | ||
| 563 | }; | ||
| 561 | pub const Threaded = @import("Io/Threaded.zig"); | 564 | pub const Threaded = @import("Io/Threaded.zig"); |
| 562 | pub const net = @import("Io/net.zig"); | 565 | pub const net = @import("Io/net.zig"); |
| 563 | 566 |
lib/std/Io/EventLoop.zig deleted-1618
| ... | @@ -1,1618 +0,0 @@ | ||
| 1 | const std = @import("../std.zig"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | const assert = std.debug.assert; | ||
| 4 | const Allocator = std.mem.Allocator; | ||
| 5 | const Io = std.Io; | ||
| 6 | const EventLoop = @This(); | ||
| 7 | const Alignment = std.mem.Alignment; | ||
| 8 | const IoUring = std.os.linux.IoUring; | ||
| 9 | |||
| 10 | /// Must be a thread-safe allocator. | ||
| 11 | gpa: Allocator, | ||
| 12 | main_fiber_buffer: [@sizeOf(Fiber) + Fiber.max_result_size]u8 align(@alignOf(Fiber)), | ||
| 13 | threads: Thread.List, | ||
| 14 | detached: struct { | ||
| 15 | mutex: std.Io.Mutex, | ||
| 16 | list: std.DoublyLinkedList, | ||
| 17 | }, | ||
| 18 | |||
| 19 | /// Empirically saw >128KB being used by the self-hosted backend to panic. | ||
| 20 | const idle_stack_size = 256 * 1024; | ||
| 21 | |||
| 22 | const max_idle_search = 4; | ||
| 23 | const max_steal_ready_search = 4; | ||
| 24 | |||
| 25 | const io_uring_entries = 64; | ||
| 26 | |||
| 27 | const Thread = struct { | ||
| 28 | thread: std.Thread, | ||
| 29 | idle_context: Context, | ||
| 30 | current_context: *Context, | ||
| 31 | ready_queue: ?*Fiber, | ||
| 32 | io_uring: IoUring, | ||
| 33 | idle_search_index: u32, | ||
| 34 | steal_ready_search_index: u32, | ||
| 35 | |||
| 36 | const canceling: ?*Thread = @ptrFromInt(@alignOf(Thread)); | ||
| 37 | |||
| 38 | threadlocal var self: *Thread = undefined; | ||
| 39 | |||
| 40 | fn current() *Thread { | ||
| 41 | return self; | ||
| 42 | } | ||
| 43 | |||
| 44 | fn currentFiber(thread: *Thread) *Fiber { | ||
| 45 | return @fieldParentPtr("context", thread.current_context); | ||
| 46 | } | ||
| 47 | |||
| 48 | const List = struct { | ||
| 49 | allocated: []Thread, | ||
| 50 | reserved: u32, | ||
| 51 | active: u32, | ||
| 52 | }; | ||
| 53 | }; | ||
| 54 | |||
| 55 | const Fiber = struct { | ||
| 56 | required_align: void align(4), | ||
| 57 | context: Context, | ||
| 58 | awaiter: ?*Fiber, | ||
| 59 | queue_next: ?*Fiber, | ||
| 60 | cancel_thread: ?*Thread, | ||
| 61 | awaiting_completions: std.StaticBitSet(3), | ||
| 62 | |||
| 63 | const finished: ?*Fiber = @ptrFromInt(@alignOf(Thread)); | ||
| 64 | |||
| 65 | const max_result_align: Alignment = .@"16"; | ||
| 66 | const max_result_size = max_result_align.forward(64); | ||
| 67 | /// This includes any stack realignments that need to happen, and also the | ||
| 68 | /// initial frame return address slot and argument frame, depending on target. | ||
| 69 | const min_stack_size = 4 * 1024 * 1024; | ||
| 70 | const max_context_align: Alignment = .@"16"; | ||
| 71 | const max_context_size = max_context_align.forward(1024); | ||
| 72 | const max_closure_size: usize = @max(@sizeOf(AsyncClosure), @sizeOf(DetachedClosure)); | ||
| 73 | const max_closure_align: Alignment = .max(.of(AsyncClosure), .of(DetachedClosure)); | ||
| 74 | const allocation_size = std.mem.alignForward( | ||
| 75 | usize, | ||
| 76 | max_closure_align.max(max_context_align).forward( | ||
| 77 | max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size, | ||
| 78 | ) + max_closure_size + max_context_size, | ||
| 79 | std.heap.page_size_max, | ||
| 80 | ); | ||
| 81 | |||
| 82 | fn allocate(el: *EventLoop) error{OutOfMemory}!*Fiber { | ||
| 83 | return @ptrCast(try el.gpa.alignedAlloc(u8, .of(Fiber), allocation_size)); | ||
| 84 | } | ||
| 85 | |||
| 86 | fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 { | ||
| 87 | return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size]; | ||
| 88 | } | ||
| 89 | |||
| 90 | fn allocatedEnd(f: *Fiber) [*]u8 { | ||
| 91 | const allocated_slice = f.allocatedSlice(); | ||
| 92 | return allocated_slice[allocated_slice.len..].ptr; | ||
| 93 | } | ||
| 94 | |||
| 95 | fn resultPointer(f: *Fiber, comptime Result: type) *Result { | ||
| 96 | return @ptrCast(@alignCast(f.resultBytes(.of(Result)))); | ||
| 97 | } | ||
| 98 | |||
| 99 | fn resultBytes(f: *Fiber, alignment: Alignment) [*]u8 { | ||
| 100 | return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber))); | ||
| 101 | } | ||
| 102 | |||
| 103 | fn enterCancelRegion(fiber: *Fiber, thread: *Thread) error{Canceled}!void { | ||
| 104 | if (@cmpxchgStrong( | ||
| 105 | ?*Thread, | ||
| 106 | &fiber.cancel_thread, | ||
| 107 | null, | ||
| 108 | thread, | ||
| 109 | .acq_rel, | ||
| 110 | .acquire, | ||
| 111 | )) |cancel_thread| { | ||
| 112 | assert(cancel_thread == Thread.canceling); | ||
| 113 | return error.Canceled; | ||
| 114 | } | ||
| 115 | } | ||
| 116 | |||
| 117 | fn exitCancelRegion(fiber: *Fiber, thread: *Thread) void { | ||
| 118 | if (@cmpxchgStrong( | ||
| 119 | ?*Thread, | ||
| 120 | &fiber.cancel_thread, | ||
| 121 | thread, | ||
| 122 | null, | ||
| 123 | .acq_rel, | ||
| 124 | .acquire, | ||
| 125 | )) |cancel_thread| assert(cancel_thread == Thread.canceling); | ||
| 126 | } | ||
| 127 | |||
| 128 | const Queue = struct { head: *Fiber, tail: *Fiber }; | ||
| 129 | }; | ||
| 130 | |||
| 131 | fn recycle(el: *EventLoop, fiber: *Fiber) void { | ||
| 132 | std.log.debug("recyling {*}", .{fiber}); | ||
| 133 | assert(fiber.queue_next == null); | ||
| 134 | el.gpa.free(fiber.allocatedSlice()); | ||
| 135 | } | ||
| 136 | |||
| 137 | pub fn io(el: *EventLoop) Io { | ||
| 138 | return .{ | ||
| 139 | .userdata = el, | ||
| 140 | .vtable = &.{ | ||
| 141 | .async = async, | ||
| 142 | .concurrent = concurrent, | ||
| 143 | .await = await, | ||
| 144 | .asyncDetached = asyncDetached, | ||
| 145 | .select = select, | ||
| 146 | .cancel = cancel, | ||
| 147 | .cancelRequested = cancelRequested, | ||
| 148 | |||
| 149 | .mutexLock = mutexLock, | ||
| 150 | .mutexUnlock = mutexUnlock, | ||
| 151 | |||
| 152 | .conditionWait = conditionWait, | ||
| 153 | .conditionWake = conditionWake, | ||
| 154 | |||
| 155 | .createFile = createFile, | ||
| 156 | .fileOpen = fileOpen, | ||
| 157 | .fileClose = fileClose, | ||
| 158 | .pread = pread, | ||
| 159 | .pwrite = pwrite, | ||
| 160 | |||
| 161 | .now = now, | ||
| 162 | .sleep = sleep, | ||
| 163 | }, | ||
| 164 | }; | ||
| 165 | } | ||
| 166 | |||
| 167 | pub fn init(el: *EventLoop, gpa: Allocator) !void { | ||
| 168 | const threads_size = @max(std.Thread.getCpuCount() catch 1, 1) * @sizeOf(Thread); | ||
| 169 | const idle_stack_end_offset = std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max); | ||
| 170 | const allocated_slice = try gpa.alignedAlloc(u8, .of(Thread), idle_stack_end_offset); | ||
| 171 | errdefer gpa.free(allocated_slice); | ||
| 172 | el.* = .{ | ||
| 173 | .gpa = gpa, | ||
| 174 | .main_fiber_buffer = undefined, | ||
| 175 | .threads = .{ | ||
| 176 | .allocated = @ptrCast(allocated_slice[0..threads_size]), | ||
| 177 | .reserved = 1, | ||
| 178 | .active = 1, | ||
| 179 | }, | ||
| 180 | .detached = .{ | ||
| 181 | .mutex = .init, | ||
| 182 | .list = .{}, | ||
| 183 | }, | ||
| 184 | }; | ||
| 185 | const main_fiber: *Fiber = @ptrCast(&el.main_fiber_buffer); | ||
| 186 | main_fiber.* = .{ | ||
| 187 | .required_align = {}, | ||
| 188 | .context = undefined, | ||
| 189 | .awaiter = null, | ||
| 190 | .queue_next = null, | ||
| 191 | .cancel_thread = null, | ||
| 192 | .awaiting_completions = .initEmpty(), | ||
| 193 | }; | ||
| 194 | const main_thread = &el.threads.allocated[0]; | ||
| 195 | Thread.self = main_thread; | ||
| 196 | const idle_stack_end: [*]align(16) usize = @ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr)); | ||
| 197 | (idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)}; | ||
| 198 | main_thread.* = .{ | ||
| 199 | .thread = undefined, | ||
| 200 | .idle_context = switch (builtin.cpu.arch) { | ||
| 201 | .aarch64 => .{ | ||
| 202 | .sp = @intFromPtr(idle_stack_end), | ||
| 203 | .fp = 0, | ||
| 204 | .pc = @intFromPtr(&mainIdleEntry), | ||
| 205 | }, | ||
| 206 | .x86_64 => .{ | ||
| 207 | .rsp = @intFromPtr(idle_stack_end - 1), | ||
| 208 | .rbp = 0, | ||
| 209 | .rip = @intFromPtr(&mainIdleEntry), | ||
| 210 | }, | ||
| 211 | else => @compileError("unimplemented architecture"), | ||
| 212 | }, | ||
| 213 | .current_context = &main_fiber.context, | ||
| 214 | .ready_queue = null, | ||
| 215 | .io_uring = try IoUring.init(io_uring_entries, 0), | ||
| 216 | .idle_search_index = 1, | ||
| 217 | .steal_ready_search_index = 1, | ||
| 218 | }; | ||
| 219 | errdefer main_thread.io_uring.deinit(); | ||
| 220 | std.log.debug("created main idle {*}", .{&main_thread.idle_context}); | ||
| 221 | std.log.debug("created main {*}", .{main_fiber}); | ||
| 222 | } | ||
| 223 | |||
| 224 | pub fn deinit(el: *EventLoop) void { | ||
| 225 | while (true) cancel(el, detached_future: { | ||
| 226 | el.detached.mutex.lock(el.io()) catch |err| switch (err) { | ||
| 227 | error.Canceled => unreachable, // main fiber cannot be canceled | ||
| 228 | }; | ||
| 229 | defer el.detached.mutex.unlock(el.io()); | ||
| 230 | const detached: *DetachedClosure = @fieldParentPtr( | ||
| 231 | "detached_queue_node", | ||
| 232 | el.detached.list.pop() orelse break, | ||
| 233 | ); | ||
| 234 | // notify the detached fiber that it is no longer allowed to recycle itself | ||
| 235 | detached.detached_queue_node = .{ | ||
| 236 | .prev = &detached.detached_queue_node, | ||
| 237 | .next = &detached.detached_queue_node, | ||
| 238 | }; | ||
| 239 | break :detached_future @ptrCast(detached.fiber); | ||
| 240 | }, &.{}, .@"1"); | ||
| 241 | const active_threads = @atomicLoad(u32, &el.threads.active, .acquire); | ||
| 242 | for (el.threads.allocated[0..active_threads]) |*thread| { | ||
| 243 | const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic); | ||
| 244 | assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async | ||
| 245 | } | ||
| 246 | el.yield(null, .exit); | ||
| 247 | const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(el.threads.allocated.ptr)); | ||
| 248 | const idle_stack_end_offset = std.mem.alignForward(usize, el.threads.allocated.len * @sizeOf(Thread) + idle_stack_size, std.heap.page_size_max); | ||
| 249 | for (el.threads.allocated[1..active_threads]) |*thread| thread.thread.join(); | ||
| 250 | el.gpa.free(allocated_ptr[0..idle_stack_end_offset]); | ||
| 251 | el.* = undefined; | ||
| 252 | } | ||
| 253 | |||
| 254 | fn findReadyFiber(el: *EventLoop, thread: *Thread) ?*Fiber { | ||
| 255 | if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) |ready_fiber| { | ||
| 256 | @atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release); | ||
| 257 | ready_fiber.queue_next = null; | ||
| 258 | return ready_fiber; | ||
| 259 | } | ||
| 260 | const active_threads = @atomicLoad(u32, &el.threads.active, .acquire); | ||
| 261 | for (0..@min(max_steal_ready_search, active_threads)) |_| { | ||
| 262 | defer thread.steal_ready_search_index += 1; | ||
| 263 | if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0; | ||
| 264 | const steal_ready_search_thread = &el.threads.allocated[0..active_threads][thread.steal_ready_search_index]; | ||
| 265 | if (steal_ready_search_thread == thread) continue; | ||
| 266 | const ready_fiber = @atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue; | ||
| 267 | if (ready_fiber == Fiber.finished) continue; | ||
| 268 | if (@cmpxchgWeak( | ||
| 269 | ?*Fiber, | ||
| 270 | &steal_ready_search_thread.ready_queue, | ||
| 271 | ready_fiber, | ||
| 272 | null, | ||
| 273 | .acquire, | ||
| 274 | .monotonic, | ||
| 275 | )) |_| continue; | ||
| 276 | @atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release); | ||
| 277 | ready_fiber.queue_next = null; | ||
| 278 | return ready_fiber; | ||
| 279 | } | ||
| 280 | // couldn't find anything to do, so we are now open for business | ||
| 281 | @atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic); | ||
| 282 | return null; | ||
| 283 | } | ||
| 284 | |||
| 285 | fn yield(el: *EventLoop, maybe_ready_fiber: ?*Fiber, pending_task: SwitchMessage.PendingTask) void { | ||
| 286 | const thread: *Thread = .current(); | ||
| 287 | const ready_context = if (maybe_ready_fiber orelse el.findReadyFiber(thread)) |ready_fiber| | ||
| 288 | &ready_fiber.context | ||
| 289 | else | ||
| 290 | &thread.idle_context; | ||
| 291 | const message: SwitchMessage = .{ | ||
| 292 | .contexts = .{ | ||
| 293 | .prev = thread.current_context, | ||
| 294 | .ready = ready_context, | ||
| 295 | }, | ||
| 296 | .pending_task = pending_task, | ||
| 297 | }; | ||
| 298 | std.log.debug("switching from {*} to {*}", .{ message.contexts.prev, message.contexts.ready }); | ||
| 299 | contextSwitch(&message).handle(el); | ||
| 300 | } | ||
| 301 | |||
| 302 | fn schedule(el: *EventLoop, thread: *Thread, ready_queue: Fiber.Queue) void { | ||
| 303 | { | ||
| 304 | var fiber = ready_queue.head; | ||
| 305 | while (true) { | ||
| 306 | std.log.debug("scheduling {*}", .{fiber}); | ||
| 307 | fiber = fiber.queue_next orelse break; | ||
| 308 | } | ||
| 309 | assert(fiber == ready_queue.tail); | ||
| 310 | } | ||
| 311 | // shared fields of previous `Thread` must be initialized before later ones are marked as active | ||
| 312 | const new_thread_index = @atomicLoad(u32, &el.threads.active, .acquire); | ||
| 313 | for (0..@min(max_idle_search, new_thread_index)) |_| { | ||
| 314 | defer thread.idle_search_index += 1; | ||
| 315 | if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0; | ||
| 316 | const idle_search_thread = &el.threads.allocated[0..new_thread_index][thread.idle_search_index]; | ||
| 317 | if (idle_search_thread == thread) continue; | ||
| 318 | if (@cmpxchgWeak( | ||
| 319 | ?*Fiber, | ||
| 320 | &idle_search_thread.ready_queue, | ||
| 321 | null, | ||
| 322 | ready_queue.head, | ||
| 323 | .release, | ||
| 324 | .monotonic, | ||
| 325 | )) |_| continue; | ||
| 326 | getSqe(&thread.io_uring).* = .{ | ||
| 327 | .opcode = .MSG_RING, | ||
| 328 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 329 | .ioprio = 0, | ||
| 330 | .fd = idle_search_thread.io_uring.fd, | ||
| 331 | .off = @intFromEnum(Completion.UserData.wakeup), | ||
| 332 | .addr = 0, | ||
| 333 | .len = 0, | ||
| 334 | .rw_flags = 0, | ||
| 335 | .user_data = @intFromEnum(Completion.UserData.wakeup), | ||
| 336 | .buf_index = 0, | ||
| 337 | .personality = 0, | ||
| 338 | .splice_fd_in = 0, | ||
| 339 | .addr3 = 0, | ||
| 340 | .resv = 0, | ||
| 341 | }; | ||
| 342 | return; | ||
| 343 | } | ||
| 344 | spawn_thread: { | ||
| 345 | // previous failed reservations must have completed before retrying | ||
| 346 | if (new_thread_index == el.threads.allocated.len or @cmpxchgWeak( | ||
| 347 | u32, | ||
| 348 | &el.threads.reserved, | ||
| 349 | new_thread_index, | ||
| 350 | new_thread_index + 1, | ||
| 351 | .acquire, | ||
| 352 | .monotonic, | ||
| 353 | ) != null) break :spawn_thread; | ||
| 354 | const new_thread = &el.threads.allocated[new_thread_index]; | ||
| 355 | const next_thread_index = new_thread_index + 1; | ||
| 356 | new_thread.* = .{ | ||
| 357 | .thread = undefined, | ||
| 358 | .idle_context = undefined, | ||
| 359 | .current_context = &new_thread.idle_context, | ||
| 360 | .ready_queue = ready_queue.head, | ||
| 361 | .io_uring = IoUring.init(io_uring_entries, 0) catch |err| { | ||
| 362 | @atomicStore(u32, &el.threads.reserved, new_thread_index, .release); | ||
| 363 | // no more access to `thread` after giving up reservation | ||
| 364 | std.log.warn("unable to create worker thread due to io_uring init failure: {s}", .{@errorName(err)}); | ||
| 365 | break :spawn_thread; | ||
| 366 | }, | ||
| 367 | .idle_search_index = 0, | ||
| 368 | .steal_ready_search_index = 0, | ||
| 369 | }; | ||
| 370 | new_thread.thread = std.Thread.spawn(.{ | ||
| 371 | .stack_size = idle_stack_size, | ||
| 372 | .allocator = el.gpa, | ||
| 373 | }, threadEntry, .{ el, new_thread_index }) catch |err| { | ||
| 374 | new_thread.io_uring.deinit(); | ||
| 375 | @atomicStore(u32, &el.threads.reserved, new_thread_index, .release); | ||
| 376 | // no more access to `thread` after giving up reservation | ||
| 377 | std.log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)}); | ||
| 378 | break :spawn_thread; | ||
| 379 | }; | ||
| 380 | // shared fields of `Thread` must be initialized before being marked active | ||
| 381 | @atomicStore(u32, &el.threads.active, next_thread_index, .release); | ||
| 382 | return; | ||
| 383 | } | ||
| 384 | // nobody wanted it, so just queue it on ourselves | ||
| 385 | while (@cmpxchgWeak( | ||
| 386 | ?*Fiber, | ||
| 387 | &thread.ready_queue, | ||
| 388 | ready_queue.tail.queue_next, | ||
| 389 | ready_queue.head, | ||
| 390 | .acq_rel, | ||
| 391 | .acquire, | ||
| 392 | )) |old_head| ready_queue.tail.queue_next = old_head; | ||
| 393 | } | ||
| 394 | |||
| 395 | fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn { | ||
| 396 | message.handle(el); | ||
| 397 | el.idle(&el.threads.allocated[0]); | ||
| 398 | el.yield(@ptrCast(&el.main_fiber_buffer), .nothing); | ||
| 399 | unreachable; // switched to dead fiber | ||
| 400 | } | ||
| 401 | |||
| 402 | fn threadEntry(el: *EventLoop, index: u32) void { | ||
| 403 | const thread: *Thread = &el.threads.allocated[index]; | ||
| 404 | Thread.self = thread; | ||
| 405 | std.log.debug("created thread idle {*}", .{&thread.idle_context}); | ||
| 406 | el.idle(thread); | ||
| 407 | } | ||
| 408 | |||
| 409 | const Completion = struct { | ||
| 410 | const UserData = enum(usize) { | ||
| 411 | unused, | ||
| 412 | wakeup, | ||
| 413 | cleanup, | ||
| 414 | exit, | ||
| 415 | /// *Fiber | ||
| 416 | _, | ||
| 417 | }; | ||
| 418 | result: i32, | ||
| 419 | flags: u32, | ||
| 420 | }; | ||
| 421 | |||
| 422 | fn idle(el: *EventLoop, thread: *Thread) void { | ||
| 423 | var maybe_ready_fiber: ?*Fiber = null; | ||
| 424 | while (true) { | ||
| 425 | while (maybe_ready_fiber orelse el.findReadyFiber(thread)) |ready_fiber| { | ||
| 426 | el.yield(ready_fiber, .nothing); | ||
| 427 | maybe_ready_fiber = null; | ||
| 428 | } | ||
| 429 | _ = thread.io_uring.submit_and_wait(1) catch |err| switch (err) { | ||
| 430 | error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}), | ||
| 431 | else => |e| @panic(@errorName(e)), | ||
| 432 | }; | ||
| 433 | var cqes_buffer: [io_uring_entries]std.os.linux.io_uring_cqe = undefined; | ||
| 434 | var maybe_ready_queue: ?Fiber.Queue = null; | ||
| 435 | for (cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch |err| switch (err) { | ||
| 436 | error.SignalInterrupt => cqes_len: { | ||
| 437 | std.log.warn("copy_cqes failed with SignalInterrupt", .{}); | ||
| 438 | break :cqes_len 0; | ||
| 439 | }, | ||
| 440 | else => |e| @panic(@errorName(e)), | ||
| 441 | }]) |cqe| switch (@as(Completion.UserData, @enumFromInt(cqe.user_data))) { | ||
| 442 | .unused => unreachable, // bad submission queued? | ||
| 443 | .wakeup => {}, | ||
| 444 | .cleanup => @panic("failed to notify other threads that we are exiting"), | ||
| 445 | .exit => { | ||
| 446 | assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async | ||
| 447 | return; | ||
| 448 | }, | ||
| 449 | _ => switch (errno(cqe.res)) { | ||
| 450 | .INTR => getSqe(&thread.io_uring).* = .{ | ||
| 451 | .opcode = .ASYNC_CANCEL, | ||
| 452 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 453 | .ioprio = 0, | ||
| 454 | .fd = 0, | ||
| 455 | .off = 0, | ||
| 456 | .addr = cqe.user_data, | ||
| 457 | .len = 0, | ||
| 458 | .rw_flags = 0, | ||
| 459 | .user_data = @intFromEnum(Completion.UserData.wakeup), | ||
| 460 | .buf_index = 0, | ||
| 461 | .personality = 0, | ||
| 462 | .splice_fd_in = 0, | ||
| 463 | .addr3 = 0, | ||
| 464 | .resv = 0, | ||
| 465 | }, | ||
| 466 | else => { | ||
| 467 | const fiber: *Fiber = @ptrFromInt(cqe.user_data); | ||
| 468 | assert(fiber.queue_next == null); | ||
| 469 | fiber.resultPointer(Completion).* = .{ | ||
| 470 | .result = cqe.res, | ||
| 471 | .flags = cqe.flags, | ||
| 472 | }; | ||
| 473 | if (maybe_ready_fiber == null) maybe_ready_fiber = fiber else if (maybe_ready_queue) |*ready_queue| { | ||
| 474 | ready_queue.tail.queue_next = fiber; | ||
| 475 | ready_queue.tail = fiber; | ||
| 476 | } else maybe_ready_queue = .{ .head = fiber, .tail = fiber }; | ||
| 477 | }, | ||
| 478 | }, | ||
| 479 | }; | ||
| 480 | if (maybe_ready_queue) |ready_queue| el.schedule(thread, ready_queue); | ||
| 481 | } | ||
| 482 | } | ||
| 483 | |||
| 484 | const SwitchMessage = struct { | ||
| 485 | contexts: extern struct { | ||
| 486 | prev: *Context, | ||
| 487 | ready: *Context, | ||
| 488 | }, | ||
| 489 | pending_task: PendingTask, | ||
| 490 | |||
| 491 | const PendingTask = union(enum) { | ||
| 492 | nothing, | ||
| 493 | reschedule, | ||
| 494 | recycle, | ||
| 495 | register_awaiter: *?*Fiber, | ||
| 496 | register_select: []const *Io.AnyFuture, | ||
| 497 | mutex_lock: struct { | ||
| 498 | prev_state: Io.Mutex.State, | ||
| 499 | mutex: *Io.Mutex, | ||
| 500 | }, | ||
| 501 | condition_wait: struct { | ||
| 502 | cond: *Io.Condition, | ||
| 503 | mutex: *Io.Mutex, | ||
| 504 | }, | ||
| 505 | exit, | ||
| 506 | }; | ||
| 507 | |||
| 508 | fn handle(message: *const SwitchMessage, el: *EventLoop) void { | ||
| 509 | const thread: *Thread = .current(); | ||
| 510 | thread.current_context = message.contexts.ready; | ||
| 511 | switch (message.pending_task) { | ||
| 512 | .nothing => {}, | ||
| 513 | .reschedule => if (message.contexts.prev != &thread.idle_context) { | ||
| 514 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 515 | assert(prev_fiber.queue_next == null); | ||
| 516 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 517 | }, | ||
| 518 | .recycle => { | ||
| 519 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 520 | assert(prev_fiber.queue_next == null); | ||
| 521 | el.recycle(prev_fiber); | ||
| 522 | }, | ||
| 523 | .register_awaiter => |awaiter| { | ||
| 524 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 525 | assert(prev_fiber.queue_next == null); | ||
| 526 | if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) | ||
| 527 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 528 | }, | ||
| 529 | .register_select => |futures| { | ||
| 530 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 531 | assert(prev_fiber.queue_next == null); | ||
| 532 | for (futures) |any_future| { | ||
| 533 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 534 | if (@atomicRmw(?*Fiber, &future_fiber.awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) { | ||
| 535 | const closure: *AsyncClosure = .fromFiber(future_fiber); | ||
| 536 | if (!@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .seq_cst)) { | ||
| 537 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 538 | } | ||
| 539 | } | ||
| 540 | } | ||
| 541 | }, | ||
| 542 | .mutex_lock => |mutex_lock| { | ||
| 543 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 544 | assert(prev_fiber.queue_next == null); | ||
| 545 | var prev_state = mutex_lock.prev_state; | ||
| 546 | while (switch (prev_state) { | ||
| 547 | else => next_state: { | ||
| 548 | prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state)); | ||
| 549 | break :next_state @cmpxchgWeak( | ||
| 550 | Io.Mutex.State, | ||
| 551 | &mutex_lock.mutex.state, | ||
| 552 | prev_state, | ||
| 553 | @enumFromInt(@intFromPtr(prev_fiber)), | ||
| 554 | .release, | ||
| 555 | .acquire, | ||
| 556 | ); | ||
| 557 | }, | ||
| 558 | .unlocked => @cmpxchgWeak( | ||
| 559 | Io.Mutex.State, | ||
| 560 | &mutex_lock.mutex.state, | ||
| 561 | .unlocked, | ||
| 562 | .locked_once, | ||
| 563 | .acquire, | ||
| 564 | .acquire, | ||
| 565 | ) orelse { | ||
| 566 | prev_fiber.queue_next = null; | ||
| 567 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 568 | return; | ||
| 569 | }, | ||
| 570 | }) |next_state| prev_state = next_state; | ||
| 571 | }, | ||
| 572 | .condition_wait => |condition_wait| { | ||
| 573 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 574 | assert(prev_fiber.queue_next == null); | ||
| 575 | const cond_impl = prev_fiber.resultPointer(ConditionImpl); | ||
| 576 | cond_impl.* = .{ | ||
| 577 | .tail = prev_fiber, | ||
| 578 | .event = .queued, | ||
| 579 | }; | ||
| 580 | if (@cmpxchgStrong( | ||
| 581 | ?*Fiber, | ||
| 582 | @as(*?*Fiber, @ptrCast(&condition_wait.cond.state)), | ||
| 583 | null, | ||
| 584 | prev_fiber, | ||
| 585 | .release, | ||
| 586 | .acquire, | ||
| 587 | )) |waiting_fiber| { | ||
| 588 | const waiting_cond_impl = waiting_fiber.?.resultPointer(ConditionImpl); | ||
| 589 | assert(waiting_cond_impl.tail.queue_next == null); | ||
| 590 | waiting_cond_impl.tail.queue_next = prev_fiber; | ||
| 591 | waiting_cond_impl.tail = prev_fiber; | ||
| 592 | } | ||
| 593 | condition_wait.mutex.unlock(el.io()); | ||
| 594 | }, | ||
| 595 | .exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) |*each_thread| { | ||
| 596 | getSqe(&thread.io_uring).* = .{ | ||
| 597 | .opcode = .MSG_RING, | ||
| 598 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 599 | .ioprio = 0, | ||
| 600 | .fd = each_thread.io_uring.fd, | ||
| 601 | .off = @intFromEnum(Completion.UserData.exit), | ||
| 602 | .addr = 0, | ||
| 603 | .len = 0, | ||
| 604 | .rw_flags = 0, | ||
| 605 | .user_data = @intFromEnum(Completion.UserData.cleanup), | ||
| 606 | .buf_index = 0, | ||
| 607 | .personality = 0, | ||
| 608 | .splice_fd_in = 0, | ||
| 609 | .addr3 = 0, | ||
| 610 | .resv = 0, | ||
| 611 | }; | ||
| 612 | }, | ||
| 613 | } | ||
| 614 | } | ||
| 615 | }; | ||
| 616 | |||
| 617 | const Context = switch (builtin.cpu.arch) { | ||
| 618 | .aarch64 => extern struct { | ||
| 619 | sp: u64, | ||
| 620 | fp: u64, | ||
| 621 | pc: u64, | ||
| 622 | }, | ||
| 623 | .x86_64 => extern struct { | ||
| 624 | rsp: u64, | ||
| 625 | rbp: u64, | ||
| 626 | rip: u64, | ||
| 627 | }, | ||
| 628 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 629 | }; | ||
| 630 | |||
| 631 | inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage { | ||
| 632 | return @fieldParentPtr("contexts", switch (builtin.cpu.arch) { | ||
| 633 | .aarch64 => asm volatile ( | ||
| 634 | \\ ldp x0, x2, [x1] | ||
| 635 | \\ ldr x3, [x2, #16] | ||
| 636 | \\ mov x4, sp | ||
| 637 | \\ stp x4, fp, [x0] | ||
| 638 | \\ adr x5, 0f | ||
| 639 | \\ ldp x4, fp, [x2] | ||
| 640 | \\ str x5, [x0, #16] | ||
| 641 | \\ mov sp, x4 | ||
| 642 | \\ br x3 | ||
| 643 | \\0: | ||
| 644 | : [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")), | ||
| 645 | : [message_to_send] "{x1}" (&message.contexts), | ||
| 646 | : .{ | ||
| 647 | .x0 = true, | ||
| 648 | .x1 = true, | ||
| 649 | .x2 = true, | ||
| 650 | .x3 = true, | ||
| 651 | .x4 = true, | ||
| 652 | .x5 = true, | ||
| 653 | .x6 = true, | ||
| 654 | .x7 = true, | ||
| 655 | .x8 = true, | ||
| 656 | .x9 = true, | ||
| 657 | .x10 = true, | ||
| 658 | .x11 = true, | ||
| 659 | .x12 = true, | ||
| 660 | .x13 = true, | ||
| 661 | .x14 = true, | ||
| 662 | .x15 = true, | ||
| 663 | .x16 = true, | ||
| 664 | .x17 = true, | ||
| 665 | .x18 = true, | ||
| 666 | .x19 = true, | ||
| 667 | .x20 = true, | ||
| 668 | .x21 = true, | ||
| 669 | .x22 = true, | ||
| 670 | .x23 = true, | ||
| 671 | .x24 = true, | ||
| 672 | .x25 = true, | ||
| 673 | .x26 = true, | ||
| 674 | .x27 = true, | ||
| 675 | .x28 = true, | ||
| 676 | .x30 = true, | ||
| 677 | .z0 = true, | ||
| 678 | .z1 = true, | ||
| 679 | .z2 = true, | ||
| 680 | .z3 = true, | ||
| 681 | .z4 = true, | ||
| 682 | .z5 = true, | ||
| 683 | .z6 = true, | ||
| 684 | .z7 = true, | ||
| 685 | .z8 = true, | ||
| 686 | .z9 = true, | ||
| 687 | .z10 = true, | ||
| 688 | .z11 = true, | ||
| 689 | .z12 = true, | ||
| 690 | .z13 = true, | ||
| 691 | .z14 = true, | ||
| 692 | .z15 = true, | ||
| 693 | .z16 = true, | ||
| 694 | .z17 = true, | ||
| 695 | .z18 = true, | ||
| 696 | .z19 = true, | ||
| 697 | .z20 = true, | ||
| 698 | .z21 = true, | ||
| 699 | .z22 = true, | ||
| 700 | .z23 = true, | ||
| 701 | .z24 = true, | ||
| 702 | .z25 = true, | ||
| 703 | .z26 = true, | ||
| 704 | .z27 = true, | ||
| 705 | .z28 = true, | ||
| 706 | .z29 = true, | ||
| 707 | .z30 = true, | ||
| 708 | .z31 = true, | ||
| 709 | .p0 = true, | ||
| 710 | .p1 = true, | ||
| 711 | .p2 = true, | ||
| 712 | .p3 = true, | ||
| 713 | .p4 = true, | ||
| 714 | .p5 = true, | ||
| 715 | .p6 = true, | ||
| 716 | .p7 = true, | ||
| 717 | .p8 = true, | ||
| 718 | .p9 = true, | ||
| 719 | .p10 = true, | ||
| 720 | .p11 = true, | ||
| 721 | .p12 = true, | ||
| 722 | .p13 = true, | ||
| 723 | .p14 = true, | ||
| 724 | .p15 = true, | ||
| 725 | .fpcr = true, | ||
| 726 | .fpsr = true, | ||
| 727 | .ffr = true, | ||
| 728 | .memory = true, | ||
| 729 | }), | ||
| 730 | .x86_64 => asm volatile ( | ||
| 731 | \\ movq 0(%%rsi), %%rax | ||
| 732 | \\ movq 8(%%rsi), %%rcx | ||
| 733 | \\ leaq 0f(%%rip), %%rdx | ||
| 734 | \\ movq %%rsp, 0(%%rax) | ||
| 735 | \\ movq %%rbp, 8(%%rax) | ||
| 736 | \\ movq %%rdx, 16(%%rax) | ||
| 737 | \\ movq 0(%%rcx), %%rsp | ||
| 738 | \\ movq 8(%%rcx), %%rbp | ||
| 739 | \\ jmpq *16(%%rcx) | ||
| 740 | \\0: | ||
| 741 | : [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")), | ||
| 742 | : [message_to_send] "{rsi}" (&message.contexts), | ||
| 743 | : .{ | ||
| 744 | .rax = true, | ||
| 745 | .rcx = true, | ||
| 746 | .rdx = true, | ||
| 747 | .rbx = true, | ||
| 748 | .rsi = true, | ||
| 749 | .rdi = true, | ||
| 750 | .r8 = true, | ||
| 751 | .r9 = true, | ||
| 752 | .r10 = true, | ||
| 753 | .r11 = true, | ||
| 754 | .r12 = true, | ||
| 755 | .r13 = true, | ||
| 756 | .r14 = true, | ||
| 757 | .r15 = true, | ||
| 758 | .mm0 = true, | ||
| 759 | .mm1 = true, | ||
| 760 | .mm2 = true, | ||
| 761 | .mm3 = true, | ||
| 762 | .mm4 = true, | ||
| 763 | .mm5 = true, | ||
| 764 | .mm6 = true, | ||
| 765 | .mm7 = true, | ||
| 766 | .zmm0 = true, | ||
| 767 | .zmm1 = true, | ||
| 768 | .zmm2 = true, | ||
| 769 | .zmm3 = true, | ||
| 770 | .zmm4 = true, | ||
| 771 | .zmm5 = true, | ||
| 772 | .zmm6 = true, | ||
| 773 | .zmm7 = true, | ||
| 774 | .zmm8 = true, | ||
| 775 | .zmm9 = true, | ||
| 776 | .zmm10 = true, | ||
| 777 | .zmm11 = true, | ||
| 778 | .zmm12 = true, | ||
| 779 | .zmm13 = true, | ||
| 780 | .zmm14 = true, | ||
| 781 | .zmm15 = true, | ||
| 782 | .zmm16 = true, | ||
| 783 | .zmm17 = true, | ||
| 784 | .zmm18 = true, | ||
| 785 | .zmm19 = true, | ||
| 786 | .zmm20 = true, | ||
| 787 | .zmm21 = true, | ||
| 788 | .zmm22 = true, | ||
| 789 | .zmm23 = true, | ||
| 790 | .zmm24 = true, | ||
| 791 | .zmm25 = true, | ||
| 792 | .zmm26 = true, | ||
| 793 | .zmm27 = true, | ||
| 794 | .zmm28 = true, | ||
| 795 | .zmm29 = true, | ||
| 796 | .zmm30 = true, | ||
| 797 | .zmm31 = true, | ||
| 798 | .fpsr = true, | ||
| 799 | .fpcr = true, | ||
| 800 | .mxcsr = true, | ||
| 801 | .rflags = true, | ||
| 802 | .dirflag = true, | ||
| 803 | .memory = true, | ||
| 804 | }), | ||
| 805 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 806 | }); | ||
| 807 | } | ||
| 808 | |||
| 809 | fn mainIdleEntry() callconv(.naked) void { | ||
| 810 | switch (builtin.cpu.arch) { | ||
| 811 | .x86_64 => asm volatile ( | ||
| 812 | \\ movq (%%rsp), %%rdi | ||
| 813 | \\ jmp %[mainIdle:P] | ||
| 814 | : | ||
| 815 | : [mainIdle] "X" (&mainIdle), | ||
| 816 | ), | ||
| 817 | .aarch64 => asm volatile ( | ||
| 818 | \\ ldr x0, [sp, #-8] | ||
| 819 | \\ b %[mainIdle] | ||
| 820 | : | ||
| 821 | : [mainIdle] "X" (&mainIdle), | ||
| 822 | ), | ||
| 823 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 824 | } | ||
| 825 | } | ||
| 826 | |||
| 827 | fn fiberEntry() callconv(.naked) void { | ||
| 828 | switch (builtin.cpu.arch) { | ||
| 829 | .x86_64 => asm volatile ( | ||
| 830 | \\ leaq 8(%%rsp), %%rdi | ||
| 831 | \\ jmpq *(%%rsp) | ||
| 832 | ), | ||
| 833 | .aarch64 => asm volatile ( | ||
| 834 | \\ mov x0, sp | ||
| 835 | \\ ldr x2, [sp, #-8] | ||
| 836 | \\ br x2 | ||
| 837 | ), | ||
| 838 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 839 | } | ||
| 840 | } | ||
| 841 | |||
| 842 | const AsyncClosure = struct { | ||
| 843 | event_loop: *EventLoop, | ||
| 844 | fiber: *Fiber, | ||
| 845 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 846 | result_align: Alignment, | ||
| 847 | already_awaited: bool, | ||
| 848 | |||
| 849 | fn contextPointer(closure: *AsyncClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 { | ||
| 850 | return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure)); | ||
| 851 | } | ||
| 852 | |||
| 853 | fn call(closure: *AsyncClosure, message: *const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn { | ||
| 854 | message.handle(closure.event_loop); | ||
| 855 | const fiber = closure.fiber; | ||
| 856 | std.log.debug("{*} performing async", .{fiber}); | ||
| 857 | closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align)); | ||
| 858 | const awaiter = @atomicRmw(?*Fiber, &fiber.awaiter, .Xchg, Fiber.finished, .acq_rel); | ||
| 859 | const ready_awaiter = r: { | ||
| 860 | const a = awaiter orelse break :r null; | ||
| 861 | if (@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .acq_rel)) break :r null; | ||
| 862 | break :r a; | ||
| 863 | }; | ||
| 864 | closure.event_loop.yield(ready_awaiter, .nothing); | ||
| 865 | unreachable; // switched to dead fiber | ||
| 866 | } | ||
| 867 | |||
| 868 | fn fromFiber(fiber: *Fiber) *AsyncClosure { | ||
| 869 | return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward( | ||
| 870 | @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size, | ||
| 871 | ) - @sizeOf(AsyncClosure)); | ||
| 872 | } | ||
| 873 | }; | ||
| 874 | |||
| 875 | fn async( | ||
| 876 | userdata: ?*anyopaque, | ||
| 877 | result: []u8, | ||
| 878 | result_alignment: Alignment, | ||
| 879 | context: []const u8, | ||
| 880 | context_alignment: Alignment, | ||
| 881 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 882 | ) ?*std.Io.AnyFuture { | ||
| 883 | return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { | ||
| 884 | start(context.ptr, result.ptr); | ||
| 885 | return null; | ||
| 886 | }; | ||
| 887 | } | ||
| 888 | |||
| 889 | fn concurrent( | ||
| 890 | userdata: ?*anyopaque, | ||
| 891 | result_len: usize, | ||
| 892 | result_alignment: Alignment, | ||
| 893 | context: []const u8, | ||
| 894 | context_alignment: Alignment, | ||
| 895 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 896 | ) error{OutOfMemory}!*std.Io.AnyFuture { | ||
| 897 | assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO | ||
| 898 | assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO | ||
| 899 | assert(result_len <= Fiber.max_result_size); // TODO | ||
| 900 | assert(context.len <= Fiber.max_context_size); // TODO | ||
| 901 | |||
| 902 | const event_loop: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 903 | const fiber = try Fiber.allocate(event_loop); | ||
| 904 | std.log.debug("allocated {*}", .{fiber}); | ||
| 905 | |||
| 906 | const closure: *AsyncClosure = .fromFiber(fiber); | ||
| 907 | const stack_end: [*]align(16) usize = @ptrCast(@alignCast(closure)); | ||
| 908 | (stack_end - 1)[0..1].* = .{@intFromPtr(&AsyncClosure.call)}; | ||
| 909 | fiber.* = .{ | ||
| 910 | .required_align = {}, | ||
| 911 | .context = switch (builtin.cpu.arch) { | ||
| 912 | .x86_64 => .{ | ||
| 913 | .rsp = @intFromPtr(stack_end - 1), | ||
| 914 | .rbp = 0, | ||
| 915 | .rip = @intFromPtr(&fiberEntry), | ||
| 916 | }, | ||
| 917 | .aarch64 => .{ | ||
| 918 | .sp = @intFromPtr(stack_end), | ||
| 919 | .fp = 0, | ||
| 920 | .pc = @intFromPtr(&fiberEntry), | ||
| 921 | }, | ||
| 922 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 923 | }, | ||
| 924 | .awaiter = null, | ||
| 925 | .queue_next = null, | ||
| 926 | .cancel_thread = null, | ||
| 927 | .awaiting_completions = .initEmpty(), | ||
| 928 | }; | ||
| 929 | closure.* = .{ | ||
| 930 | .event_loop = event_loop, | ||
| 931 | .fiber = fiber, | ||
| 932 | .start = start, | ||
| 933 | .result_align = result_alignment, | ||
| 934 | .already_awaited = false, | ||
| 935 | }; | ||
| 936 | @memcpy(closure.contextPointer(), context); | ||
| 937 | |||
| 938 | event_loop.schedule(.current(), .{ .head = fiber, .tail = fiber }); | ||
| 939 | return @ptrCast(fiber); | ||
| 940 | } | ||
| 941 | |||
| 942 | const DetachedClosure = struct { | ||
| 943 | event_loop: *EventLoop, | ||
| 944 | fiber: *Fiber, | ||
| 945 | start: *const fn (context: *const anyopaque) void, | ||
| 946 | detached_queue_node: std.DoublyLinkedList.Node, | ||
| 947 | |||
| 948 | fn contextPointer(closure: *DetachedClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 { | ||
| 949 | return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(DetachedClosure)); | ||
| 950 | } | ||
| 951 | |||
| 952 | fn call(closure: *DetachedClosure, message: *const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(DetachedClosure))) noreturn { | ||
| 953 | message.handle(closure.event_loop); | ||
| 954 | std.log.debug("{*} performing async detached", .{closure.fiber}); | ||
| 955 | closure.start(closure.contextPointer()); | ||
| 956 | const awaiter = @atomicRmw(?*Fiber, &closure.fiber.awaiter, .Xchg, Fiber.finished, .acq_rel); | ||
| 957 | closure.event_loop.yield(awaiter, pending_task: { | ||
| 958 | closure.event_loop.detached.mutex.lock(closure.event_loop.io()) catch |err| switch (err) { | ||
| 959 | error.Canceled => break :pending_task .nothing, | ||
| 960 | }; | ||
| 961 | defer closure.event_loop.detached.mutex.unlock(closure.event_loop.io()); | ||
| 962 | if (closure.detached_queue_node.next == &closure.detached_queue_node) break :pending_task .nothing; | ||
| 963 | closure.event_loop.detached.list.remove(&closure.detached_queue_node); | ||
| 964 | break :pending_task .recycle; | ||
| 965 | }); | ||
| 966 | unreachable; // switched to dead fiber | ||
| 967 | } | ||
| 968 | }; | ||
| 969 | |||
| 970 | fn asyncDetached( | ||
| 971 | userdata: ?*anyopaque, | ||
| 972 | context: []const u8, | ||
| 973 | context_alignment: std.mem.Alignment, | ||
| 974 | start: *const fn (context: *const anyopaque) void, | ||
| 975 | ) void { | ||
| 976 | assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO | ||
| 977 | assert(context.len <= Fiber.max_context_size); // TODO | ||
| 978 | |||
| 979 | const event_loop: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 980 | const fiber = Fiber.allocate(event_loop) catch { | ||
| 981 | start(context.ptr); | ||
| 982 | return; | ||
| 983 | }; | ||
| 984 | std.log.debug("allocated {*}", .{fiber}); | ||
| 985 | |||
| 986 | const current_thread: *Thread = .current(); | ||
| 987 | const closure: *DetachedClosure = @ptrFromInt(Fiber.max_context_align.max(.of(DetachedClosure)).backward( | ||
| 988 | @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size, | ||
| 989 | ) - @sizeOf(DetachedClosure)); | ||
| 990 | const stack_end: [*]align(16) usize = @ptrCast(@alignCast(closure)); | ||
| 991 | (stack_end - 1)[0..1].* = .{@intFromPtr(&DetachedClosure.call)}; | ||
| 992 | fiber.* = .{ | ||
| 993 | .required_align = {}, | ||
| 994 | .context = switch (builtin.cpu.arch) { | ||
| 995 | .x86_64 => .{ | ||
| 996 | .rsp = @intFromPtr(stack_end - 1), | ||
| 997 | .rbp = 0, | ||
| 998 | .rip = @intFromPtr(&fiberEntry), | ||
| 999 | }, | ||
| 1000 | .aarch64 => .{ | ||
| 1001 | .sp = @intFromPtr(stack_end), | ||
| 1002 | .fp = 0, | ||
| 1003 | .pc = @intFromPtr(&fiberEntry), | ||
| 1004 | }, | ||
| 1005 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 1006 | }, | ||
| 1007 | .awaiter = null, | ||
| 1008 | .queue_next = null, | ||
| 1009 | .cancel_thread = null, | ||
| 1010 | .awaiting_completions = .initEmpty(), | ||
| 1011 | }; | ||
| 1012 | closure.* = .{ | ||
| 1013 | .event_loop = event_loop, | ||
| 1014 | .fiber = fiber, | ||
| 1015 | .start = start, | ||
| 1016 | .detached_queue_node = .{}, | ||
| 1017 | }; | ||
| 1018 | { | ||
| 1019 | event_loop.detached.mutex.lock(event_loop.io()) catch |err| switch (err) { | ||
| 1020 | error.Canceled => { | ||
| 1021 | event_loop.recycle(fiber); | ||
| 1022 | start(context.ptr); | ||
| 1023 | return; | ||
| 1024 | }, | ||
| 1025 | }; | ||
| 1026 | defer event_loop.detached.mutex.unlock(event_loop.io()); | ||
| 1027 | event_loop.detached.list.append(&closure.detached_queue_node); | ||
| 1028 | } | ||
| 1029 | @memcpy(closure.contextPointer(), context); | ||
| 1030 | |||
| 1031 | event_loop.schedule(current_thread, .{ .head = fiber, .tail = fiber }); | ||
| 1032 | } | ||
| 1033 | |||
| 1034 | fn await( | ||
| 1035 | userdata: ?*anyopaque, | ||
| 1036 | any_future: *std.Io.AnyFuture, | ||
| 1037 | result: []u8, | ||
| 1038 | result_alignment: Alignment, | ||
| 1039 | ) void { | ||
| 1040 | const event_loop: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1041 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1042 | if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished) | ||
| 1043 | event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter }); | ||
| 1044 | @memcpy(result, future_fiber.resultBytes(result_alignment)); | ||
| 1045 | event_loop.recycle(future_fiber); | ||
| 1046 | } | ||
| 1047 | |||
| 1048 | fn select(userdata: ?*anyopaque, futures: []const *Io.AnyFuture) usize { | ||
| 1049 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1050 | |||
| 1051 | // Optimization to avoid the yield below. | ||
| 1052 | for (futures, 0..) |any_future, i| { | ||
| 1053 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1054 | if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) == Fiber.finished) | ||
| 1055 | return i; | ||
| 1056 | } | ||
| 1057 | |||
| 1058 | el.yield(null, .{ .register_select = futures }); | ||
| 1059 | |||
| 1060 | std.log.debug("back from select yield", .{}); | ||
| 1061 | |||
| 1062 | const my_thread: *Thread = .current(); | ||
| 1063 | const my_fiber = my_thread.currentFiber(); | ||
| 1064 | var result: ?usize = null; | ||
| 1065 | |||
| 1066 | for (futures, 0..) |any_future, i| { | ||
| 1067 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1068 | if (@cmpxchgStrong(?*Fiber, &future_fiber.awaiter, my_fiber, null, .seq_cst, .seq_cst)) |awaiter| { | ||
| 1069 | if (awaiter == Fiber.finished) { | ||
| 1070 | if (result == null) result = i; | ||
| 1071 | } else if (awaiter) |a| { | ||
| 1072 | const closure: *AsyncClosure = .fromFiber(a); | ||
| 1073 | closure.already_awaited = false; | ||
| 1074 | } | ||
| 1075 | } else { | ||
| 1076 | const closure: *AsyncClosure = .fromFiber(my_fiber); | ||
| 1077 | closure.already_awaited = false; | ||
| 1078 | } | ||
| 1079 | } | ||
| 1080 | |||
| 1081 | return result.?; | ||
| 1082 | } | ||
| 1083 | |||
| 1084 | fn cancel( | ||
| 1085 | userdata: ?*anyopaque, | ||
| 1086 | any_future: *std.Io.AnyFuture, | ||
| 1087 | result: []u8, | ||
| 1088 | result_alignment: Alignment, | ||
| 1089 | ) void { | ||
| 1090 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1091 | if (@atomicRmw( | ||
| 1092 | ?*Thread, | ||
| 1093 | &future_fiber.cancel_thread, | ||
| 1094 | .Xchg, | ||
| 1095 | Thread.canceling, | ||
| 1096 | .acq_rel, | ||
| 1097 | )) |cancel_thread| if (cancel_thread != Thread.canceling) { | ||
| 1098 | getSqe(&Thread.current().io_uring).* = .{ | ||
| 1099 | .opcode = .MSG_RING, | ||
| 1100 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 1101 | .ioprio = 0, | ||
| 1102 | .fd = cancel_thread.io_uring.fd, | ||
| 1103 | .off = @intFromPtr(future_fiber), | ||
| 1104 | .addr = 0, | ||
| 1105 | .len = @bitCast(-@as(i32, @intFromEnum(std.os.linux.E.INTR))), | ||
| 1106 | .rw_flags = 0, | ||
| 1107 | .user_data = @intFromEnum(Completion.UserData.cleanup), | ||
| 1108 | .buf_index = 0, | ||
| 1109 | .personality = 0, | ||
| 1110 | .splice_fd_in = 0, | ||
| 1111 | .addr3 = 0, | ||
| 1112 | .resv = 0, | ||
| 1113 | }; | ||
| 1114 | }; | ||
| 1115 | await(userdata, any_future, result, result_alignment); | ||
| 1116 | } | ||
| 1117 | |||
| 1118 | fn cancelRequested(userdata: ?*anyopaque) bool { | ||
| 1119 | _ = userdata; | ||
| 1120 | return @atomicLoad(?*Thread, &Thread.current().currentFiber().cancel_thread, .acquire) == Thread.canceling; | ||
| 1121 | } | ||
| 1122 | |||
| 1123 | fn createFile( | ||
| 1124 | userdata: ?*anyopaque, | ||
| 1125 | dir: Io.Dir, | ||
| 1126 | sub_path: []const u8, | ||
| 1127 | flags: Io.File.CreateFlags, | ||
| 1128 | ) Io.File.OpenError!Io.File { | ||
| 1129 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1130 | const thread: *Thread = .current(); | ||
| 1131 | const iou = &thread.io_uring; | ||
| 1132 | const fiber = thread.currentFiber(); | ||
| 1133 | try fiber.enterCancelRegion(thread); | ||
| 1134 | |||
| 1135 | const posix = std.posix; | ||
| 1136 | const sub_path_c = try posix.toPosixPath(sub_path); | ||
| 1137 | |||
| 1138 | var os_flags: posix.O = .{ | ||
| 1139 | .ACCMODE = if (flags.read) .RDWR else .WRONLY, | ||
| 1140 | .CREAT = true, | ||
| 1141 | .TRUNC = flags.truncate, | ||
| 1142 | .EXCL = flags.exclusive, | ||
| 1143 | }; | ||
| 1144 | if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true; | ||
| 1145 | if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true; | ||
| 1146 | |||
| 1147 | // Use the O locking flags if the os supports them to acquire the lock | ||
| 1148 | // atomically. Note that the NONBLOCK flag is removed after the openat() | ||
| 1149 | // call is successful. | ||
| 1150 | const has_flock_open_flags = @hasField(posix.O, "EXLOCK"); | ||
| 1151 | if (has_flock_open_flags) switch (flags.lock) { | ||
| 1152 | .none => {}, | ||
| 1153 | .shared => { | ||
| 1154 | os_flags.SHLOCK = true; | ||
| 1155 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1156 | }, | ||
| 1157 | .exclusive => { | ||
| 1158 | os_flags.EXLOCK = true; | ||
| 1159 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1160 | }, | ||
| 1161 | }; | ||
| 1162 | const have_flock = @TypeOf(posix.system.flock) != void; | ||
| 1163 | |||
| 1164 | if (have_flock and !has_flock_open_flags and flags.lock != .none) { | ||
| 1165 | @panic("TODO"); | ||
| 1166 | } | ||
| 1167 | |||
| 1168 | if (has_flock_open_flags and flags.lock_nonblocking) { | ||
| 1169 | @panic("TODO"); | ||
| 1170 | } | ||
| 1171 | |||
| 1172 | getSqe(iou).* = .{ | ||
| 1173 | .opcode = .OPENAT, | ||
| 1174 | .flags = 0, | ||
| 1175 | .ioprio = 0, | ||
| 1176 | .fd = dir.handle, | ||
| 1177 | .off = 0, | ||
| 1178 | .addr = @intFromPtr(&sub_path_c), | ||
| 1179 | .len = @intCast(flags.mode), | ||
| 1180 | .rw_flags = @bitCast(os_flags), | ||
| 1181 | .user_data = @intFromPtr(fiber), | ||
| 1182 | .buf_index = 0, | ||
| 1183 | .personality = 0, | ||
| 1184 | .splice_fd_in = 0, | ||
| 1185 | .addr3 = 0, | ||
| 1186 | .resv = 0, | ||
| 1187 | }; | ||
| 1188 | |||
| 1189 | el.yield(null, .nothing); | ||
| 1190 | fiber.exitCancelRegion(thread); | ||
| 1191 | |||
| 1192 | const completion = fiber.resultPointer(Completion); | ||
| 1193 | switch (errno(completion.result)) { | ||
| 1194 | .SUCCESS => return .{ .handle = completion.result }, | ||
| 1195 | .INTR => unreachable, | ||
| 1196 | .CANCELED => return error.Canceled, | ||
| 1197 | |||
| 1198 | .FAULT => unreachable, | ||
| 1199 | .INVAL => return error.BadPathName, | ||
| 1200 | .BADF => unreachable, | ||
| 1201 | .ACCES => return error.AccessDenied, | ||
| 1202 | .FBIG => return error.FileTooBig, | ||
| 1203 | .OVERFLOW => return error.FileTooBig, | ||
| 1204 | .ISDIR => return error.IsDir, | ||
| 1205 | .LOOP => return error.SymLinkLoop, | ||
| 1206 | .MFILE => return error.ProcessFdQuotaExceeded, | ||
| 1207 | .NAMETOOLONG => return error.NameTooLong, | ||
| 1208 | .NFILE => return error.SystemFdQuotaExceeded, | ||
| 1209 | .NODEV => return error.NoDevice, | ||
| 1210 | .NOENT => return error.FileNotFound, | ||
| 1211 | .NOMEM => return error.SystemResources, | ||
| 1212 | .NOSPC => return error.NoSpaceLeft, | ||
| 1213 | .NOTDIR => return error.NotDir, | ||
| 1214 | .PERM => return error.PermissionDenied, | ||
| 1215 | .EXIST => return error.PathAlreadyExists, | ||
| 1216 | .BUSY => return error.DeviceBusy, | ||
| 1217 | .OPNOTSUPP => return error.FileLocksNotSupported, | ||
| 1218 | .AGAIN => return error.WouldBlock, | ||
| 1219 | .TXTBSY => return error.FileBusy, | ||
| 1220 | .NXIO => return error.NoDevice, | ||
| 1221 | else => |err| return posix.unexpectedErrno(err), | ||
| 1222 | } | ||
| 1223 | } | ||
| 1224 | |||
| 1225 | fn fileOpen( | ||
| 1226 | userdata: ?*anyopaque, | ||
| 1227 | dir: Io.Dir, | ||
| 1228 | sub_path: []const u8, | ||
| 1229 | flags: Io.File.OpenFlags, | ||
| 1230 | ) Io.File.OpenError!Io.File { | ||
| 1231 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1232 | const thread: *Thread = .current(); | ||
| 1233 | const iou = &thread.io_uring; | ||
| 1234 | const fiber = thread.currentFiber(); | ||
| 1235 | try fiber.enterCancelRegion(thread); | ||
| 1236 | |||
| 1237 | const posix = std.posix; | ||
| 1238 | const sub_path_c = try posix.toPosixPath(sub_path); | ||
| 1239 | |||
| 1240 | var os_flags: posix.O = .{ | ||
| 1241 | .ACCMODE = switch (flags.mode) { | ||
| 1242 | .read_only => .RDONLY, | ||
| 1243 | .write_only => .WRONLY, | ||
| 1244 | .read_write => .RDWR, | ||
| 1245 | }, | ||
| 1246 | }; | ||
| 1247 | |||
| 1248 | if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true; | ||
| 1249 | if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true; | ||
| 1250 | if (@hasField(posix.O, "NOCTTY")) os_flags.NOCTTY = !flags.allow_ctty; | ||
| 1251 | |||
| 1252 | // Use the O locking flags if the os supports them to acquire the lock | ||
| 1253 | // atomically. | ||
| 1254 | const has_flock_open_flags = @hasField(posix.O, "EXLOCK"); | ||
| 1255 | if (has_flock_open_flags) { | ||
| 1256 | // Note that the NONBLOCK flag is removed after the openat() call | ||
| 1257 | // is successful. | ||
| 1258 | switch (flags.lock) { | ||
| 1259 | .none => {}, | ||
| 1260 | .shared => { | ||
| 1261 | os_flags.SHLOCK = true; | ||
| 1262 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1263 | }, | ||
| 1264 | .exclusive => { | ||
| 1265 | os_flags.EXLOCK = true; | ||
| 1266 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1267 | }, | ||
| 1268 | } | ||
| 1269 | } | ||
| 1270 | const have_flock = @TypeOf(posix.system.flock) != void; | ||
| 1271 | |||
| 1272 | if (have_flock and !has_flock_open_flags and flags.lock != .none) { | ||
| 1273 | @panic("TODO"); | ||
| 1274 | } | ||
| 1275 | |||
| 1276 | if (has_flock_open_flags and flags.lock_nonblocking) { | ||
| 1277 | @panic("TODO"); | ||
| 1278 | } | ||
| 1279 | |||
| 1280 | getSqe(iou).* = .{ | ||
| 1281 | .opcode = .OPENAT, | ||
| 1282 | .flags = 0, | ||
| 1283 | .ioprio = 0, | ||
| 1284 | .fd = dir.handle, | ||
| 1285 | .off = 0, | ||
| 1286 | .addr = @intFromPtr(&sub_path_c), | ||
| 1287 | .len = 0, | ||
| 1288 | .rw_flags = @bitCast(os_flags), | ||
| 1289 | .user_data = @intFromPtr(fiber), | ||
| 1290 | .buf_index = 0, | ||
| 1291 | .personality = 0, | ||
| 1292 | .splice_fd_in = 0, | ||
| 1293 | .addr3 = 0, | ||
| 1294 | .resv = 0, | ||
| 1295 | }; | ||
| 1296 | |||
| 1297 | el.yield(null, .nothing); | ||
| 1298 | fiber.exitCancelRegion(thread); | ||
| 1299 | |||
| 1300 | const completion = fiber.resultPointer(Completion); | ||
| 1301 | switch (errno(completion.result)) { | ||
| 1302 | .SUCCESS => return .{ .handle = completion.result }, | ||
| 1303 | .INTR => unreachable, | ||
| 1304 | .CANCELED => return error.Canceled, | ||
| 1305 | |||
| 1306 | .FAULT => unreachable, | ||
| 1307 | .INVAL => return error.BadPathName, | ||
| 1308 | .BADF => unreachable, | ||
| 1309 | .ACCES => return error.AccessDenied, | ||
| 1310 | .FBIG => return error.FileTooBig, | ||
| 1311 | .OVERFLOW => return error.FileTooBig, | ||
| 1312 | .ISDIR => return error.IsDir, | ||
| 1313 | .LOOP => return error.SymLinkLoop, | ||
| 1314 | .MFILE => return error.ProcessFdQuotaExceeded, | ||
| 1315 | .NAMETOOLONG => return error.NameTooLong, | ||
| 1316 | .NFILE => return error.SystemFdQuotaExceeded, | ||
| 1317 | .NODEV => return error.NoDevice, | ||
| 1318 | .NOENT => return error.FileNotFound, | ||
| 1319 | .NOMEM => return error.SystemResources, | ||
| 1320 | .NOSPC => return error.NoSpaceLeft, | ||
| 1321 | .NOTDIR => return error.NotDir, | ||
| 1322 | .PERM => return error.PermissionDenied, | ||
| 1323 | .EXIST => return error.PathAlreadyExists, | ||
| 1324 | .BUSY => return error.DeviceBusy, | ||
| 1325 | .OPNOTSUPP => return error.FileLocksNotSupported, | ||
| 1326 | .AGAIN => return error.WouldBlock, | ||
| 1327 | .TXTBSY => return error.FileBusy, | ||
| 1328 | .NXIO => return error.NoDevice, | ||
| 1329 | else => |err| return posix.unexpectedErrno(err), | ||
| 1330 | } | ||
| 1331 | } | ||
| 1332 | |||
| 1333 | fn fileClose(userdata: ?*anyopaque, file: Io.File) void { | ||
| 1334 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1335 | const thread: *Thread = .current(); | ||
| 1336 | const iou = &thread.io_uring; | ||
| 1337 | const fiber = thread.currentFiber(); | ||
| 1338 | |||
| 1339 | getSqe(iou).* = .{ | ||
| 1340 | .opcode = .CLOSE, | ||
| 1341 | .flags = 0, | ||
| 1342 | .ioprio = 0, | ||
| 1343 | .fd = file.handle, | ||
| 1344 | .off = 0, | ||
| 1345 | .addr = 0, | ||
| 1346 | .len = 0, | ||
| 1347 | .rw_flags = 0, | ||
| 1348 | .user_data = @intFromPtr(fiber), | ||
| 1349 | .buf_index = 0, | ||
| 1350 | .personality = 0, | ||
| 1351 | .splice_fd_in = 0, | ||
| 1352 | .addr3 = 0, | ||
| 1353 | .resv = 0, | ||
| 1354 | }; | ||
| 1355 | |||
| 1356 | el.yield(null, .nothing); | ||
| 1357 | |||
| 1358 | const completion = fiber.resultPointer(Completion); | ||
| 1359 | switch (errno(completion.result)) { | ||
| 1360 | .SUCCESS => return, | ||
| 1361 | .INTR => unreachable, | ||
| 1362 | .CANCELED => return, | ||
| 1363 | |||
| 1364 | .BADF => unreachable, // Always a race condition. | ||
| 1365 | else => return, | ||
| 1366 | } | ||
| 1367 | } | ||
| 1368 | |||
| 1369 | fn pread(userdata: ?*anyopaque, file: Io.File, buffer: []u8, offset: std.posix.off_t) Io.File.PReadError!usize { | ||
| 1370 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1371 | const thread: *Thread = .current(); | ||
| 1372 | const iou = &thread.io_uring; | ||
| 1373 | const fiber = thread.currentFiber(); | ||
| 1374 | try fiber.enterCancelRegion(thread); | ||
| 1375 | |||
| 1376 | getSqe(iou).* = .{ | ||
| 1377 | .opcode = .READ, | ||
| 1378 | .flags = 0, | ||
| 1379 | .ioprio = 0, | ||
| 1380 | .fd = file.handle, | ||
| 1381 | .off = @bitCast(offset), | ||
| 1382 | .addr = @intFromPtr(buffer.ptr), | ||
| 1383 | .len = @min(buffer.len, 0x7ffff000), | ||
| 1384 | .rw_flags = 0, | ||
| 1385 | .user_data = @intFromPtr(fiber), | ||
| 1386 | .buf_index = 0, | ||
| 1387 | .personality = 0, | ||
| 1388 | .splice_fd_in = 0, | ||
| 1389 | .addr3 = 0, | ||
| 1390 | .resv = 0, | ||
| 1391 | }; | ||
| 1392 | |||
| 1393 | el.yield(null, .nothing); | ||
| 1394 | fiber.exitCancelRegion(thread); | ||
| 1395 | |||
| 1396 | const completion = fiber.resultPointer(Completion); | ||
| 1397 | switch (errno(completion.result)) { | ||
| 1398 | .SUCCESS => return @as(u32, @bitCast(completion.result)), | ||
| 1399 | .INTR => unreachable, | ||
| 1400 | .CANCELED => return error.Canceled, | ||
| 1401 | |||
| 1402 | .INVAL => unreachable, | ||
| 1403 | .FAULT => unreachable, | ||
| 1404 | .NOENT => return error.ProcessNotFound, | ||
| 1405 | .AGAIN => return error.WouldBlock, | ||
| 1406 | .BADF => return error.NotOpenForReading, // Can be a race condition. | ||
| 1407 | .IO => return error.InputOutput, | ||
| 1408 | .ISDIR => return error.IsDir, | ||
| 1409 | .NOBUFS => return error.SystemResources, | ||
| 1410 | .NOMEM => return error.SystemResources, | ||
| 1411 | .NOTCONN => return error.SocketUnconnected, | ||
| 1412 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 1413 | .TIMEDOUT => return error.Timeout, | ||
| 1414 | .NXIO => return error.Unseekable, | ||
| 1415 | .SPIPE => return error.Unseekable, | ||
| 1416 | .OVERFLOW => return error.Unseekable, | ||
| 1417 | else => |err| return std.posix.unexpectedErrno(err), | ||
| 1418 | } | ||
| 1419 | } | ||
| 1420 | |||
| 1421 | fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: std.posix.off_t) Io.File.PWriteError!usize { | ||
| 1422 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1423 | const thread: *Thread = .current(); | ||
| 1424 | const iou = &thread.io_uring; | ||
| 1425 | const fiber = thread.currentFiber(); | ||
| 1426 | try fiber.enterCancelRegion(thread); | ||
| 1427 | |||
| 1428 | getSqe(iou).* = .{ | ||
| 1429 | .opcode = .WRITE, | ||
| 1430 | .flags = 0, | ||
| 1431 | .ioprio = 0, | ||
| 1432 | .fd = file.handle, | ||
| 1433 | .off = @bitCast(offset), | ||
| 1434 | .addr = @intFromPtr(buffer.ptr), | ||
| 1435 | .len = @min(buffer.len, 0x7ffff000), | ||
| 1436 | .rw_flags = 0, | ||
| 1437 | .user_data = @intFromPtr(fiber), | ||
| 1438 | .buf_index = 0, | ||
| 1439 | .personality = 0, | ||
| 1440 | .splice_fd_in = 0, | ||
| 1441 | .addr3 = 0, | ||
| 1442 | .resv = 0, | ||
| 1443 | }; | ||
| 1444 | |||
| 1445 | el.yield(null, .nothing); | ||
| 1446 | fiber.exitCancelRegion(thread); | ||
| 1447 | |||
| 1448 | const completion = fiber.resultPointer(Completion); | ||
| 1449 | switch (errno(completion.result)) { | ||
| 1450 | .SUCCESS => return @as(u32, @bitCast(completion.result)), | ||
| 1451 | .INTR => unreachable, | ||
| 1452 | .CANCELED => return error.Canceled, | ||
| 1453 | |||
| 1454 | .INVAL => return error.InvalidArgument, | ||
| 1455 | .FAULT => unreachable, | ||
| 1456 | .NOENT => return error.ProcessNotFound, | ||
| 1457 | .AGAIN => return error.WouldBlock, | ||
| 1458 | .BADF => return error.NotOpenForWriting, // can be a race condition. | ||
| 1459 | .DESTADDRREQ => unreachable, // `connect` was never called. | ||
| 1460 | .DQUOT => return error.DiskQuota, | ||
| 1461 | .FBIG => return error.FileTooBig, | ||
| 1462 | .IO => return error.InputOutput, | ||
| 1463 | .NOSPC => return error.NoSpaceLeft, | ||
| 1464 | .ACCES => return error.AccessDenied, | ||
| 1465 | .PERM => return error.PermissionDenied, | ||
| 1466 | .PIPE => return error.BrokenPipe, | ||
| 1467 | .NXIO => return error.Unseekable, | ||
| 1468 | .SPIPE => return error.Unseekable, | ||
| 1469 | .OVERFLOW => return error.Unseekable, | ||
| 1470 | .BUSY => return error.DeviceBusy, | ||
| 1471 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 1472 | .MSGSIZE => return error.MessageTooBig, | ||
| 1473 | else => |err| return std.posix.unexpectedErrno(err), | ||
| 1474 | } | ||
| 1475 | } | ||
| 1476 | |||
| 1477 | fn now(userdata: ?*anyopaque, clockid: std.posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp { | ||
| 1478 | _ = userdata; | ||
| 1479 | const timespec = try std.posix.clock_gettime(clockid); | ||
| 1480 | return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec); | ||
| 1481 | } | ||
| 1482 | |||
| 1483 | fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void { | ||
| 1484 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1485 | const thread: *Thread = .current(); | ||
| 1486 | const iou = &thread.io_uring; | ||
| 1487 | const fiber = thread.currentFiber(); | ||
| 1488 | try fiber.enterCancelRegion(thread); | ||
| 1489 | |||
| 1490 | const deadline_nanoseconds: i96 = switch (deadline) { | ||
| 1491 | .duration => |duration| duration.nanoseconds, | ||
| 1492 | .timestamp => |timestamp| @intFromEnum(timestamp), | ||
| 1493 | }; | ||
| 1494 | const timespec: std.os.linux.kernel_timespec = .{ | ||
| 1495 | .sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 1496 | .nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 1497 | }; | ||
| 1498 | getSqe(iou).* = .{ | ||
| 1499 | .opcode = .TIMEOUT, | ||
| 1500 | .flags = 0, | ||
| 1501 | .ioprio = 0, | ||
| 1502 | .fd = 0, | ||
| 1503 | .off = 0, | ||
| 1504 | .addr = @intFromPtr(×pec), | ||
| 1505 | .len = 1, | ||
| 1506 | .rw_flags = @as(u32, switch (deadline) { | ||
| 1507 | .duration => 0, | ||
| 1508 | .timestamp => std.os.linux.IORING_TIMEOUT_ABS, | ||
| 1509 | }) | @as(u32, switch (clockid) { | ||
| 1510 | .REALTIME => std.os.linux.IORING_TIMEOUT_REALTIME, | ||
| 1511 | .MONOTONIC => 0, | ||
| 1512 | .BOOTTIME => std.os.linux.IORING_TIMEOUT_BOOTTIME, | ||
| 1513 | else => return error.UnsupportedClock, | ||
| 1514 | }), | ||
| 1515 | .user_data = @intFromPtr(fiber), | ||
| 1516 | .buf_index = 0, | ||
| 1517 | .personality = 0, | ||
| 1518 | .splice_fd_in = 0, | ||
| 1519 | .addr3 = 0, | ||
| 1520 | .resv = 0, | ||
| 1521 | }; | ||
| 1522 | |||
| 1523 | el.yield(null, .nothing); | ||
| 1524 | fiber.exitCancelRegion(thread); | ||
| 1525 | |||
| 1526 | const completion = fiber.resultPointer(Completion); | ||
| 1527 | switch (errno(completion.result)) { | ||
| 1528 | .SUCCESS, .TIME => return, | ||
| 1529 | .INTR => unreachable, | ||
| 1530 | .CANCELED => return error.Canceled, | ||
| 1531 | |||
| 1532 | else => |err| return std.posix.unexpectedErrno(err), | ||
| 1533 | } | ||
| 1534 | } | ||
| 1535 | |||
| 1536 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | ||
| 1537 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1538 | el.yield(null, .{ .mutex_lock = .{ .prev_state = prev_state, .mutex = mutex } }); | ||
| 1539 | } | ||
| 1540 | fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) void { | ||
| 1541 | var maybe_waiting_fiber: ?*Fiber = @ptrFromInt(@intFromEnum(prev_state)); | ||
| 1542 | while (if (maybe_waiting_fiber) |waiting_fiber| @cmpxchgWeak( | ||
| 1543 | Io.Mutex.State, | ||
| 1544 | &mutex.state, | ||
| 1545 | @enumFromInt(@intFromPtr(waiting_fiber)), | ||
| 1546 | @enumFromInt(@intFromPtr(waiting_fiber.queue_next)), | ||
| 1547 | .release, | ||
| 1548 | .acquire, | ||
| 1549 | ) else @cmpxchgWeak( | ||
| 1550 | Io.Mutex.State, | ||
| 1551 | &mutex.state, | ||
| 1552 | .locked_once, | ||
| 1553 | .unlocked, | ||
| 1554 | .release, | ||
| 1555 | .acquire, | ||
| 1556 | ) orelse return) |next_state| maybe_waiting_fiber = @ptrFromInt(@intFromEnum(next_state)); | ||
| 1557 | maybe_waiting_fiber.?.queue_next = null; | ||
| 1558 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1559 | el.yield(maybe_waiting_fiber.?, .reschedule); | ||
| 1560 | } | ||
| 1561 | |||
| 1562 | const ConditionImpl = struct { | ||
| 1563 | tail: *Fiber, | ||
| 1564 | event: union(enum) { | ||
| 1565 | queued, | ||
| 1566 | wake: Io.Condition.Wake, | ||
| 1567 | }, | ||
| 1568 | }; | ||
| 1569 | |||
| 1570 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | ||
| 1571 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1572 | el.yield(null, .{ .condition_wait = .{ .cond = cond, .mutex = mutex } }); | ||
| 1573 | const thread = Thread.current(); | ||
| 1574 | const fiber = thread.currentFiber(); | ||
| 1575 | const cond_impl = fiber.resultPointer(ConditionImpl); | ||
| 1576 | try mutex.lock(el.io()); | ||
| 1577 | switch (cond_impl.event) { | ||
| 1578 | .queued => {}, | ||
| 1579 | .wake => |wake| if (fiber.queue_next) |next_fiber| switch (wake) { | ||
| 1580 | .one => if (@cmpxchgStrong( | ||
| 1581 | ?*Fiber, | ||
| 1582 | @as(*?*Fiber, @ptrCast(&cond.state)), | ||
| 1583 | null, | ||
| 1584 | next_fiber, | ||
| 1585 | .release, | ||
| 1586 | .acquire, | ||
| 1587 | )) |old_fiber| { | ||
| 1588 | const old_cond_impl = old_fiber.?.resultPointer(ConditionImpl); | ||
| 1589 | assert(old_cond_impl.tail.queue_next == null); | ||
| 1590 | old_cond_impl.tail.queue_next = next_fiber; | ||
| 1591 | old_cond_impl.tail = cond_impl.tail; | ||
| 1592 | }, | ||
| 1593 | .all => el.schedule(thread, .{ .head = next_fiber, .tail = cond_impl.tail }), | ||
| 1594 | }, | ||
| 1595 | } | ||
| 1596 | fiber.queue_next = null; | ||
| 1597 | } | ||
| 1598 | |||
| 1599 | fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition.Wake) void { | ||
| 1600 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1601 | const waiting_fiber = @atomicRmw(?*Fiber, @as(*?*Fiber, @ptrCast(&cond.state)), .Xchg, null, .acquire) orelse return; | ||
| 1602 | waiting_fiber.resultPointer(ConditionImpl).event = .{ .wake = wake }; | ||
| 1603 | el.yield(waiting_fiber, .reschedule); | ||
| 1604 | } | ||
| 1605 | |||
| 1606 | fn errno(signed: i32) std.os.linux.E { | ||
| 1607 | return .init(@bitCast(@as(isize, signed))); | ||
| 1608 | } | ||
| 1609 | |||
| 1610 | fn getSqe(iou: *IoUring) *std.os.linux.io_uring_sqe { | ||
| 1611 | while (true) return iou.get_sqe() catch { | ||
| 1612 | _ = iou.submit_and_wait(0) catch |err| switch (err) { | ||
| 1613 | error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}), | ||
| 1614 | else => |e| @panic(@errorName(e)), | ||
| 1615 | }; | ||
| 1616 | continue; | ||
| 1617 | }; | ||
| 1618 | } | ||
lib/std/Io/IoUring.zig created+1618
| ... | @@ -0,0 +1,1618 @@ | ||
| 1 | const std = @import("../std.zig"); | ||
| 2 | const builtin = @import("builtin"); | ||
| 3 | const assert = std.debug.assert; | ||
| 4 | const Allocator = std.mem.Allocator; | ||
| 5 | const Io = std.Io; | ||
| 6 | const EventLoop = @This(); | ||
| 7 | const Alignment = std.mem.Alignment; | ||
| 8 | const IoUring = std.os.linux.IoUring; | ||
| 9 | |||
| 10 | /// Must be a thread-safe allocator. | ||
| 11 | gpa: Allocator, | ||
| 12 | main_fiber_buffer: [@sizeOf(Fiber) + Fiber.max_result_size]u8 align(@alignOf(Fiber)), | ||
| 13 | threads: Thread.List, | ||
| 14 | detached: struct { | ||
| 15 | mutex: std.Io.Mutex, | ||
| 16 | list: std.DoublyLinkedList, | ||
| 17 | }, | ||
| 18 | |||
| 19 | /// Empirically saw >128KB being used by the self-hosted backend to panic. | ||
| 20 | const idle_stack_size = 256 * 1024; | ||
| 21 | |||
| 22 | const max_idle_search = 4; | ||
| 23 | const max_steal_ready_search = 4; | ||
| 24 | |||
| 25 | const io_uring_entries = 64; | ||
| 26 | |||
| 27 | const Thread = struct { | ||
| 28 | thread: std.Thread, | ||
| 29 | idle_context: Context, | ||
| 30 | current_context: *Context, | ||
| 31 | ready_queue: ?*Fiber, | ||
| 32 | io_uring: IoUring, | ||
| 33 | idle_search_index: u32, | ||
| 34 | steal_ready_search_index: u32, | ||
| 35 | |||
| 36 | const canceling: ?*Thread = @ptrFromInt(@alignOf(Thread)); | ||
| 37 | |||
| 38 | threadlocal var self: *Thread = undefined; | ||
| 39 | |||
| 40 | fn current() *Thread { | ||
| 41 | return self; | ||
| 42 | } | ||
| 43 | |||
| 44 | fn currentFiber(thread: *Thread) *Fiber { | ||
| 45 | return @fieldParentPtr("context", thread.current_context); | ||
| 46 | } | ||
| 47 | |||
| 48 | const List = struct { | ||
| 49 | allocated: []Thread, | ||
| 50 | reserved: u32, | ||
| 51 | active: u32, | ||
| 52 | }; | ||
| 53 | }; | ||
| 54 | |||
| 55 | const Fiber = struct { | ||
| 56 | required_align: void align(4), | ||
| 57 | context: Context, | ||
| 58 | awaiter: ?*Fiber, | ||
| 59 | queue_next: ?*Fiber, | ||
| 60 | cancel_thread: ?*Thread, | ||
| 61 | awaiting_completions: std.StaticBitSet(3), | ||
| 62 | |||
| 63 | const finished: ?*Fiber = @ptrFromInt(@alignOf(Thread)); | ||
| 64 | |||
| 65 | const max_result_align: Alignment = .@"16"; | ||
| 66 | const max_result_size = max_result_align.forward(64); | ||
| 67 | /// This includes any stack realignments that need to happen, and also the | ||
| 68 | /// initial frame return address slot and argument frame, depending on target. | ||
| 69 | const min_stack_size = 4 * 1024 * 1024; | ||
| 70 | const max_context_align: Alignment = .@"16"; | ||
| 71 | const max_context_size = max_context_align.forward(1024); | ||
| 72 | const max_closure_size: usize = @max(@sizeOf(AsyncClosure), @sizeOf(DetachedClosure)); | ||
| 73 | const max_closure_align: Alignment = .max(.of(AsyncClosure), .of(DetachedClosure)); | ||
| 74 | const allocation_size = std.mem.alignForward( | ||
| 75 | usize, | ||
| 76 | max_closure_align.max(max_context_align).forward( | ||
| 77 | max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size, | ||
| 78 | ) + max_closure_size + max_context_size, | ||
| 79 | std.heap.page_size_max, | ||
| 80 | ); | ||
| 81 | |||
| 82 | fn allocate(el: *EventLoop) error{OutOfMemory}!*Fiber { | ||
| 83 | return @ptrCast(try el.gpa.alignedAlloc(u8, .of(Fiber), allocation_size)); | ||
| 84 | } | ||
| 85 | |||
| 86 | fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 { | ||
| 87 | return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size]; | ||
| 88 | } | ||
| 89 | |||
| 90 | fn allocatedEnd(f: *Fiber) [*]u8 { | ||
| 91 | const allocated_slice = f.allocatedSlice(); | ||
| 92 | return allocated_slice[allocated_slice.len..].ptr; | ||
| 93 | } | ||
| 94 | |||
| 95 | fn resultPointer(f: *Fiber, comptime Result: type) *Result { | ||
| 96 | return @ptrCast(@alignCast(f.resultBytes(.of(Result)))); | ||
| 97 | } | ||
| 98 | |||
| 99 | fn resultBytes(f: *Fiber, alignment: Alignment) [*]u8 { | ||
| 100 | return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber))); | ||
| 101 | } | ||
| 102 | |||
| 103 | fn enterCancelRegion(fiber: *Fiber, thread: *Thread) error{Canceled}!void { | ||
| 104 | if (@cmpxchgStrong( | ||
| 105 | ?*Thread, | ||
| 106 | &fiber.cancel_thread, | ||
| 107 | null, | ||
| 108 | thread, | ||
| 109 | .acq_rel, | ||
| 110 | .acquire, | ||
| 111 | )) |cancel_thread| { | ||
| 112 | assert(cancel_thread == Thread.canceling); | ||
| 113 | return error.Canceled; | ||
| 114 | } | ||
| 115 | } | ||
| 116 | |||
| 117 | fn exitCancelRegion(fiber: *Fiber, thread: *Thread) void { | ||
| 118 | if (@cmpxchgStrong( | ||
| 119 | ?*Thread, | ||
| 120 | &fiber.cancel_thread, | ||
| 121 | thread, | ||
| 122 | null, | ||
| 123 | .acq_rel, | ||
| 124 | .acquire, | ||
| 125 | )) |cancel_thread| assert(cancel_thread == Thread.canceling); | ||
| 126 | } | ||
| 127 | |||
| 128 | const Queue = struct { head: *Fiber, tail: *Fiber }; | ||
| 129 | }; | ||
| 130 | |||
| 131 | fn recycle(el: *EventLoop, fiber: *Fiber) void { | ||
| 132 | std.log.debug("recyling {*}", .{fiber}); | ||
| 133 | assert(fiber.queue_next == null); | ||
| 134 | el.gpa.free(fiber.allocatedSlice()); | ||
| 135 | } | ||
| 136 | |||
| 137 | pub fn io(el: *EventLoop) Io { | ||
| 138 | return .{ | ||
| 139 | .userdata = el, | ||
| 140 | .vtable = &.{ | ||
| 141 | .async = async, | ||
| 142 | .concurrent = concurrent, | ||
| 143 | .await = await, | ||
| 144 | .asyncDetached = asyncDetached, | ||
| 145 | .select = select, | ||
| 146 | .cancel = cancel, | ||
| 147 | .cancelRequested = cancelRequested, | ||
| 148 | |||
| 149 | .mutexLock = mutexLock, | ||
| 150 | .mutexUnlock = mutexUnlock, | ||
| 151 | |||
| 152 | .conditionWait = conditionWait, | ||
| 153 | .conditionWake = conditionWake, | ||
| 154 | |||
| 155 | .createFile = createFile, | ||
| 156 | .fileOpen = fileOpen, | ||
| 157 | .fileClose = fileClose, | ||
| 158 | .pread = pread, | ||
| 159 | .pwrite = pwrite, | ||
| 160 | |||
| 161 | .now = now, | ||
| 162 | .sleep = sleep, | ||
| 163 | }, | ||
| 164 | }; | ||
| 165 | } | ||
| 166 | |||
| 167 | pub fn init(el: *EventLoop, gpa: Allocator) !void { | ||
| 168 | const threads_size = @max(std.Thread.getCpuCount() catch 1, 1) * @sizeOf(Thread); | ||
| 169 | const idle_stack_end_offset = std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max); | ||
| 170 | const allocated_slice = try gpa.alignedAlloc(u8, .of(Thread), idle_stack_end_offset); | ||
| 171 | errdefer gpa.free(allocated_slice); | ||
| 172 | el.* = .{ | ||
| 173 | .gpa = gpa, | ||
| 174 | .main_fiber_buffer = undefined, | ||
| 175 | .threads = .{ | ||
| 176 | .allocated = @ptrCast(allocated_slice[0..threads_size]), | ||
| 177 | .reserved = 1, | ||
| 178 | .active = 1, | ||
| 179 | }, | ||
| 180 | .detached = .{ | ||
| 181 | .mutex = .init, | ||
| 182 | .list = .{}, | ||
| 183 | }, | ||
| 184 | }; | ||
| 185 | const main_fiber: *Fiber = @ptrCast(&el.main_fiber_buffer); | ||
| 186 | main_fiber.* = .{ | ||
| 187 | .required_align = {}, | ||
| 188 | .context = undefined, | ||
| 189 | .awaiter = null, | ||
| 190 | .queue_next = null, | ||
| 191 | .cancel_thread = null, | ||
| 192 | .awaiting_completions = .initEmpty(), | ||
| 193 | }; | ||
| 194 | const main_thread = &el.threads.allocated[0]; | ||
| 195 | Thread.self = main_thread; | ||
| 196 | const idle_stack_end: [*]align(16) usize = @ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr)); | ||
| 197 | (idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)}; | ||
| 198 | main_thread.* = .{ | ||
| 199 | .thread = undefined, | ||
| 200 | .idle_context = switch (builtin.cpu.arch) { | ||
| 201 | .aarch64 => .{ | ||
| 202 | .sp = @intFromPtr(idle_stack_end), | ||
| 203 | .fp = 0, | ||
| 204 | .pc = @intFromPtr(&mainIdleEntry), | ||
| 205 | }, | ||
| 206 | .x86_64 => .{ | ||
| 207 | .rsp = @intFromPtr(idle_stack_end - 1), | ||
| 208 | .rbp = 0, | ||
| 209 | .rip = @intFromPtr(&mainIdleEntry), | ||
| 210 | }, | ||
| 211 | else => @compileError("unimplemented architecture"), | ||
| 212 | }, | ||
| 213 | .current_context = &main_fiber.context, | ||
| 214 | .ready_queue = null, | ||
| 215 | .io_uring = try IoUring.init(io_uring_entries, 0), | ||
| 216 | .idle_search_index = 1, | ||
| 217 | .steal_ready_search_index = 1, | ||
| 218 | }; | ||
| 219 | errdefer main_thread.io_uring.deinit(); | ||
| 220 | std.log.debug("created main idle {*}", .{&main_thread.idle_context}); | ||
| 221 | std.log.debug("created main {*}", .{main_fiber}); | ||
| 222 | } | ||
| 223 | |||
| 224 | pub fn deinit(el: *EventLoop) void { | ||
| 225 | while (true) cancel(el, detached_future: { | ||
| 226 | el.detached.mutex.lock(el.io()) catch |err| switch (err) { | ||
| 227 | error.Canceled => unreachable, // main fiber cannot be canceled | ||
| 228 | }; | ||
| 229 | defer el.detached.mutex.unlock(el.io()); | ||
| 230 | const detached: *DetachedClosure = @fieldParentPtr( | ||
| 231 | "detached_queue_node", | ||
| 232 | el.detached.list.pop() orelse break, | ||
| 233 | ); | ||
| 234 | // notify the detached fiber that it is no longer allowed to recycle itself | ||
| 235 | detached.detached_queue_node = .{ | ||
| 236 | .prev = &detached.detached_queue_node, | ||
| 237 | .next = &detached.detached_queue_node, | ||
| 238 | }; | ||
| 239 | break :detached_future @ptrCast(detached.fiber); | ||
| 240 | }, &.{}, .@"1"); | ||
| 241 | const active_threads = @atomicLoad(u32, &el.threads.active, .acquire); | ||
| 242 | for (el.threads.allocated[0..active_threads]) |*thread| { | ||
| 243 | const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic); | ||
| 244 | assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async | ||
| 245 | } | ||
| 246 | el.yield(null, .exit); | ||
| 247 | const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(el.threads.allocated.ptr)); | ||
| 248 | const idle_stack_end_offset = std.mem.alignForward(usize, el.threads.allocated.len * @sizeOf(Thread) + idle_stack_size, std.heap.page_size_max); | ||
| 249 | for (el.threads.allocated[1..active_threads]) |*thread| thread.thread.join(); | ||
| 250 | el.gpa.free(allocated_ptr[0..idle_stack_end_offset]); | ||
| 251 | el.* = undefined; | ||
| 252 | } | ||
| 253 | |||
| 254 | fn findReadyFiber(el: *EventLoop, thread: *Thread) ?*Fiber { | ||
| 255 | if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) |ready_fiber| { | ||
| 256 | @atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release); | ||
| 257 | ready_fiber.queue_next = null; | ||
| 258 | return ready_fiber; | ||
| 259 | } | ||
| 260 | const active_threads = @atomicLoad(u32, &el.threads.active, .acquire); | ||
| 261 | for (0..@min(max_steal_ready_search, active_threads)) |_| { | ||
| 262 | defer thread.steal_ready_search_index += 1; | ||
| 263 | if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0; | ||
| 264 | const steal_ready_search_thread = &el.threads.allocated[0..active_threads][thread.steal_ready_search_index]; | ||
| 265 | if (steal_ready_search_thread == thread) continue; | ||
| 266 | const ready_fiber = @atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue; | ||
| 267 | if (ready_fiber == Fiber.finished) continue; | ||
| 268 | if (@cmpxchgWeak( | ||
| 269 | ?*Fiber, | ||
| 270 | &steal_ready_search_thread.ready_queue, | ||
| 271 | ready_fiber, | ||
| 272 | null, | ||
| 273 | .acquire, | ||
| 274 | .monotonic, | ||
| 275 | )) |_| continue; | ||
| 276 | @atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release); | ||
| 277 | ready_fiber.queue_next = null; | ||
| 278 | return ready_fiber; | ||
| 279 | } | ||
| 280 | // couldn't find anything to do, so we are now open for business | ||
| 281 | @atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic); | ||
| 282 | return null; | ||
| 283 | } | ||
| 284 | |||
| 285 | fn yield(el: *EventLoop, maybe_ready_fiber: ?*Fiber, pending_task: SwitchMessage.PendingTask) void { | ||
| 286 | const thread: *Thread = .current(); | ||
| 287 | const ready_context = if (maybe_ready_fiber orelse el.findReadyFiber(thread)) |ready_fiber| | ||
| 288 | &ready_fiber.context | ||
| 289 | else | ||
| 290 | &thread.idle_context; | ||
| 291 | const message: SwitchMessage = .{ | ||
| 292 | .contexts = .{ | ||
| 293 | .prev = thread.current_context, | ||
| 294 | .ready = ready_context, | ||
| 295 | }, | ||
| 296 | .pending_task = pending_task, | ||
| 297 | }; | ||
| 298 | std.log.debug("switching from {*} to {*}", .{ message.contexts.prev, message.contexts.ready }); | ||
| 299 | contextSwitch(&message).handle(el); | ||
| 300 | } | ||
| 301 | |||
| 302 | fn schedule(el: *EventLoop, thread: *Thread, ready_queue: Fiber.Queue) void { | ||
| 303 | { | ||
| 304 | var fiber = ready_queue.head; | ||
| 305 | while (true) { | ||
| 306 | std.log.debug("scheduling {*}", .{fiber}); | ||
| 307 | fiber = fiber.queue_next orelse break; | ||
| 308 | } | ||
| 309 | assert(fiber == ready_queue.tail); | ||
| 310 | } | ||
| 311 | // shared fields of previous `Thread` must be initialized before later ones are marked as active | ||
| 312 | const new_thread_index = @atomicLoad(u32, &el.threads.active, .acquire); | ||
| 313 | for (0..@min(max_idle_search, new_thread_index)) |_| { | ||
| 314 | defer thread.idle_search_index += 1; | ||
| 315 | if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0; | ||
| 316 | const idle_search_thread = &el.threads.allocated[0..new_thread_index][thread.idle_search_index]; | ||
| 317 | if (idle_search_thread == thread) continue; | ||
| 318 | if (@cmpxchgWeak( | ||
| 319 | ?*Fiber, | ||
| 320 | &idle_search_thread.ready_queue, | ||
| 321 | null, | ||
| 322 | ready_queue.head, | ||
| 323 | .release, | ||
| 324 | .monotonic, | ||
| 325 | )) |_| continue; | ||
| 326 | getSqe(&thread.io_uring).* = .{ | ||
| 327 | .opcode = .MSG_RING, | ||
| 328 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 329 | .ioprio = 0, | ||
| 330 | .fd = idle_search_thread.io_uring.fd, | ||
| 331 | .off = @intFromEnum(Completion.UserData.wakeup), | ||
| 332 | .addr = 0, | ||
| 333 | .len = 0, | ||
| 334 | .rw_flags = 0, | ||
| 335 | .user_data = @intFromEnum(Completion.UserData.wakeup), | ||
| 336 | .buf_index = 0, | ||
| 337 | .personality = 0, | ||
| 338 | .splice_fd_in = 0, | ||
| 339 | .addr3 = 0, | ||
| 340 | .resv = 0, | ||
| 341 | }; | ||
| 342 | return; | ||
| 343 | } | ||
| 344 | spawn_thread: { | ||
| 345 | // previous failed reservations must have completed before retrying | ||
| 346 | if (new_thread_index == el.threads.allocated.len or @cmpxchgWeak( | ||
| 347 | u32, | ||
| 348 | &el.threads.reserved, | ||
| 349 | new_thread_index, | ||
| 350 | new_thread_index + 1, | ||
| 351 | .acquire, | ||
| 352 | .monotonic, | ||
| 353 | ) != null) break :spawn_thread; | ||
| 354 | const new_thread = &el.threads.allocated[new_thread_index]; | ||
| 355 | const next_thread_index = new_thread_index + 1; | ||
| 356 | new_thread.* = .{ | ||
| 357 | .thread = undefined, | ||
| 358 | .idle_context = undefined, | ||
| 359 | .current_context = &new_thread.idle_context, | ||
| 360 | .ready_queue = ready_queue.head, | ||
| 361 | .io_uring = IoUring.init(io_uring_entries, 0) catch |err| { | ||
| 362 | @atomicStore(u32, &el.threads.reserved, new_thread_index, .release); | ||
| 363 | // no more access to `thread` after giving up reservation | ||
| 364 | std.log.warn("unable to create worker thread due to io_uring init failure: {s}", .{@errorName(err)}); | ||
| 365 | break :spawn_thread; | ||
| 366 | }, | ||
| 367 | .idle_search_index = 0, | ||
| 368 | .steal_ready_search_index = 0, | ||
| 369 | }; | ||
| 370 | new_thread.thread = std.Thread.spawn(.{ | ||
| 371 | .stack_size = idle_stack_size, | ||
| 372 | .allocator = el.gpa, | ||
| 373 | }, threadEntry, .{ el, new_thread_index }) catch |err| { | ||
| 374 | new_thread.io_uring.deinit(); | ||
| 375 | @atomicStore(u32, &el.threads.reserved, new_thread_index, .release); | ||
| 376 | // no more access to `thread` after giving up reservation | ||
| 377 | std.log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)}); | ||
| 378 | break :spawn_thread; | ||
| 379 | }; | ||
| 380 | // shared fields of `Thread` must be initialized before being marked active | ||
| 381 | @atomicStore(u32, &el.threads.active, next_thread_index, .release); | ||
| 382 | return; | ||
| 383 | } | ||
| 384 | // nobody wanted it, so just queue it on ourselves | ||
| 385 | while (@cmpxchgWeak( | ||
| 386 | ?*Fiber, | ||
| 387 | &thread.ready_queue, | ||
| 388 | ready_queue.tail.queue_next, | ||
| 389 | ready_queue.head, | ||
| 390 | .acq_rel, | ||
| 391 | .acquire, | ||
| 392 | )) |old_head| ready_queue.tail.queue_next = old_head; | ||
| 393 | } | ||
| 394 | |||
| 395 | fn mainIdle(el: *EventLoop, message: *const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn { | ||
| 396 | message.handle(el); | ||
| 397 | el.idle(&el.threads.allocated[0]); | ||
| 398 | el.yield(@ptrCast(&el.main_fiber_buffer), .nothing); | ||
| 399 | unreachable; // switched to dead fiber | ||
| 400 | } | ||
| 401 | |||
| 402 | fn threadEntry(el: *EventLoop, index: u32) void { | ||
| 403 | const thread: *Thread = &el.threads.allocated[index]; | ||
| 404 | Thread.self = thread; | ||
| 405 | std.log.debug("created thread idle {*}", .{&thread.idle_context}); | ||
| 406 | el.idle(thread); | ||
| 407 | } | ||
| 408 | |||
| 409 | const Completion = struct { | ||
| 410 | const UserData = enum(usize) { | ||
| 411 | unused, | ||
| 412 | wakeup, | ||
| 413 | cleanup, | ||
| 414 | exit, | ||
| 415 | /// *Fiber | ||
| 416 | _, | ||
| 417 | }; | ||
| 418 | result: i32, | ||
| 419 | flags: u32, | ||
| 420 | }; | ||
| 421 | |||
| 422 | fn idle(el: *EventLoop, thread: *Thread) void { | ||
| 423 | var maybe_ready_fiber: ?*Fiber = null; | ||
| 424 | while (true) { | ||
| 425 | while (maybe_ready_fiber orelse el.findReadyFiber(thread)) |ready_fiber| { | ||
| 426 | el.yield(ready_fiber, .nothing); | ||
| 427 | maybe_ready_fiber = null; | ||
| 428 | } | ||
| 429 | _ = thread.io_uring.submit_and_wait(1) catch |err| switch (err) { | ||
| 430 | error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}), | ||
| 431 | else => |e| @panic(@errorName(e)), | ||
| 432 | }; | ||
| 433 | var cqes_buffer: [io_uring_entries]std.os.linux.io_uring_cqe = undefined; | ||
| 434 | var maybe_ready_queue: ?Fiber.Queue = null; | ||
| 435 | for (cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch |err| switch (err) { | ||
| 436 | error.SignalInterrupt => cqes_len: { | ||
| 437 | std.log.warn("copy_cqes failed with SignalInterrupt", .{}); | ||
| 438 | break :cqes_len 0; | ||
| 439 | }, | ||
| 440 | else => |e| @panic(@errorName(e)), | ||
| 441 | }]) |cqe| switch (@as(Completion.UserData, @enumFromInt(cqe.user_data))) { | ||
| 442 | .unused => unreachable, // bad submission queued? | ||
| 443 | .wakeup => {}, | ||
| 444 | .cleanup => @panic("failed to notify other threads that we are exiting"), | ||
| 445 | .exit => { | ||
| 446 | assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async | ||
| 447 | return; | ||
| 448 | }, | ||
| 449 | _ => switch (errno(cqe.res)) { | ||
| 450 | .INTR => getSqe(&thread.io_uring).* = .{ | ||
| 451 | .opcode = .ASYNC_CANCEL, | ||
| 452 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 453 | .ioprio = 0, | ||
| 454 | .fd = 0, | ||
| 455 | .off = 0, | ||
| 456 | .addr = cqe.user_data, | ||
| 457 | .len = 0, | ||
| 458 | .rw_flags = 0, | ||
| 459 | .user_data = @intFromEnum(Completion.UserData.wakeup), | ||
| 460 | .buf_index = 0, | ||
| 461 | .personality = 0, | ||
| 462 | .splice_fd_in = 0, | ||
| 463 | .addr3 = 0, | ||
| 464 | .resv = 0, | ||
| 465 | }, | ||
| 466 | else => { | ||
| 467 | const fiber: *Fiber = @ptrFromInt(cqe.user_data); | ||
| 468 | assert(fiber.queue_next == null); | ||
| 469 | fiber.resultPointer(Completion).* = .{ | ||
| 470 | .result = cqe.res, | ||
| 471 | .flags = cqe.flags, | ||
| 472 | }; | ||
| 473 | if (maybe_ready_fiber == null) maybe_ready_fiber = fiber else if (maybe_ready_queue) |*ready_queue| { | ||
| 474 | ready_queue.tail.queue_next = fiber; | ||
| 475 | ready_queue.tail = fiber; | ||
| 476 | } else maybe_ready_queue = .{ .head = fiber, .tail = fiber }; | ||
| 477 | }, | ||
| 478 | }, | ||
| 479 | }; | ||
| 480 | if (maybe_ready_queue) |ready_queue| el.schedule(thread, ready_queue); | ||
| 481 | } | ||
| 482 | } | ||
| 483 | |||
| 484 | const SwitchMessage = struct { | ||
| 485 | contexts: extern struct { | ||
| 486 | prev: *Context, | ||
| 487 | ready: *Context, | ||
| 488 | }, | ||
| 489 | pending_task: PendingTask, | ||
| 490 | |||
| 491 | const PendingTask = union(enum) { | ||
| 492 | nothing, | ||
| 493 | reschedule, | ||
| 494 | recycle, | ||
| 495 | register_awaiter: *?*Fiber, | ||
| 496 | register_select: []const *Io.AnyFuture, | ||
| 497 | mutex_lock: struct { | ||
| 498 | prev_state: Io.Mutex.State, | ||
| 499 | mutex: *Io.Mutex, | ||
| 500 | }, | ||
| 501 | condition_wait: struct { | ||
| 502 | cond: *Io.Condition, | ||
| 503 | mutex: *Io.Mutex, | ||
| 504 | }, | ||
| 505 | exit, | ||
| 506 | }; | ||
| 507 | |||
| 508 | fn handle(message: *const SwitchMessage, el: *EventLoop) void { | ||
| 509 | const thread: *Thread = .current(); | ||
| 510 | thread.current_context = message.contexts.ready; | ||
| 511 | switch (message.pending_task) { | ||
| 512 | .nothing => {}, | ||
| 513 | .reschedule => if (message.contexts.prev != &thread.idle_context) { | ||
| 514 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 515 | assert(prev_fiber.queue_next == null); | ||
| 516 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 517 | }, | ||
| 518 | .recycle => { | ||
| 519 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 520 | assert(prev_fiber.queue_next == null); | ||
| 521 | el.recycle(prev_fiber); | ||
| 522 | }, | ||
| 523 | .register_awaiter => |awaiter| { | ||
| 524 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 525 | assert(prev_fiber.queue_next == null); | ||
| 526 | if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) | ||
| 527 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 528 | }, | ||
| 529 | .register_select => |futures| { | ||
| 530 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 531 | assert(prev_fiber.queue_next == null); | ||
| 532 | for (futures) |any_future| { | ||
| 533 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 534 | if (@atomicRmw(?*Fiber, &future_fiber.awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) { | ||
| 535 | const closure: *AsyncClosure = .fromFiber(future_fiber); | ||
| 536 | if (!@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .seq_cst)) { | ||
| 537 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 538 | } | ||
| 539 | } | ||
| 540 | } | ||
| 541 | }, | ||
| 542 | .mutex_lock => |mutex_lock| { | ||
| 543 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 544 | assert(prev_fiber.queue_next == null); | ||
| 545 | var prev_state = mutex_lock.prev_state; | ||
| 546 | while (switch (prev_state) { | ||
| 547 | else => next_state: { | ||
| 548 | prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state)); | ||
| 549 | break :next_state @cmpxchgWeak( | ||
| 550 | Io.Mutex.State, | ||
| 551 | &mutex_lock.mutex.state, | ||
| 552 | prev_state, | ||
| 553 | @enumFromInt(@intFromPtr(prev_fiber)), | ||
| 554 | .release, | ||
| 555 | .acquire, | ||
| 556 | ); | ||
| 557 | }, | ||
| 558 | .unlocked => @cmpxchgWeak( | ||
| 559 | Io.Mutex.State, | ||
| 560 | &mutex_lock.mutex.state, | ||
| 561 | .unlocked, | ||
| 562 | .locked_once, | ||
| 563 | .acquire, | ||
| 564 | .acquire, | ||
| 565 | ) orelse { | ||
| 566 | prev_fiber.queue_next = null; | ||
| 567 | el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber }); | ||
| 568 | return; | ||
| 569 | }, | ||
| 570 | }) |next_state| prev_state = next_state; | ||
| 571 | }, | ||
| 572 | .condition_wait => |condition_wait| { | ||
| 573 | const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev)); | ||
| 574 | assert(prev_fiber.queue_next == null); | ||
| 575 | const cond_impl = prev_fiber.resultPointer(ConditionImpl); | ||
| 576 | cond_impl.* = .{ | ||
| 577 | .tail = prev_fiber, | ||
| 578 | .event = .queued, | ||
| 579 | }; | ||
| 580 | if (@cmpxchgStrong( | ||
| 581 | ?*Fiber, | ||
| 582 | @as(*?*Fiber, @ptrCast(&condition_wait.cond.state)), | ||
| 583 | null, | ||
| 584 | prev_fiber, | ||
| 585 | .release, | ||
| 586 | .acquire, | ||
| 587 | )) |waiting_fiber| { | ||
| 588 | const waiting_cond_impl = waiting_fiber.?.resultPointer(ConditionImpl); | ||
| 589 | assert(waiting_cond_impl.tail.queue_next == null); | ||
| 590 | waiting_cond_impl.tail.queue_next = prev_fiber; | ||
| 591 | waiting_cond_impl.tail = prev_fiber; | ||
| 592 | } | ||
| 593 | condition_wait.mutex.unlock(el.io()); | ||
| 594 | }, | ||
| 595 | .exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) |*each_thread| { | ||
| 596 | getSqe(&thread.io_uring).* = .{ | ||
| 597 | .opcode = .MSG_RING, | ||
| 598 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 599 | .ioprio = 0, | ||
| 600 | .fd = each_thread.io_uring.fd, | ||
| 601 | .off = @intFromEnum(Completion.UserData.exit), | ||
| 602 | .addr = 0, | ||
| 603 | .len = 0, | ||
| 604 | .rw_flags = 0, | ||
| 605 | .user_data = @intFromEnum(Completion.UserData.cleanup), | ||
| 606 | .buf_index = 0, | ||
| 607 | .personality = 0, | ||
| 608 | .splice_fd_in = 0, | ||
| 609 | .addr3 = 0, | ||
| 610 | .resv = 0, | ||
| 611 | }; | ||
| 612 | }, | ||
| 613 | } | ||
| 614 | } | ||
| 615 | }; | ||
| 616 | |||
| 617 | const Context = switch (builtin.cpu.arch) { | ||
| 618 | .aarch64 => extern struct { | ||
| 619 | sp: u64, | ||
| 620 | fp: u64, | ||
| 621 | pc: u64, | ||
| 622 | }, | ||
| 623 | .x86_64 => extern struct { | ||
| 624 | rsp: u64, | ||
| 625 | rbp: u64, | ||
| 626 | rip: u64, | ||
| 627 | }, | ||
| 628 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 629 | }; | ||
| 630 | |||
| 631 | inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage { | ||
| 632 | return @fieldParentPtr("contexts", switch (builtin.cpu.arch) { | ||
| 633 | .aarch64 => asm volatile ( | ||
| 634 | \\ ldp x0, x2, [x1] | ||
| 635 | \\ ldr x3, [x2, #16] | ||
| 636 | \\ mov x4, sp | ||
| 637 | \\ stp x4, fp, [x0] | ||
| 638 | \\ adr x5, 0f | ||
| 639 | \\ ldp x4, fp, [x2] | ||
| 640 | \\ str x5, [x0, #16] | ||
| 641 | \\ mov sp, x4 | ||
| 642 | \\ br x3 | ||
| 643 | \\0: | ||
| 644 | : [received_message] "={x1}" (-> *const @FieldType(SwitchMessage, "contexts")), | ||
| 645 | : [message_to_send] "{x1}" (&message.contexts), | ||
| 646 | : .{ | ||
| 647 | .x0 = true, | ||
| 648 | .x1 = true, | ||
| 649 | .x2 = true, | ||
| 650 | .x3 = true, | ||
| 651 | .x4 = true, | ||
| 652 | .x5 = true, | ||
| 653 | .x6 = true, | ||
| 654 | .x7 = true, | ||
| 655 | .x8 = true, | ||
| 656 | .x9 = true, | ||
| 657 | .x10 = true, | ||
| 658 | .x11 = true, | ||
| 659 | .x12 = true, | ||
| 660 | .x13 = true, | ||
| 661 | .x14 = true, | ||
| 662 | .x15 = true, | ||
| 663 | .x16 = true, | ||
| 664 | .x17 = true, | ||
| 665 | .x18 = true, | ||
| 666 | .x19 = true, | ||
| 667 | .x20 = true, | ||
| 668 | .x21 = true, | ||
| 669 | .x22 = true, | ||
| 670 | .x23 = true, | ||
| 671 | .x24 = true, | ||
| 672 | .x25 = true, | ||
| 673 | .x26 = true, | ||
| 674 | .x27 = true, | ||
| 675 | .x28 = true, | ||
| 676 | .x30 = true, | ||
| 677 | .z0 = true, | ||
| 678 | .z1 = true, | ||
| 679 | .z2 = true, | ||
| 680 | .z3 = true, | ||
| 681 | .z4 = true, | ||
| 682 | .z5 = true, | ||
| 683 | .z6 = true, | ||
| 684 | .z7 = true, | ||
| 685 | .z8 = true, | ||
| 686 | .z9 = true, | ||
| 687 | .z10 = true, | ||
| 688 | .z11 = true, | ||
| 689 | .z12 = true, | ||
| 690 | .z13 = true, | ||
| 691 | .z14 = true, | ||
| 692 | .z15 = true, | ||
| 693 | .z16 = true, | ||
| 694 | .z17 = true, | ||
| 695 | .z18 = true, | ||
| 696 | .z19 = true, | ||
| 697 | .z20 = true, | ||
| 698 | .z21 = true, | ||
| 699 | .z22 = true, | ||
| 700 | .z23 = true, | ||
| 701 | .z24 = true, | ||
| 702 | .z25 = true, | ||
| 703 | .z26 = true, | ||
| 704 | .z27 = true, | ||
| 705 | .z28 = true, | ||
| 706 | .z29 = true, | ||
| 707 | .z30 = true, | ||
| 708 | .z31 = true, | ||
| 709 | .p0 = true, | ||
| 710 | .p1 = true, | ||
| 711 | .p2 = true, | ||
| 712 | .p3 = true, | ||
| 713 | .p4 = true, | ||
| 714 | .p5 = true, | ||
| 715 | .p6 = true, | ||
| 716 | .p7 = true, | ||
| 717 | .p8 = true, | ||
| 718 | .p9 = true, | ||
| 719 | .p10 = true, | ||
| 720 | .p11 = true, | ||
| 721 | .p12 = true, | ||
| 722 | .p13 = true, | ||
| 723 | .p14 = true, | ||
| 724 | .p15 = true, | ||
| 725 | .fpcr = true, | ||
| 726 | .fpsr = true, | ||
| 727 | .ffr = true, | ||
| 728 | .memory = true, | ||
| 729 | }), | ||
| 730 | .x86_64 => asm volatile ( | ||
| 731 | \\ movq 0(%%rsi), %%rax | ||
| 732 | \\ movq 8(%%rsi), %%rcx | ||
| 733 | \\ leaq 0f(%%rip), %%rdx | ||
| 734 | \\ movq %%rsp, 0(%%rax) | ||
| 735 | \\ movq %%rbp, 8(%%rax) | ||
| 736 | \\ movq %%rdx, 16(%%rax) | ||
| 737 | \\ movq 0(%%rcx), %%rsp | ||
| 738 | \\ movq 8(%%rcx), %%rbp | ||
| 739 | \\ jmpq *16(%%rcx) | ||
| 740 | \\0: | ||
| 741 | : [received_message] "={rsi}" (-> *const @FieldType(SwitchMessage, "contexts")), | ||
| 742 | : [message_to_send] "{rsi}" (&message.contexts), | ||
| 743 | : .{ | ||
| 744 | .rax = true, | ||
| 745 | .rcx = true, | ||
| 746 | .rdx = true, | ||
| 747 | .rbx = true, | ||
| 748 | .rsi = true, | ||
| 749 | .rdi = true, | ||
| 750 | .r8 = true, | ||
| 751 | .r9 = true, | ||
| 752 | .r10 = true, | ||
| 753 | .r11 = true, | ||
| 754 | .r12 = true, | ||
| 755 | .r13 = true, | ||
| 756 | .r14 = true, | ||
| 757 | .r15 = true, | ||
| 758 | .mm0 = true, | ||
| 759 | .mm1 = true, | ||
| 760 | .mm2 = true, | ||
| 761 | .mm3 = true, | ||
| 762 | .mm4 = true, | ||
| 763 | .mm5 = true, | ||
| 764 | .mm6 = true, | ||
| 765 | .mm7 = true, | ||
| 766 | .zmm0 = true, | ||
| 767 | .zmm1 = true, | ||
| 768 | .zmm2 = true, | ||
| 769 | .zmm3 = true, | ||
| 770 | .zmm4 = true, | ||
| 771 | .zmm5 = true, | ||
| 772 | .zmm6 = true, | ||
| 773 | .zmm7 = true, | ||
| 774 | .zmm8 = true, | ||
| 775 | .zmm9 = true, | ||
| 776 | .zmm10 = true, | ||
| 777 | .zmm11 = true, | ||
| 778 | .zmm12 = true, | ||
| 779 | .zmm13 = true, | ||
| 780 | .zmm14 = true, | ||
| 781 | .zmm15 = true, | ||
| 782 | .zmm16 = true, | ||
| 783 | .zmm17 = true, | ||
| 784 | .zmm18 = true, | ||
| 785 | .zmm19 = true, | ||
| 786 | .zmm20 = true, | ||
| 787 | .zmm21 = true, | ||
| 788 | .zmm22 = true, | ||
| 789 | .zmm23 = true, | ||
| 790 | .zmm24 = true, | ||
| 791 | .zmm25 = true, | ||
| 792 | .zmm26 = true, | ||
| 793 | .zmm27 = true, | ||
| 794 | .zmm28 = true, | ||
| 795 | .zmm29 = true, | ||
| 796 | .zmm30 = true, | ||
| 797 | .zmm31 = true, | ||
| 798 | .fpsr = true, | ||
| 799 | .fpcr = true, | ||
| 800 | .mxcsr = true, | ||
| 801 | .rflags = true, | ||
| 802 | .dirflag = true, | ||
| 803 | .memory = true, | ||
| 804 | }), | ||
| 805 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 806 | }); | ||
| 807 | } | ||
| 808 | |||
| 809 | fn mainIdleEntry() callconv(.naked) void { | ||
| 810 | switch (builtin.cpu.arch) { | ||
| 811 | .x86_64 => asm volatile ( | ||
| 812 | \\ movq (%%rsp), %%rdi | ||
| 813 | \\ jmp %[mainIdle:P] | ||
| 814 | : | ||
| 815 | : [mainIdle] "X" (&mainIdle), | ||
| 816 | ), | ||
| 817 | .aarch64 => asm volatile ( | ||
| 818 | \\ ldr x0, [sp, #-8] | ||
| 819 | \\ b %[mainIdle] | ||
| 820 | : | ||
| 821 | : [mainIdle] "X" (&mainIdle), | ||
| 822 | ), | ||
| 823 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 824 | } | ||
| 825 | } | ||
| 826 | |||
| 827 | fn fiberEntry() callconv(.naked) void { | ||
| 828 | switch (builtin.cpu.arch) { | ||
| 829 | .x86_64 => asm volatile ( | ||
| 830 | \\ leaq 8(%%rsp), %%rdi | ||
| 831 | \\ jmpq *(%%rsp) | ||
| 832 | ), | ||
| 833 | .aarch64 => asm volatile ( | ||
| 834 | \\ mov x0, sp | ||
| 835 | \\ ldr x2, [sp, #-8] | ||
| 836 | \\ br x2 | ||
| 837 | ), | ||
| 838 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 839 | } | ||
| 840 | } | ||
| 841 | |||
| 842 | const AsyncClosure = struct { | ||
| 843 | event_loop: *EventLoop, | ||
| 844 | fiber: *Fiber, | ||
| 845 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 846 | result_align: Alignment, | ||
| 847 | already_awaited: bool, | ||
| 848 | |||
| 849 | fn contextPointer(closure: *AsyncClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 { | ||
| 850 | return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure)); | ||
| 851 | } | ||
| 852 | |||
| 853 | fn call(closure: *AsyncClosure, message: *const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn { | ||
| 854 | message.handle(closure.event_loop); | ||
| 855 | const fiber = closure.fiber; | ||
| 856 | std.log.debug("{*} performing async", .{fiber}); | ||
| 857 | closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align)); | ||
| 858 | const awaiter = @atomicRmw(?*Fiber, &fiber.awaiter, .Xchg, Fiber.finished, .acq_rel); | ||
| 859 | const ready_awaiter = r: { | ||
| 860 | const a = awaiter orelse break :r null; | ||
| 861 | if (@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .acq_rel)) break :r null; | ||
| 862 | break :r a; | ||
| 863 | }; | ||
| 864 | closure.event_loop.yield(ready_awaiter, .nothing); | ||
| 865 | unreachable; // switched to dead fiber | ||
| 866 | } | ||
| 867 | |||
| 868 | fn fromFiber(fiber: *Fiber) *AsyncClosure { | ||
| 869 | return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward( | ||
| 870 | @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size, | ||
| 871 | ) - @sizeOf(AsyncClosure)); | ||
| 872 | } | ||
| 873 | }; | ||
| 874 | |||
| 875 | fn async( | ||
| 876 | userdata: ?*anyopaque, | ||
| 877 | result: []u8, | ||
| 878 | result_alignment: Alignment, | ||
| 879 | context: []const u8, | ||
| 880 | context_alignment: Alignment, | ||
| 881 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 882 | ) ?*std.Io.AnyFuture { | ||
| 883 | return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch { | ||
| 884 | start(context.ptr, result.ptr); | ||
| 885 | return null; | ||
| 886 | }; | ||
| 887 | } | ||
| 888 | |||
| 889 | fn concurrent( | ||
| 890 | userdata: ?*anyopaque, | ||
| 891 | result_len: usize, | ||
| 892 | result_alignment: Alignment, | ||
| 893 | context: []const u8, | ||
| 894 | context_alignment: Alignment, | ||
| 895 | start: *const fn (context: *const anyopaque, result: *anyopaque) void, | ||
| 896 | ) error{OutOfMemory}!*std.Io.AnyFuture { | ||
| 897 | assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO | ||
| 898 | assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO | ||
| 899 | assert(result_len <= Fiber.max_result_size); // TODO | ||
| 900 | assert(context.len <= Fiber.max_context_size); // TODO | ||
| 901 | |||
| 902 | const event_loop: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 903 | const fiber = try Fiber.allocate(event_loop); | ||
| 904 | std.log.debug("allocated {*}", .{fiber}); | ||
| 905 | |||
| 906 | const closure: *AsyncClosure = .fromFiber(fiber); | ||
| 907 | const stack_end: [*]align(16) usize = @ptrCast(@alignCast(closure)); | ||
| 908 | (stack_end - 1)[0..1].* = .{@intFromPtr(&AsyncClosure.call)}; | ||
| 909 | fiber.* = .{ | ||
| 910 | .required_align = {}, | ||
| 911 | .context = switch (builtin.cpu.arch) { | ||
| 912 | .x86_64 => .{ | ||
| 913 | .rsp = @intFromPtr(stack_end - 1), | ||
| 914 | .rbp = 0, | ||
| 915 | .rip = @intFromPtr(&fiberEntry), | ||
| 916 | }, | ||
| 917 | .aarch64 => .{ | ||
| 918 | .sp = @intFromPtr(stack_end), | ||
| 919 | .fp = 0, | ||
| 920 | .pc = @intFromPtr(&fiberEntry), | ||
| 921 | }, | ||
| 922 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 923 | }, | ||
| 924 | .awaiter = null, | ||
| 925 | .queue_next = null, | ||
| 926 | .cancel_thread = null, | ||
| 927 | .awaiting_completions = .initEmpty(), | ||
| 928 | }; | ||
| 929 | closure.* = .{ | ||
| 930 | .event_loop = event_loop, | ||
| 931 | .fiber = fiber, | ||
| 932 | .start = start, | ||
| 933 | .result_align = result_alignment, | ||
| 934 | .already_awaited = false, | ||
| 935 | }; | ||
| 936 | @memcpy(closure.contextPointer(), context); | ||
| 937 | |||
| 938 | event_loop.schedule(.current(), .{ .head = fiber, .tail = fiber }); | ||
| 939 | return @ptrCast(fiber); | ||
| 940 | } | ||
| 941 | |||
| 942 | const DetachedClosure = struct { | ||
| 943 | event_loop: *EventLoop, | ||
| 944 | fiber: *Fiber, | ||
| 945 | start: *const fn (context: *const anyopaque) void, | ||
| 946 | detached_queue_node: std.DoublyLinkedList.Node, | ||
| 947 | |||
| 948 | fn contextPointer(closure: *DetachedClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 { | ||
| 949 | return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(DetachedClosure)); | ||
| 950 | } | ||
| 951 | |||
| 952 | fn call(closure: *DetachedClosure, message: *const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(DetachedClosure))) noreturn { | ||
| 953 | message.handle(closure.event_loop); | ||
| 954 | std.log.debug("{*} performing async detached", .{closure.fiber}); | ||
| 955 | closure.start(closure.contextPointer()); | ||
| 956 | const awaiter = @atomicRmw(?*Fiber, &closure.fiber.awaiter, .Xchg, Fiber.finished, .acq_rel); | ||
| 957 | closure.event_loop.yield(awaiter, pending_task: { | ||
| 958 | closure.event_loop.detached.mutex.lock(closure.event_loop.io()) catch |err| switch (err) { | ||
| 959 | error.Canceled => break :pending_task .nothing, | ||
| 960 | }; | ||
| 961 | defer closure.event_loop.detached.mutex.unlock(closure.event_loop.io()); | ||
| 962 | if (closure.detached_queue_node.next == &closure.detached_queue_node) break :pending_task .nothing; | ||
| 963 | closure.event_loop.detached.list.remove(&closure.detached_queue_node); | ||
| 964 | break :pending_task .recycle; | ||
| 965 | }); | ||
| 966 | unreachable; // switched to dead fiber | ||
| 967 | } | ||
| 968 | }; | ||
| 969 | |||
| 970 | fn asyncDetached( | ||
| 971 | userdata: ?*anyopaque, | ||
| 972 | context: []const u8, | ||
| 973 | context_alignment: std.mem.Alignment, | ||
| 974 | start: *const fn (context: *const anyopaque) void, | ||
| 975 | ) void { | ||
| 976 | assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO | ||
| 977 | assert(context.len <= Fiber.max_context_size); // TODO | ||
| 978 | |||
| 979 | const event_loop: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 980 | const fiber = Fiber.allocate(event_loop) catch { | ||
| 981 | start(context.ptr); | ||
| 982 | return; | ||
| 983 | }; | ||
| 984 | std.log.debug("allocated {*}", .{fiber}); | ||
| 985 | |||
| 986 | const current_thread: *Thread = .current(); | ||
| 987 | const closure: *DetachedClosure = @ptrFromInt(Fiber.max_context_align.max(.of(DetachedClosure)).backward( | ||
| 988 | @intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size, | ||
| 989 | ) - @sizeOf(DetachedClosure)); | ||
| 990 | const stack_end: [*]align(16) usize = @ptrCast(@alignCast(closure)); | ||
| 991 | (stack_end - 1)[0..1].* = .{@intFromPtr(&DetachedClosure.call)}; | ||
| 992 | fiber.* = .{ | ||
| 993 | .required_align = {}, | ||
| 994 | .context = switch (builtin.cpu.arch) { | ||
| 995 | .x86_64 => .{ | ||
| 996 | .rsp = @intFromPtr(stack_end - 1), | ||
| 997 | .rbp = 0, | ||
| 998 | .rip = @intFromPtr(&fiberEntry), | ||
| 999 | }, | ||
| 1000 | .aarch64 => .{ | ||
| 1001 | .sp = @intFromPtr(stack_end), | ||
| 1002 | .fp = 0, | ||
| 1003 | .pc = @intFromPtr(&fiberEntry), | ||
| 1004 | }, | ||
| 1005 | else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)), | ||
| 1006 | }, | ||
| 1007 | .awaiter = null, | ||
| 1008 | .queue_next = null, | ||
| 1009 | .cancel_thread = null, | ||
| 1010 | .awaiting_completions = .initEmpty(), | ||
| 1011 | }; | ||
| 1012 | closure.* = .{ | ||
| 1013 | .event_loop = event_loop, | ||
| 1014 | .fiber = fiber, | ||
| 1015 | .start = start, | ||
| 1016 | .detached_queue_node = .{}, | ||
| 1017 | }; | ||
| 1018 | { | ||
| 1019 | event_loop.detached.mutex.lock(event_loop.io()) catch |err| switch (err) { | ||
| 1020 | error.Canceled => { | ||
| 1021 | event_loop.recycle(fiber); | ||
| 1022 | start(context.ptr); | ||
| 1023 | return; | ||
| 1024 | }, | ||
| 1025 | }; | ||
| 1026 | defer event_loop.detached.mutex.unlock(event_loop.io()); | ||
| 1027 | event_loop.detached.list.append(&closure.detached_queue_node); | ||
| 1028 | } | ||
| 1029 | @memcpy(closure.contextPointer(), context); | ||
| 1030 | |||
| 1031 | event_loop.schedule(current_thread, .{ .head = fiber, .tail = fiber }); | ||
| 1032 | } | ||
| 1033 | |||
| 1034 | fn await( | ||
| 1035 | userdata: ?*anyopaque, | ||
| 1036 | any_future: *std.Io.AnyFuture, | ||
| 1037 | result: []u8, | ||
| 1038 | result_alignment: Alignment, | ||
| 1039 | ) void { | ||
| 1040 | const event_loop: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1041 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1042 | if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished) | ||
| 1043 | event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter }); | ||
| 1044 | @memcpy(result, future_fiber.resultBytes(result_alignment)); | ||
| 1045 | event_loop.recycle(future_fiber); | ||
| 1046 | } | ||
| 1047 | |||
| 1048 | fn select(userdata: ?*anyopaque, futures: []const *Io.AnyFuture) usize { | ||
| 1049 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1050 | |||
| 1051 | // Optimization to avoid the yield below. | ||
| 1052 | for (futures, 0..) |any_future, i| { | ||
| 1053 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1054 | if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) == Fiber.finished) | ||
| 1055 | return i; | ||
| 1056 | } | ||
| 1057 | |||
| 1058 | el.yield(null, .{ .register_select = futures }); | ||
| 1059 | |||
| 1060 | std.log.debug("back from select yield", .{}); | ||
| 1061 | |||
| 1062 | const my_thread: *Thread = .current(); | ||
| 1063 | const my_fiber = my_thread.currentFiber(); | ||
| 1064 | var result: ?usize = null; | ||
| 1065 | |||
| 1066 | for (futures, 0..) |any_future, i| { | ||
| 1067 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1068 | if (@cmpxchgStrong(?*Fiber, &future_fiber.awaiter, my_fiber, null, .seq_cst, .seq_cst)) |awaiter| { | ||
| 1069 | if (awaiter == Fiber.finished) { | ||
| 1070 | if (result == null) result = i; | ||
| 1071 | } else if (awaiter) |a| { | ||
| 1072 | const closure: *AsyncClosure = .fromFiber(a); | ||
| 1073 | closure.already_awaited = false; | ||
| 1074 | } | ||
| 1075 | } else { | ||
| 1076 | const closure: *AsyncClosure = .fromFiber(my_fiber); | ||
| 1077 | closure.already_awaited = false; | ||
| 1078 | } | ||
| 1079 | } | ||
| 1080 | |||
| 1081 | return result.?; | ||
| 1082 | } | ||
| 1083 | |||
| 1084 | fn cancel( | ||
| 1085 | userdata: ?*anyopaque, | ||
| 1086 | any_future: *std.Io.AnyFuture, | ||
| 1087 | result: []u8, | ||
| 1088 | result_alignment: Alignment, | ||
| 1089 | ) void { | ||
| 1090 | const future_fiber: *Fiber = @ptrCast(@alignCast(any_future)); | ||
| 1091 | if (@atomicRmw( | ||
| 1092 | ?*Thread, | ||
| 1093 | &future_fiber.cancel_thread, | ||
| 1094 | .Xchg, | ||
| 1095 | Thread.canceling, | ||
| 1096 | .acq_rel, | ||
| 1097 | )) |cancel_thread| if (cancel_thread != Thread.canceling) { | ||
| 1098 | getSqe(&Thread.current().io_uring).* = .{ | ||
| 1099 | .opcode = .MSG_RING, | ||
| 1100 | .flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS, | ||
| 1101 | .ioprio = 0, | ||
| 1102 | .fd = cancel_thread.io_uring.fd, | ||
| 1103 | .off = @intFromPtr(future_fiber), | ||
| 1104 | .addr = 0, | ||
| 1105 | .len = @bitCast(-@as(i32, @intFromEnum(std.os.linux.E.INTR))), | ||
| 1106 | .rw_flags = 0, | ||
| 1107 | .user_data = @intFromEnum(Completion.UserData.cleanup), | ||
| 1108 | .buf_index = 0, | ||
| 1109 | .personality = 0, | ||
| 1110 | .splice_fd_in = 0, | ||
| 1111 | .addr3 = 0, | ||
| 1112 | .resv = 0, | ||
| 1113 | }; | ||
| 1114 | }; | ||
| 1115 | await(userdata, any_future, result, result_alignment); | ||
| 1116 | } | ||
| 1117 | |||
| 1118 | fn cancelRequested(userdata: ?*anyopaque) bool { | ||
| 1119 | _ = userdata; | ||
| 1120 | return @atomicLoad(?*Thread, &Thread.current().currentFiber().cancel_thread, .acquire) == Thread.canceling; | ||
| 1121 | } | ||
| 1122 | |||
| 1123 | fn createFile( | ||
| 1124 | userdata: ?*anyopaque, | ||
| 1125 | dir: Io.Dir, | ||
| 1126 | sub_path: []const u8, | ||
| 1127 | flags: Io.File.CreateFlags, | ||
| 1128 | ) Io.File.OpenError!Io.File { | ||
| 1129 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1130 | const thread: *Thread = .current(); | ||
| 1131 | const iou = &thread.io_uring; | ||
| 1132 | const fiber = thread.currentFiber(); | ||
| 1133 | try fiber.enterCancelRegion(thread); | ||
| 1134 | |||
| 1135 | const posix = std.posix; | ||
| 1136 | const sub_path_c = try posix.toPosixPath(sub_path); | ||
| 1137 | |||
| 1138 | var os_flags: posix.O = .{ | ||
| 1139 | .ACCMODE = if (flags.read) .RDWR else .WRONLY, | ||
| 1140 | .CREAT = true, | ||
| 1141 | .TRUNC = flags.truncate, | ||
| 1142 | .EXCL = flags.exclusive, | ||
| 1143 | }; | ||
| 1144 | if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true; | ||
| 1145 | if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true; | ||
| 1146 | |||
| 1147 | // Use the O locking flags if the os supports them to acquire the lock | ||
| 1148 | // atomically. Note that the NONBLOCK flag is removed after the openat() | ||
| 1149 | // call is successful. | ||
| 1150 | const has_flock_open_flags = @hasField(posix.O, "EXLOCK"); | ||
| 1151 | if (has_flock_open_flags) switch (flags.lock) { | ||
| 1152 | .none => {}, | ||
| 1153 | .shared => { | ||
| 1154 | os_flags.SHLOCK = true; | ||
| 1155 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1156 | }, | ||
| 1157 | .exclusive => { | ||
| 1158 | os_flags.EXLOCK = true; | ||
| 1159 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1160 | }, | ||
| 1161 | }; | ||
| 1162 | const have_flock = @TypeOf(posix.system.flock) != void; | ||
| 1163 | |||
| 1164 | if (have_flock and !has_flock_open_flags and flags.lock != .none) { | ||
| 1165 | @panic("TODO"); | ||
| 1166 | } | ||
| 1167 | |||
| 1168 | if (has_flock_open_flags and flags.lock_nonblocking) { | ||
| 1169 | @panic("TODO"); | ||
| 1170 | } | ||
| 1171 | |||
| 1172 | getSqe(iou).* = .{ | ||
| 1173 | .opcode = .OPENAT, | ||
| 1174 | .flags = 0, | ||
| 1175 | .ioprio = 0, | ||
| 1176 | .fd = dir.handle, | ||
| 1177 | .off = 0, | ||
| 1178 | .addr = @intFromPtr(&sub_path_c), | ||
| 1179 | .len = @intCast(flags.mode), | ||
| 1180 | .rw_flags = @bitCast(os_flags), | ||
| 1181 | .user_data = @intFromPtr(fiber), | ||
| 1182 | .buf_index = 0, | ||
| 1183 | .personality = 0, | ||
| 1184 | .splice_fd_in = 0, | ||
| 1185 | .addr3 = 0, | ||
| 1186 | .resv = 0, | ||
| 1187 | }; | ||
| 1188 | |||
| 1189 | el.yield(null, .nothing); | ||
| 1190 | fiber.exitCancelRegion(thread); | ||
| 1191 | |||
| 1192 | const completion = fiber.resultPointer(Completion); | ||
| 1193 | switch (errno(completion.result)) { | ||
| 1194 | .SUCCESS => return .{ .handle = completion.result }, | ||
| 1195 | .INTR => unreachable, | ||
| 1196 | .CANCELED => return error.Canceled, | ||
| 1197 | |||
| 1198 | .FAULT => unreachable, | ||
| 1199 | .INVAL => return error.BadPathName, | ||
| 1200 | .BADF => unreachable, | ||
| 1201 | .ACCES => return error.AccessDenied, | ||
| 1202 | .FBIG => return error.FileTooBig, | ||
| 1203 | .OVERFLOW => return error.FileTooBig, | ||
| 1204 | .ISDIR => return error.IsDir, | ||
| 1205 | .LOOP => return error.SymLinkLoop, | ||
| 1206 | .MFILE => return error.ProcessFdQuotaExceeded, | ||
| 1207 | .NAMETOOLONG => return error.NameTooLong, | ||
| 1208 | .NFILE => return error.SystemFdQuotaExceeded, | ||
| 1209 | .NODEV => return error.NoDevice, | ||
| 1210 | .NOENT => return error.FileNotFound, | ||
| 1211 | .NOMEM => return error.SystemResources, | ||
| 1212 | .NOSPC => return error.NoSpaceLeft, | ||
| 1213 | .NOTDIR => return error.NotDir, | ||
| 1214 | .PERM => return error.PermissionDenied, | ||
| 1215 | .EXIST => return error.PathAlreadyExists, | ||
| 1216 | .BUSY => return error.DeviceBusy, | ||
| 1217 | .OPNOTSUPP => return error.FileLocksNotSupported, | ||
| 1218 | .AGAIN => return error.WouldBlock, | ||
| 1219 | .TXTBSY => return error.FileBusy, | ||
| 1220 | .NXIO => return error.NoDevice, | ||
| 1221 | else => |err| return posix.unexpectedErrno(err), | ||
| 1222 | } | ||
| 1223 | } | ||
| 1224 | |||
| 1225 | fn fileOpen( | ||
| 1226 | userdata: ?*anyopaque, | ||
| 1227 | dir: Io.Dir, | ||
| 1228 | sub_path: []const u8, | ||
| 1229 | flags: Io.File.OpenFlags, | ||
| 1230 | ) Io.File.OpenError!Io.File { | ||
| 1231 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1232 | const thread: *Thread = .current(); | ||
| 1233 | const iou = &thread.io_uring; | ||
| 1234 | const fiber = thread.currentFiber(); | ||
| 1235 | try fiber.enterCancelRegion(thread); | ||
| 1236 | |||
| 1237 | const posix = std.posix; | ||
| 1238 | const sub_path_c = try posix.toPosixPath(sub_path); | ||
| 1239 | |||
| 1240 | var os_flags: posix.O = .{ | ||
| 1241 | .ACCMODE = switch (flags.mode) { | ||
| 1242 | .read_only => .RDONLY, | ||
| 1243 | .write_only => .WRONLY, | ||
| 1244 | .read_write => .RDWR, | ||
| 1245 | }, | ||
| 1246 | }; | ||
| 1247 | |||
| 1248 | if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true; | ||
| 1249 | if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true; | ||
| 1250 | if (@hasField(posix.O, "NOCTTY")) os_flags.NOCTTY = !flags.allow_ctty; | ||
| 1251 | |||
| 1252 | // Use the O locking flags if the os supports them to acquire the lock | ||
| 1253 | // atomically. | ||
| 1254 | const has_flock_open_flags = @hasField(posix.O, "EXLOCK"); | ||
| 1255 | if (has_flock_open_flags) { | ||
| 1256 | // Note that the NONBLOCK flag is removed after the openat() call | ||
| 1257 | // is successful. | ||
| 1258 | switch (flags.lock) { | ||
| 1259 | .none => {}, | ||
| 1260 | .shared => { | ||
| 1261 | os_flags.SHLOCK = true; | ||
| 1262 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1263 | }, | ||
| 1264 | .exclusive => { | ||
| 1265 | os_flags.EXLOCK = true; | ||
| 1266 | os_flags.NONBLOCK = flags.lock_nonblocking; | ||
| 1267 | }, | ||
| 1268 | } | ||
| 1269 | } | ||
| 1270 | const have_flock = @TypeOf(posix.system.flock) != void; | ||
| 1271 | |||
| 1272 | if (have_flock and !has_flock_open_flags and flags.lock != .none) { | ||
| 1273 | @panic("TODO"); | ||
| 1274 | } | ||
| 1275 | |||
| 1276 | if (has_flock_open_flags and flags.lock_nonblocking) { | ||
| 1277 | @panic("TODO"); | ||
| 1278 | } | ||
| 1279 | |||
| 1280 | getSqe(iou).* = .{ | ||
| 1281 | .opcode = .OPENAT, | ||
| 1282 | .flags = 0, | ||
| 1283 | .ioprio = 0, | ||
| 1284 | .fd = dir.handle, | ||
| 1285 | .off = 0, | ||
| 1286 | .addr = @intFromPtr(&sub_path_c), | ||
| 1287 | .len = 0, | ||
| 1288 | .rw_flags = @bitCast(os_flags), | ||
| 1289 | .user_data = @intFromPtr(fiber), | ||
| 1290 | .buf_index = 0, | ||
| 1291 | .personality = 0, | ||
| 1292 | .splice_fd_in = 0, | ||
| 1293 | .addr3 = 0, | ||
| 1294 | .resv = 0, | ||
| 1295 | }; | ||
| 1296 | |||
| 1297 | el.yield(null, .nothing); | ||
| 1298 | fiber.exitCancelRegion(thread); | ||
| 1299 | |||
| 1300 | const completion = fiber.resultPointer(Completion); | ||
| 1301 | switch (errno(completion.result)) { | ||
| 1302 | .SUCCESS => return .{ .handle = completion.result }, | ||
| 1303 | .INTR => unreachable, | ||
| 1304 | .CANCELED => return error.Canceled, | ||
| 1305 | |||
| 1306 | .FAULT => unreachable, | ||
| 1307 | .INVAL => return error.BadPathName, | ||
| 1308 | .BADF => unreachable, | ||
| 1309 | .ACCES => return error.AccessDenied, | ||
| 1310 | .FBIG => return error.FileTooBig, | ||
| 1311 | .OVERFLOW => return error.FileTooBig, | ||
| 1312 | .ISDIR => return error.IsDir, | ||
| 1313 | .LOOP => return error.SymLinkLoop, | ||
| 1314 | .MFILE => return error.ProcessFdQuotaExceeded, | ||
| 1315 | .NAMETOOLONG => return error.NameTooLong, | ||
| 1316 | .NFILE => return error.SystemFdQuotaExceeded, | ||
| 1317 | .NODEV => return error.NoDevice, | ||
| 1318 | .NOENT => return error.FileNotFound, | ||
| 1319 | .NOMEM => return error.SystemResources, | ||
| 1320 | .NOSPC => return error.NoSpaceLeft, | ||
| 1321 | .NOTDIR => return error.NotDir, | ||
| 1322 | .PERM => return error.PermissionDenied, | ||
| 1323 | .EXIST => return error.PathAlreadyExists, | ||
| 1324 | .BUSY => return error.DeviceBusy, | ||
| 1325 | .OPNOTSUPP => return error.FileLocksNotSupported, | ||
| 1326 | .AGAIN => return error.WouldBlock, | ||
| 1327 | .TXTBSY => return error.FileBusy, | ||
| 1328 | .NXIO => return error.NoDevice, | ||
| 1329 | else => |err| return posix.unexpectedErrno(err), | ||
| 1330 | } | ||
| 1331 | } | ||
| 1332 | |||
| 1333 | fn fileClose(userdata: ?*anyopaque, file: Io.File) void { | ||
| 1334 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1335 | const thread: *Thread = .current(); | ||
| 1336 | const iou = &thread.io_uring; | ||
| 1337 | const fiber = thread.currentFiber(); | ||
| 1338 | |||
| 1339 | getSqe(iou).* = .{ | ||
| 1340 | .opcode = .CLOSE, | ||
| 1341 | .flags = 0, | ||
| 1342 | .ioprio = 0, | ||
| 1343 | .fd = file.handle, | ||
| 1344 | .off = 0, | ||
| 1345 | .addr = 0, | ||
| 1346 | .len = 0, | ||
| 1347 | .rw_flags = 0, | ||
| 1348 | .user_data = @intFromPtr(fiber), | ||
| 1349 | .buf_index = 0, | ||
| 1350 | .personality = 0, | ||
| 1351 | .splice_fd_in = 0, | ||
| 1352 | .addr3 = 0, | ||
| 1353 | .resv = 0, | ||
| 1354 | }; | ||
| 1355 | |||
| 1356 | el.yield(null, .nothing); | ||
| 1357 | |||
| 1358 | const completion = fiber.resultPointer(Completion); | ||
| 1359 | switch (errno(completion.result)) { | ||
| 1360 | .SUCCESS => return, | ||
| 1361 | .INTR => unreachable, | ||
| 1362 | .CANCELED => return, | ||
| 1363 | |||
| 1364 | .BADF => unreachable, // Always a race condition. | ||
| 1365 | else => return, | ||
| 1366 | } | ||
| 1367 | } | ||
| 1368 | |||
| 1369 | fn pread(userdata: ?*anyopaque, file: Io.File, buffer: []u8, offset: std.posix.off_t) Io.File.PReadError!usize { | ||
| 1370 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1371 | const thread: *Thread = .current(); | ||
| 1372 | const iou = &thread.io_uring; | ||
| 1373 | const fiber = thread.currentFiber(); | ||
| 1374 | try fiber.enterCancelRegion(thread); | ||
| 1375 | |||
| 1376 | getSqe(iou).* = .{ | ||
| 1377 | .opcode = .READ, | ||
| 1378 | .flags = 0, | ||
| 1379 | .ioprio = 0, | ||
| 1380 | .fd = file.handle, | ||
| 1381 | .off = @bitCast(offset), | ||
| 1382 | .addr = @intFromPtr(buffer.ptr), | ||
| 1383 | .len = @min(buffer.len, 0x7ffff000), | ||
| 1384 | .rw_flags = 0, | ||
| 1385 | .user_data = @intFromPtr(fiber), | ||
| 1386 | .buf_index = 0, | ||
| 1387 | .personality = 0, | ||
| 1388 | .splice_fd_in = 0, | ||
| 1389 | .addr3 = 0, | ||
| 1390 | .resv = 0, | ||
| 1391 | }; | ||
| 1392 | |||
| 1393 | el.yield(null, .nothing); | ||
| 1394 | fiber.exitCancelRegion(thread); | ||
| 1395 | |||
| 1396 | const completion = fiber.resultPointer(Completion); | ||
| 1397 | switch (errno(completion.result)) { | ||
| 1398 | .SUCCESS => return @as(u32, @bitCast(completion.result)), | ||
| 1399 | .INTR => unreachable, | ||
| 1400 | .CANCELED => return error.Canceled, | ||
| 1401 | |||
| 1402 | .INVAL => unreachable, | ||
| 1403 | .FAULT => unreachable, | ||
| 1404 | .NOENT => return error.ProcessNotFound, | ||
| 1405 | .AGAIN => return error.WouldBlock, | ||
| 1406 | .BADF => return error.NotOpenForReading, // Can be a race condition. | ||
| 1407 | .IO => return error.InputOutput, | ||
| 1408 | .ISDIR => return error.IsDir, | ||
| 1409 | .NOBUFS => return error.SystemResources, | ||
| 1410 | .NOMEM => return error.SystemResources, | ||
| 1411 | .NOTCONN => return error.SocketUnconnected, | ||
| 1412 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 1413 | .TIMEDOUT => return error.Timeout, | ||
| 1414 | .NXIO => return error.Unseekable, | ||
| 1415 | .SPIPE => return error.Unseekable, | ||
| 1416 | .OVERFLOW => return error.Unseekable, | ||
| 1417 | else => |err| return std.posix.unexpectedErrno(err), | ||
| 1418 | } | ||
| 1419 | } | ||
| 1420 | |||
| 1421 | fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: std.posix.off_t) Io.File.PWriteError!usize { | ||
| 1422 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1423 | const thread: *Thread = .current(); | ||
| 1424 | const iou = &thread.io_uring; | ||
| 1425 | const fiber = thread.currentFiber(); | ||
| 1426 | try fiber.enterCancelRegion(thread); | ||
| 1427 | |||
| 1428 | getSqe(iou).* = .{ | ||
| 1429 | .opcode = .WRITE, | ||
| 1430 | .flags = 0, | ||
| 1431 | .ioprio = 0, | ||
| 1432 | .fd = file.handle, | ||
| 1433 | .off = @bitCast(offset), | ||
| 1434 | .addr = @intFromPtr(buffer.ptr), | ||
| 1435 | .len = @min(buffer.len, 0x7ffff000), | ||
| 1436 | .rw_flags = 0, | ||
| 1437 | .user_data = @intFromPtr(fiber), | ||
| 1438 | .buf_index = 0, | ||
| 1439 | .personality = 0, | ||
| 1440 | .splice_fd_in = 0, | ||
| 1441 | .addr3 = 0, | ||
| 1442 | .resv = 0, | ||
| 1443 | }; | ||
| 1444 | |||
| 1445 | el.yield(null, .nothing); | ||
| 1446 | fiber.exitCancelRegion(thread); | ||
| 1447 | |||
| 1448 | const completion = fiber.resultPointer(Completion); | ||
| 1449 | switch (errno(completion.result)) { | ||
| 1450 | .SUCCESS => return @as(u32, @bitCast(completion.result)), | ||
| 1451 | .INTR => unreachable, | ||
| 1452 | .CANCELED => return error.Canceled, | ||
| 1453 | |||
| 1454 | .INVAL => return error.InvalidArgument, | ||
| 1455 | .FAULT => unreachable, | ||
| 1456 | .NOENT => return error.ProcessNotFound, | ||
| 1457 | .AGAIN => return error.WouldBlock, | ||
| 1458 | .BADF => return error.NotOpenForWriting, // can be a race condition. | ||
| 1459 | .DESTADDRREQ => unreachable, // `connect` was never called. | ||
| 1460 | .DQUOT => return error.DiskQuota, | ||
| 1461 | .FBIG => return error.FileTooBig, | ||
| 1462 | .IO => return error.InputOutput, | ||
| 1463 | .NOSPC => return error.NoSpaceLeft, | ||
| 1464 | .ACCES => return error.AccessDenied, | ||
| 1465 | .PERM => return error.PermissionDenied, | ||
| 1466 | .PIPE => return error.BrokenPipe, | ||
| 1467 | .NXIO => return error.Unseekable, | ||
| 1468 | .SPIPE => return error.Unseekable, | ||
| 1469 | .OVERFLOW => return error.Unseekable, | ||
| 1470 | .BUSY => return error.DeviceBusy, | ||
| 1471 | .CONNRESET => return error.ConnectionResetByPeer, | ||
| 1472 | .MSGSIZE => return error.MessageTooBig, | ||
| 1473 | else => |err| return std.posix.unexpectedErrno(err), | ||
| 1474 | } | ||
| 1475 | } | ||
| 1476 | |||
| 1477 | fn now(userdata: ?*anyopaque, clockid: std.posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp { | ||
| 1478 | _ = userdata; | ||
| 1479 | const timespec = try std.posix.clock_gettime(clockid); | ||
| 1480 | return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec); | ||
| 1481 | } | ||
| 1482 | |||
| 1483 | fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void { | ||
| 1484 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1485 | const thread: *Thread = .current(); | ||
| 1486 | const iou = &thread.io_uring; | ||
| 1487 | const fiber = thread.currentFiber(); | ||
| 1488 | try fiber.enterCancelRegion(thread); | ||
| 1489 | |||
| 1490 | const deadline_nanoseconds: i96 = switch (deadline) { | ||
| 1491 | .duration => |duration| duration.nanoseconds, | ||
| 1492 | .timestamp => |timestamp| @intFromEnum(timestamp), | ||
| 1493 | }; | ||
| 1494 | const timespec: std.os.linux.kernel_timespec = .{ | ||
| 1495 | .sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 1496 | .nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)), | ||
| 1497 | }; | ||
| 1498 | getSqe(iou).* = .{ | ||
| 1499 | .opcode = .TIMEOUT, | ||
| 1500 | .flags = 0, | ||
| 1501 | .ioprio = 0, | ||
| 1502 | .fd = 0, | ||
| 1503 | .off = 0, | ||
| 1504 | .addr = @intFromPtr(×pec), | ||
| 1505 | .len = 1, | ||
| 1506 | .rw_flags = @as(u32, switch (deadline) { | ||
| 1507 | .duration => 0, | ||
| 1508 | .timestamp => std.os.linux.IORING_TIMEOUT_ABS, | ||
| 1509 | }) | @as(u32, switch (clockid) { | ||
| 1510 | .REALTIME => std.os.linux.IORING_TIMEOUT_REALTIME, | ||
| 1511 | .MONOTONIC => 0, | ||
| 1512 | .BOOTTIME => std.os.linux.IORING_TIMEOUT_BOOTTIME, | ||
| 1513 | else => return error.UnsupportedClock, | ||
| 1514 | }), | ||
| 1515 | .user_data = @intFromPtr(fiber), | ||
| 1516 | .buf_index = 0, | ||
| 1517 | .personality = 0, | ||
| 1518 | .splice_fd_in = 0, | ||
| 1519 | .addr3 = 0, | ||
| 1520 | .resv = 0, | ||
| 1521 | }; | ||
| 1522 | |||
| 1523 | el.yield(null, .nothing); | ||
| 1524 | fiber.exitCancelRegion(thread); | ||
| 1525 | |||
| 1526 | const completion = fiber.resultPointer(Completion); | ||
| 1527 | switch (errno(completion.result)) { | ||
| 1528 | .SUCCESS, .TIME => return, | ||
| 1529 | .INTR => unreachable, | ||
| 1530 | .CANCELED => return error.Canceled, | ||
| 1531 | |||
| 1532 | else => |err| return std.posix.unexpectedErrno(err), | ||
| 1533 | } | ||
| 1534 | } | ||
| 1535 | |||
| 1536 | fn mutexLock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) error{Canceled}!void { | ||
| 1537 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1538 | el.yield(null, .{ .mutex_lock = .{ .prev_state = prev_state, .mutex = mutex } }); | ||
| 1539 | } | ||
| 1540 | fn mutexUnlock(userdata: ?*anyopaque, prev_state: Io.Mutex.State, mutex: *Io.Mutex) void { | ||
| 1541 | var maybe_waiting_fiber: ?*Fiber = @ptrFromInt(@intFromEnum(prev_state)); | ||
| 1542 | while (if (maybe_waiting_fiber) |waiting_fiber| @cmpxchgWeak( | ||
| 1543 | Io.Mutex.State, | ||
| 1544 | &mutex.state, | ||
| 1545 | @enumFromInt(@intFromPtr(waiting_fiber)), | ||
| 1546 | @enumFromInt(@intFromPtr(waiting_fiber.queue_next)), | ||
| 1547 | .release, | ||
| 1548 | .acquire, | ||
| 1549 | ) else @cmpxchgWeak( | ||
| 1550 | Io.Mutex.State, | ||
| 1551 | &mutex.state, | ||
| 1552 | .locked_once, | ||
| 1553 | .unlocked, | ||
| 1554 | .release, | ||
| 1555 | .acquire, | ||
| 1556 | ) orelse return) |next_state| maybe_waiting_fiber = @ptrFromInt(@intFromEnum(next_state)); | ||
| 1557 | maybe_waiting_fiber.?.queue_next = null; | ||
| 1558 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1559 | el.yield(maybe_waiting_fiber.?, .reschedule); | ||
| 1560 | } | ||
| 1561 | |||
| 1562 | const ConditionImpl = struct { | ||
| 1563 | tail: *Fiber, | ||
| 1564 | event: union(enum) { | ||
| 1565 | queued, | ||
| 1566 | wake: Io.Condition.Wake, | ||
| 1567 | }, | ||
| 1568 | }; | ||
| 1569 | |||
| 1570 | fn conditionWait(userdata: ?*anyopaque, cond: *Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void { | ||
| 1571 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1572 | el.yield(null, .{ .condition_wait = .{ .cond = cond, .mutex = mutex } }); | ||
| 1573 | const thread = Thread.current(); | ||
| 1574 | const fiber = thread.currentFiber(); | ||
| 1575 | const cond_impl = fiber.resultPointer(ConditionImpl); | ||
| 1576 | try mutex.lock(el.io()); | ||
| 1577 | switch (cond_impl.event) { | ||
| 1578 | .queued => {}, | ||
| 1579 | .wake => |wake| if (fiber.queue_next) |next_fiber| switch (wake) { | ||
| 1580 | .one => if (@cmpxchgStrong( | ||
| 1581 | ?*Fiber, | ||
| 1582 | @as(*?*Fiber, @ptrCast(&cond.state)), | ||
| 1583 | null, | ||
| 1584 | next_fiber, | ||
| 1585 | .release, | ||
| 1586 | .acquire, | ||
| 1587 | )) |old_fiber| { | ||
| 1588 | const old_cond_impl = old_fiber.?.resultPointer(ConditionImpl); | ||
| 1589 | assert(old_cond_impl.tail.queue_next == null); | ||
| 1590 | old_cond_impl.tail.queue_next = next_fiber; | ||
| 1591 | old_cond_impl.tail = cond_impl.tail; | ||
| 1592 | }, | ||
| 1593 | .all => el.schedule(thread, .{ .head = next_fiber, .tail = cond_impl.tail }), | ||
| 1594 | }, | ||
| 1595 | } | ||
| 1596 | fiber.queue_next = null; | ||
| 1597 | } | ||
| 1598 | |||
| 1599 | fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition.Wake) void { | ||
| 1600 | const el: *EventLoop = @ptrCast(@alignCast(userdata)); | ||
| 1601 | const waiting_fiber = @atomicRmw(?*Fiber, @as(*?*Fiber, @ptrCast(&cond.state)), .Xchg, null, .acquire) orelse return; | ||
| 1602 | waiting_fiber.resultPointer(ConditionImpl).event = .{ .wake = wake }; | ||
| 1603 | el.yield(waiting_fiber, .reschedule); | ||
| 1604 | } | ||
| 1605 | |||
| 1606 | fn errno(signed: i32) std.os.linux.E { | ||
| 1607 | return .init(@bitCast(@as(isize, signed))); | ||
| 1608 | } | ||
| 1609 | |||
| 1610 | fn getSqe(iou: *IoUring) *std.os.linux.io_uring_sqe { | ||
| 1611 | while (true) return iou.get_sqe() catch { | ||
| 1612 | _ = iou.submit_and_wait(0) catch |err| switch (err) { | ||
| 1613 | error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}), | ||
| 1614 | else => |e| @panic(@errorName(e)), | ||
| 1615 | }; | ||
| 1616 | continue; | ||
| 1617 | }; | ||
| 1618 | } | ||