Meg's Mirrors

zig

1	const addressFromPosix = Io.Threaded.addressFromPosix;
2	const addressToPosix = Io.Threaded.addressToPosix;
3	const Alignment = std.mem.Alignment;
4	const Allocator = std.mem.Allocator;
5	const Argv0 = Io.Threaded.Argv0;
6	const assert = std.debug.assert;
7	const builtin = @import("builtin");
8	const ChdirError = Io.Threaded.ChdirError;
9	const clockToPosix = Io.Threaded.clockToPosix;
10	const Csprng = Io.Threaded.Csprng;
11	const default_PATH = Io.Threaded.default_PATH;
12	const Dir = Io.Dir;
13	const Environ = Io.Threaded.Environ;
14	const errnoBug = Io.Threaded.errnoBug;
15	const Evented = @This();
16	const fallbackSeed = Io.Threaded.fallbackSeed;
17	const fd_t = linux.fd_t;
18	const File = Io.File;
19	const Io = std.Io;
20	const IoUring = linux.IoUring;
21	const iovec = std.posix.iovec;
22	const iovec_const = std.posix.iovec_const;
23	const linux = std.os.linux;
24	const linux_statx_request = Io.Threaded.linux_statx_request;
25	const LOCK = std.posix.LOCK;
26	const log = std.log.scoped(.@"io-uring");
27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
29	const net = Io.net;
30	const PATH_MAX = linux.PATH_MAX;
31	const pathToPosix = Io.Threaded.pathToPosix;
32	const pid_t = linux.pid_t;
33	const PosixAddress = Io.Threaded.PosixAddress;
34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
35	const posixSocketModeProtocol = Io.Threaded.posixSocketModeProtocol;
36	const process = std.process;
37	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
38	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
39	const splat_buffer_size = Io.Threaded.splat_buffer_size;
40	const statFromLinux = Io.Threaded.statFromLinux;
41	const statxKind = Io.Threaded.statxKind;
42	const std = @import("../std.zig");
43	const timestampFromPosix = Io.Threaded.timestampFromPosix;
44	const unexpectedErrno = std.posix.unexpectedErrno;
45	const winsize = std.posix.winsize;
46
47	const tracy = if (@hasDecl(@import("root"), "tracy")) @import("root").tracy else struct {
48	const enable = false;
49	inline fn fiberEnter(fiber: [*:0]const u8) void {
50	_ = fiber;
51	}
52	inline fn fiberLeave() void {}
53	};
54
55	/// Empirically saw >128KB being used by the self-hosted backend to panic.
56	/// Empirically saw glibc complain about 256KB.
57	const idle_stack_size = 512 * 1024;
58
59	const max_idle_search = 1;
60	const max_steal_ready_search = 2;
61	const max_steal_free_search = 4;
62
63	backing_allocator_needs_mutex: bool,
64	backing_allocator_mutex: Io.Mutex,
65	/// Does not need to be thread-safe if not used elsewhere.
66	backing_allocator: Allocator,
67	main_fiber_buffer: [
68	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
69	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
70	log2_ring_entries: u4,
71	threads: Thread.List,
72	sync_limit: ?Io.Semaphore,
73
74	stderr_writer_initialized: bool = false,
75	stderr_mutex: Io.Mutex,
76	stderr_writer: File.Writer = .{
77	.io = undefined,
78	.interface = Io.File.Writer.initInterface(&.{}),
79	.file = .stderr(),
80	.mode = .streaming,
81	},
82	stderr_mode: Io.Terminal.Mode = .no_color,
83
84	environ_mutex: Io.Mutex,
85	environ_initialized: bool,
86	environ: Environ,
87
88	null_fd: CachedFd,
89	random_fd: CachedFd,
90
91	csprng_mutex: Io.Mutex,
92	csprng: Csprng,
93
94	const Thread = struct {
95	required_align: void align(4),
96	thread: std.Thread,
97	idle_context: Io.fiber.Context,
98	current_context: *Io.fiber.Context,
99	ready_queue: ?*Fiber,
100	free_queue: ?*Fiber,
101	io_uring: IoUring,
102	idle_search_index: u32,
103	steal_ready_search_index: u32,
104	steal_free_search_index: u32,
105	name_arena: if (tracy.enable) std.heap.ArenaAllocator.State else struct {},
106	csprng: Csprng,
107
108	threadlocal var self: ?*Thread = null;
109
110	noinline fn current() *Thread {
111	return self.?;
112	}
113
114	fn deinit(thread: *Thread, gpa: Allocator) void {
115	var next_fiber = thread.free_queue;
116	while (next_fiber) |free_fiber| {
117	next_fiber = free_fiber.status.free_next;
118	gpa.free(free_fiber.allocatedSlice());
119	}
120	thread.io_uring.deinit();
121	}
122
123	fn currentFiber(thread: *Thread) *Fiber {
124	assert(thread.current_context != &thread.idle_context);
125	return @fieldParentPtr("context", thread.current_context);
126	}
127
128	fn enqueue(thread: *Thread) *linux.io_uring_sqe {
129	while (true) return thread.io_uring.get_sqe() catch {
130	thread.submit();
131	continue;
132	};
133	}
134
135	fn submit(thread: *Thread) void {
136	_ = thread.io_uring.submit() catch |err| switch (err) {
137	error.SignalInterrupt => {},
138	else => |e| @panic(@errorName(e)),
139	};
140	}
141
142	const List = struct {
143	allocated: []Thread,
144	reserved: u32,
145	active: u32,
146	};
147	};
148
149	const Fiber = struct {
150	required_align: void align(4),
151	context: Io.fiber.Context,
152	link: union {
153	awaiter: ?*Fiber,
154	group: struct { prev: ?*Fiber, next: ?*Fiber },
155	},
156	status: union(enum) {
157	queue_next: ?*Fiber,
158	awaiting_group: Group,
159	free_next: ?*Fiber,
160	},
161	cancel_status: CancelStatus,
162	cancel_protection: CancelProtection,
163	name: if (tracy.enable) [*:0]const u8 else void,
164
165	var next_name: u64 = 0;
166
167	const CancelStatus = packed struct(u32) {
168	requested: bool,
169	awaiting: Awaiting,
170
171	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
172
173	const Awaiting = enum(u31) {
174	nothing = std.math.maxInt(u31),
175	group = std.math.maxInt(u31) - 1,
176	/// An io_uring fd.
177	_,
178
179	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
180	return @fromBackingInt(@intCast(@backingInt(lhs) -% @backingInt(rhs)));
181	}
182
183	fn fromIoUringFd(fd: fd_t) Awaiting {
184	const awaiting: Awaiting = @fromBackingInt(@intCast(fd));
185	switch (awaiting) {
186	.nothing, .group => unreachable,
187	_ => return awaiting,
188	}
189	}
190
191	fn toIoUringFd(awaiting: Awaiting) fd_t {
192	switch (awaiting) {
193	.nothing, .group => unreachable,
194	_ => return @backingInt(awaiting),
195	}
196	}
197	};
198
199	fn changeAwaiting(
200	cancel_status: *CancelStatus,
201	old_awaiting: Awaiting,
202	new_awaiting: Awaiting,
203	) bool {
204	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
205	.requested = false,
206	.awaiting = new_awaiting.subWrap(old_awaiting),
207	}, .monotonic);
208	assert(old_cancel_status.awaiting == old_awaiting);
209	return old_cancel_status.requested;
210	}
211	};
212
213	const CancelProtection = packed struct {
214	user: Io.CancelProtection,
215	acknowledged: bool,
216
217	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
218
219	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
220	return @fromBackingInt(@intCast(@intFromBool(cancel_protection != unblocked)));
221	}
222
223	fn acknowledge(cancel_protection: *CancelProtection) void {
224	assert(!cancel_protection.acknowledged);
225	cancel_protection.acknowledged = true;
226	}
227
228	fn recancel(cancel_protection: *CancelProtection) void {
229	assert(cancel_protection.acknowledged);
230	cancel_protection.acknowledged = false;
231	}
232
233	test check {
234	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
235	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
236	.user = .unblocked,
237	.acknowledged = true,
238	}));
239	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
240	.user = .blocked,
241	.acknowledged = false,
242	}));
243	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
244	.user = .blocked,
245	.acknowledged = true,
246	}));
247	}
248	};
249
250	const finished: ?*Fiber = @ptrFromInt(@alignOf(Fiber));
251
252	const max_result_align: Alignment = .@"16";
253	const max_result_size = max_result_align.forward(512);
254	/// This includes any stack realignments that need to happen, and also the
255	/// initial frame return address slot and argument frame, depending on target.
256	const min_stack_size = 60 * 1024 * 1024;
257	const max_context_align: Alignment = .@"16";
258	const max_context_size = max_context_align.forward(1024);
259	const max_closure_size: usize = @sizeOf(AsyncClosure);
260	const max_closure_align: Alignment = .of(AsyncClosure);
261	const allocation_size = std.mem.alignForward(
262	usize,
263	max_closure_align.max(max_context_align).forward(
264	max_result_align.forward(@sizeOf(Fiber)) + max_result_size + min_stack_size,
265	) + max_closure_size + max_context_size,
266	std.heap.page_size_max,
267	);
268	comptime {
269	assert(max_result_align.compare(.gte, .of(Completion)));
270	assert(max_result_size >= @sizeOf(Completion));
271	}
272
273	fn create(ev: *Evented) error{OutOfMemory}!*Fiber {
274	const thread: *Thread = .current();
275	if (@atomicRmw(?*Fiber, &thread.free_queue, .Xchg, finished, .acquire)) |free_fiber| {
276	assert(free_fiber != finished);
277	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
278	return free_fiber;
279	}
280	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
281	for (0..@min(max_steal_free_search, active_threads)) |_| {
282	defer thread.steal_free_search_index += 1;
283	if (thread.steal_free_search_index == active_threads) thread.steal_free_search_index = 0;
284	const steal_free_search_thread =
285	&ev.threads.allocated[0..active_threads][thread.steal_free_search_index];
286	if (steal_free_search_thread == thread) continue;
287	const free_fiber =
288	@atomicLoad(?*Fiber, &steal_free_search_thread.free_queue, .monotonic) orelse continue;
289	if (free_fiber == finished) continue;
290	if (@cmpxchgWeak(
291	?*Fiber,
292	&steal_free_search_thread.free_queue,
293	free_fiber,
294	null,
295	.acquire,
296	.monotonic,
297	)) |_| continue;
298	@atomicStore(?*Fiber, &thread.free_queue, free_fiber.status.free_next, .release);
299	return free_fiber;
300	}
301	@atomicStore(?*Fiber, &thread.free_queue, null, .monotonic);
302	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
303	}
304
305	fn destroy(fiber: *Fiber) void {
306	const thread: *Thread = .current();
307	assert(fiber.status.queue_next == null);
308	fiber.status = .{ .free_next = @atomicLoad(?*Fiber, &thread.free_queue, .acquire) };
309	while (true) fiber.status.free_next = @cmpxchgWeak(
310	?*Fiber,
311	&thread.free_queue,
312	fiber.status.free_next,
313	fiber,
314	.acq_rel,
315	.acquire,
316	) orelse break;
317	}
318
319	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
320	return @as([*]align(@alignOf(Fiber)) u8, @ptrCast(f))[0..allocation_size];
321	}
322
323	fn allocatedEnd(f: *Fiber) [*]u8 {
324	const allocated_slice = f.allocatedSlice();
325	return allocated_slice[allocated_slice.len..].ptr;
326	}
327
328	fn resultPointer(f: *Fiber, comptime Result: type) *Result {
329	return @ptrCast(@alignCast(f.resultBytes(.of(Result))));
330	}
331
332	fn resultBytes(f: *Fiber, alignment: Alignment) [*]u8 {
333	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
334	}
335
336	const Queue = struct { head: *Fiber, tail: *Fiber };
337
338	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
339	/// alignment) so that those two bits can be used in a `packed struct`.
340	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
341	null = 0,
342	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
343	_,
344
345	const Split = packed struct(usize) { low: u2, high: PackedPtr };
346	fn pack(ptr: ?*Fiber) PackedPtr {
347	const split: Split = @bitCast(@intFromPtr(ptr));
348	assert(split.low == 0);
349	return split.high;
350	}
351	fn unpack(ptr: PackedPtr) ?*Fiber {
352	const split: Split = .{ .low = 0, .high = ptr };
353	return @ptrFromInt(@as(usize, @bitCast(split)));
354	}
355	};
356
357	fn requestCancel(fiber: *Fiber, ev: *Evented) void {
358	const cancel_status = @atomicRmw(
359	Fiber.CancelStatus,
360	&fiber.cancel_status,
361	.Or,
362	.{ .requested = true, .awaiting = @fromBackingInt(@intCast(0)) },
363	.acquire,
364	);
365	assert(!cancel_status.requested);
366	switch (cancel_status.awaiting) {
367	.nothing => {},
368	.group => {
369	// The awaiter received a cancelation request while awaiting a group,
370	// so propagate the cancelation to the group.
371	if (fiber.status.awaiting_group.cancel(ev, null)) {
372	fiber.status = .{ .queue_next = null };
373	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
374	}
375	},
376	_ => |awaiting| {
377	const awaiting_io_uring_fd = awaiting.toIoUringFd();
378	const thread: *Thread = .current();
379	thread.enqueue().* = if (thread.io_uring.fd == awaiting_io_uring_fd) .{
380	.opcode = .ASYNC_CANCEL,
381	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
382	.ioprio = 0,
383	.fd = 0,
384	.off = 0,
385	.addr = @intFromPtr(fiber),
386	.len = 0,
387	.rw_flags = 0,
388	.user_data = @backingInt(Completion.Userdata.wakeup),
389	.buf_index = 0,
390	.personality = 0,
391	.splice_fd_in = 0,
392	.addr3 = 0,
393	.resv = 0,
394	} else .{
395	.opcode = .MSG_RING,
396	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
397	.ioprio = 0,
398	.fd = awaiting_io_uring_fd,
399	.off = @intFromPtr(fiber) | 0b01,
400	.addr = @backingInt(linux.IORING_MSG_RING_COMMAND.DATA),
401	.len = 0,
402	.rw_flags = 0,
403	.user_data = @backingInt(Completion.Userdata.cleanup),
404	.buf_index = 0,
405	.personality = 0,
406	.splice_fd_in = 0,
407	.addr3 = 0,
408	.resv = 0,
409	};
410	},
411	}
412	}
413	};
414
415	const CancelRegion = struct {
416	fiber: *Fiber,
417	status: Fiber.CancelStatus,
418	fn init() CancelRegion {
419	const fiber = Thread.current().currentFiber();
420	return .{
421	.fiber = fiber,
422	.status = .{
423	.requested = fiber.cancel_protection.check() == .unblocked,
424	.awaiting = .nothing,
425	},
426	};
427	}
428	fn initBlocked() CancelRegion {
429	return .{
430	.fiber = Thread.current().currentFiber(),
431	.status = .{ .requested = false, .awaiting = .nothing },
432	};
433	}
434	fn deinit(cancel_region: *CancelRegion) void {
435	if (cancel_region.status.requested) {
436	@branchHint(.likely);
437	_ = cancel_region.fiber.cancel_status.changeAwaiting(
438	cancel_region.status.awaiting,
439	.nothing,
440	);
441	}
442	cancel_region.* = undefined;
443	}
444	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
445	if (!cancel_region.status.requested) {
446	@branchHint(.unlikely);
447	return;
448	}
449	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
450	if (cancel_region.fiber.cancel_status.changeAwaiting(
451	cancel_region.status.awaiting,
452	status.awaiting,
453	)) {
454	@branchHint(.unlikely);
455	cancel_region.fiber.cancel_protection.acknowledge();
456	cancel_region.status = .unrequested;
457	return error.Canceled;
458	}
459	cancel_region.status = status;
460	}
461	fn awaitIoUring(cancel_region: *CancelRegion) Io.Cancelable!*Thread {
462	const thread: *Thread = .current();
463	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
464	return thread;
465	}
466	fn completion(cancel_region: *const CancelRegion) Completion {
467	return cancel_region.fiber.resultPointer(Completion).*;
468	}
469	fn errno(cancel_region: *const CancelRegion) linux.E {
470	return cancel_region.completion().errno();
471	}
472
473	const Sync = struct {
474	cancel_region: CancelRegion,
475	fn init(ev: *Evented) Io.Cancelable!Sync {
476	if (ev.sync_limit) |*sync_limit| try sync_limit.wait(ev.io());
477	return .{ .cancel_region = .init() };
478	}
479	fn initBlocked(ev: *Evented) Sync {
480	if (ev.sync_limit) |*sync_limit| sync_limit.waitUncancelable(ev.io());
481	return .{ .cancel_region = .initBlocked() };
482	}
483	fn deinit(sync: *Sync, ev: *Evented) void {
484	sync.cancel_region.deinit();
485	if (ev.sync_limit) |*sync_limit| sync_limit.post(ev.io());
486	}
487
488	const Maybe = union(enum) {
489	cancel_region: CancelRegion,
490	sync: Sync,
491
492	fn deinit(maybe: *Maybe, ev: *Evented) void {
493	switch (maybe.*) {
494	.cancel_region => |*cancel_region| cancel_region.deinit(),
495	.sync => |*sync| sync.deinit(ev),
496	}
497	}
498
499	fn enterSync(maybe: *Maybe, ev: *Evented) Io.Cancelable!*Sync {
500	switch (maybe.*) {
501	.cancel_region => |cancel_region| {
502	if (ev.sync_limit) |*sync_limit| try sync_limit.wait(ev.io());
503	maybe.* = .{ .sync = .{ .cancel_region = cancel_region } };
504	},
505	.sync => {},
506	}
507	return &maybe.sync;
508	}
509
510	fn leaveSync(maybe: *Maybe, ev: *Evented) void {
511	switch (maybe.*) {
512	.cancel_region => {},
513	.sync => |sync| {
514	if (ev.sync_limit) |*sync_limit| sync_limit.post(ev.io());
515	maybe.* = .{ .cancel_region = sync.cancel_region };
516	},
517	}
518	}
519
520	fn cancelRegion(maybe: *Maybe) *CancelRegion {
521	return switch (maybe.*) {
522	.cancel_region => |*cancel_region| cancel_region,
523	.sync => |*sync| &sync.cancel_region,
524	};
525	}
526	};
527	};
528	};
529
530	const CachedFd = struct {
531	once: Once,
532
533	const Once = enum(fd_t) {
534	uninitialized = -1,
535	initializing = -2,
536	/// fd
537	_,
538
539	fn fromFd(fd: fd_t) Once {
540	return @fromBackingInt(@intCast(@as(u31, @intCast(fd))));
541	}
542
543	fn toFd(once: Once) fd_t {
544	return @as(u31, @intCast(@backingInt(once)));
545	}
546	};
547
548	const init: CachedFd = .{ .once = .uninitialized };
549
550	fn close(cached_fd: *CachedFd) void {
551	switch (cached_fd.once) {
552	.uninitialized => {},
553	.initializing => unreachable,
554	_ => |fd| {
555	assert(@backingInt(fd) >= 0);
556	_ = linux.close(@backingInt(fd));
557	cached_fd.* = .init;
558	},
559	}
560	}
561
562	fn open(
563	cached_fd: *CachedFd,
564	ev: *Evented,
565	cancel_region: *CancelRegion,
566	path: [*:0]const u8,
567	flags: linux.O,
568	) File.OpenError!fd_t {
569	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
570	while (true) {
571	switch (once) {
572	.uninitialized => {},
573	.initializing => try futexWait(
574	ev,
575	@ptrCast(&cached_fd.once),
576	@bitCast(@backingInt(once)),
577	.none,
578	),
579	_ => |fd| {
580	@branchHint(.likely);
581	return fd.toFd();
582	},
583	}
584	once = @cmpxchgWeak(
585	Once,
586	&cached_fd.once,
587	.uninitialized,
588	.initializing,
589	.monotonic,
590	.monotonic,
591	) orelse {
592	errdefer {
593	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
594	futexWake(ev, @ptrCast(&cached_fd.once), 1);
595	}
596	const fd = ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0) catch |err| switch (err) {
597	error.OperationUnsupported => return error.Unexpected, // TMPFILE unset.
598	else => |e| return e,
599	};
600	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
601	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
602	return fd;
603	};
604	}
605	}
606	};
607
608	pub fn allocator(ev: *Evented) std.mem.Allocator {
609	return if (ev.backing_allocator_needs_mutex) .{
610	.ptr = ev,
611	.vtable = &.{
612	.alloc = alloc,
613	.resize = resize,
614	.remap = remap,
615	.free = free,
616	},
617	} else ev.backing_allocator;
618	}
619
620	fn alloc(userdata: *anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[*]u8 {
621	const ev: *Evented = @ptrCast(@alignCast(userdata));
622	const ev_io = ev.io();
623	ev.backing_allocator_mutex.lockUncancelable(ev_io);
624	defer ev.backing_allocator_mutex.unlock(ev_io);
625	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
626	}
627
628	fn resize(
629	userdata: *anyopaque,
630	memory: []u8,
631	alignment: std.mem.Alignment,
632	new_len: usize,
633	ret_addr: usize,
634	) bool {
635	const ev: *Evented = @ptrCast(@alignCast(userdata));
636	const ev_io = ev.io();
637	ev.backing_allocator_mutex.lockUncancelable(ev_io);
638	defer ev.backing_allocator_mutex.unlock(ev_io);
639	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
640	}
641
642	fn remap(
643	userdata: *anyopaque,
644	memory: []u8,
645	alignment: Alignment,
646	new_len: usize,
647	ret_addr: usize,
648	) ?[*]u8 {
649	const ev: *Evented = @ptrCast(@alignCast(userdata));
650	const ev_io = ev.io();
651	ev.backing_allocator_mutex.lockUncancelable(ev_io);
652	defer ev.backing_allocator_mutex.unlock(ev_io);
653	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
654	}
655
656	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
657	const ev: *Evented = @ptrCast(@alignCast(userdata));
658	const ev_io = ev.io();
659	ev.backing_allocator_mutex.lockUncancelable(ev_io);
660	defer ev.backing_allocator_mutex.unlock(ev_io);
661	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
662	}
663
664	pub fn io(ev: *Evented) Io {
665	return .{
666	.userdata = ev,
667	.vtable = &.{
668	.crashHandler = crashHandler,
669
670	.async = async,
671	.concurrent = concurrent,
672	.await = await,
673	.cancel = cancel,
674
675	.groupAsync = groupAsync,
676	.groupConcurrent = groupConcurrent,
677	.groupAwait = groupAwait,
678	.groupCancel = groupCancel,
679
680	.recancel = recancel,
681	.swapCancelProtection = swapCancelProtection,
682	.checkCancel = checkCancel,
683
684	.futexWait = futexWait,
685	.futexWaitUncancelable = futexWaitUncancelable,
686	.futexWake = futexWake,
687
688	.operate = operate,
689	.batchAwaitAsync = batchAwaitAsync,
690	.batchAwaitConcurrent = batchAwaitConcurrent,
691	.batchCancel = batchCancel,
692
693	.dirCreateDir = dirCreateDir,
694	.dirCreateDirPath = dirCreateDirPath,
695	.dirCreateDirPathOpen = dirCreateDirPathOpen,
696	.dirOpenDir = dirOpenDir,
697	.dirStat = dirStat,
698	.dirStatFile = dirStatFile,
699	.dirAccess = dirAccess,
700	.dirCreateFile = dirCreateFile,
701	.dirCreateFileAtomic = dirCreateFileAtomic,
702	.dirOpenFile = dirOpenFile,
703	.dirClose = dirClose,
704	.dirRead = dirRead,
705	.dirRealPath = dirRealPath,
706	.dirRealPathFile = dirRealPathFile,
707	.dirDeleteFile = dirDeleteFile,
708	.dirDeleteDir = dirDeleteDir,
709	.dirRename = dirRename,
710	.dirRenamePreserve = dirRenamePreserve,
711	.dirSymLink = dirSymLink,
712	.dirReadLink = dirReadLink,
713	.dirSetOwner = dirSetOwner,
714	.dirSetFileOwner = dirSetFileOwner,
715	.dirSetPermissions = dirSetPermissions,
716	.dirSetFilePermissions = dirSetFilePermissions,
717	.dirSetTimestamps = dirSetTimestamps,
718	.dirHardLink = dirHardLink,
719
720	.fileStat = fileStat,
721	.fileLength = fileLength,
722	.fileClose = fileClose,
723	.fileWritePositional = fileWritePositional,
724	.fileWriteFileStreaming = fileWriteFileStreaming,
725	.fileWriteFilePositional = fileWriteFilePositional,
726	.fileReadPositional = fileReadPositional,
727	.fileSeekBy = fileSeekBy,
728	.fileSeekTo = fileSeekTo,
729	.fileSync = fileSync,
730	.fileIsTty = fileIsTty,
731	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
732	.fileSupportsAnsiEscapeCodes = fileIsTty,
733	.fileSetLength = fileSetLength,
734	.fileSetOwner = fileSetOwner,
735	.fileSetPermissions = fileSetPermissions,
736	.fileSetTimestamps = fileSetTimestamps,
737	.fileLock = fileLock,
738	.fileTryLock = fileTryLock,
739	.fileUnlock = fileUnlock,
740	.fileDowngradeLock = fileDowngradeLock,
741	.fileRealPath = fileRealPath,
742	.fileHardLink = fileHardLink,
743
744	.fileMemoryMapCreate = fileMemoryMapCreate,
745	.fileMemoryMapDestroy = fileMemoryMapDestroy,
746	.fileMemoryMapSetLength = fileMemoryMapSetLength,
747	.fileMemoryMapRead = fileMemoryMapRead,
748	.fileMemoryMapWrite = fileMemoryMapWrite,
749
750	.processExecutableOpen = processExecutableOpen,
751	.processExecutablePath = processExecutablePath,
752	.lockStderr = lockStderr,
753	.tryLockStderr = tryLockStderr,
754	.unlockStderr = unlockStderr,
755	.processCurrentPath = processCurrentPath,
756	.processSetCurrentDir = processSetCurrentDir,
757	.processSetCurrentPath = processSetCurrentPath,
758	.processReplace = processReplace,
759	.processReplacePath = processReplacePath,
760	.processSpawn = processSpawn,
761	.processSpawnPath = processSpawnPath,
762	.childWait = childWait,
763	.childKill = childKill,
764
765	.progressParentFile = progressParentFile,
766
767	.now = now,
768	.clockResolution = clockResolution,
769	.sleep = sleep,
770
771	.random = random,
772	.randomSecure = randomSecure,
773
774	.netListenIp = netListenIpUnavailable,
775	.netAccept = netAcceptUnavailable,
776	.netBindIp = netBindIp,
777	.netConnectIp = netConnectIpUnavailable,
778	.netListenUnix = netListenUnixUnavailable,
779	.netConnectUnix = netConnectUnixUnavailable,
780	.netSocketCreatePair = netSocketCreatePairUnavailable,
781	.netWriteFile = netWriteFileUnavailable,
782	.netClose = netClose,
783	.netShutdown = netShutdown,
784	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
785	.netInterfaceName = netInterfaceNameUnavailable,
786	.netLookup = netLookupUnavailable,
787	},
788	};
789	}
790
791	pub const InitOptions = struct {
792	backing_allocator_needs_mutex: bool = true,
793
794	/// Maximum thread pool size (excluding the main thread).
795	/// Defaults to one less than the number of logical CPU cores.
796	thread_limit: ?usize = null,
797	/// Maximum number of threads that may perform synchronous syscalls.
798	sync_limit: Io.Limit = .unlimited,
799
800	log2_ring_entries: u4 = 3,
801
802	/// Affects the following operations:
803	/// * `processExecutablePath` on OpenBSD and Haiku.
804	argv0: Argv0 = .empty,
805	/// Affects the following operations:
806	/// * `fileIsTty`
807	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
808	environ: process.Environ = .empty,
809	};
810
811	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
812	const threads_size = @sizeOf(Thread) * if (options.thread_limit) |thread_limit|
813	1 + thread_limit
814	else
815	@max(std.Thread.getCpuCount() catch 1, 1);
816	const idle_stack_end_offset =
817	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.pageSize());
818	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
819	errdefer backing_allocator.free(allocated_slice);
820	ev.* = .{
821	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
822	.backing_allocator_mutex = .init,
823	.backing_allocator = backing_allocator,
824	.main_fiber_buffer = undefined,
825	.log2_ring_entries = options.log2_ring_entries,
826	.threads = .{
827	.allocated = @ptrCast(allocated_slice[0..threads_size]),
828	.reserved = 1,
829	.active = 1,
830	},
831	.sync_limit = if (options.sync_limit.toInt()) |sync_limit| .{ .permits = sync_limit } else null,
832
833	.stderr_writer_initialized = false,
834	.stderr_mutex = .init,
835	.stderr_writer = .{
836	.io = ev.io(),
837	.interface = Io.File.Writer.initInterface(&.{}),
838	.file = .stderr(),
839	.mode = .streaming,
840	},
841	.stderr_mode = .no_color,
842
843	.environ_mutex = .init,
844	.environ_initialized = options.environ.block.isEmpty(),
845	.environ = .{ .process_environ = options.environ },
846
847	.null_fd = .init,
848	.random_fd = .init,
849
850	.csprng_mutex = .init,
851	.csprng = .uninitialized,
852	};
853	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
854	main_fiber.* = .{
855	.required_align = {},
856	.context = undefined,
857	.link = .{ .awaiter = null },
858	.status = .{ .queue_next = null },
859	.cancel_status = .unrequested,
860	.cancel_protection = .unblocked,
861	.name = if (tracy.enable) "main task",
862	};
863	const main_thread = &ev.threads.allocated[0];
864	Thread.self = main_thread;
865	main_thread.* = .{
866	.required_align = {},
867	.thread = undefined,
868	.idle_context = switch (builtin.cpu.arch) {
869	.aarch64 => .{
870	.sp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
871	.fp = @intFromPtr(ev),
872	.pc = @intFromPtr(&mainIdleEntry),
873	},
874	.riscv64 => .{
875	.sp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
876	.fp = @intFromPtr(ev),
877	.pc = @intFromPtr(&mainIdleEntry),
878	},
879	.x86_64 => .{
880	.rsp = @intFromPtr(allocated_slice[idle_stack_end_offset..].ptr),
881	.rbp = @intFromPtr(ev),
882	.rip = @intFromPtr(&mainIdleEntry),
883	},
884	else => @compileError("unimplemented architecture"),
885	},
886	.current_context = &main_fiber.context,
887	.ready_queue = null,
888	.free_queue = null,
889	.io_uring = try .init(
890	@as(u16, 1) << ev.log2_ring_entries,
891	linux.IORING_SETUP_COOP_TASKRUN | linux.IORING_SETUP_SINGLE_ISSUER,
892	),
893	.idle_search_index = 1,
894	.steal_ready_search_index = 1,
895	.steal_free_search_index = 1,
896	.name_arena = .{},
897	.csprng = .uninitialized,
898	};
899	errdefer main_thread.io_uring.deinit();
900	if (tracy.enable) tracy.fiberEnter(main_fiber.name);
901	}
902
903	pub fn deinit(ev: *Evented) void {
904	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
905	assert(Thread.current().currentFiber() == main_fiber);
906	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
907	for (ev.threads.allocated[0..active_threads]) |*thread| {
908	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
909	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
910	}
911	ev.yield(null, .exit);
912	ev.null_fd.close();
913	ev.random_fd.close();
914	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
915	const idle_stack_end_offset = std.mem.alignForward(
916	usize,
917	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
918	std.heap.pageSize(),
919	);
920	for (ev.threads.allocated[1..active_threads]) |*thread| thread.thread.join();
921	for (ev.threads.allocated[0..active_threads]) |*thread| thread.deinit(ev.backing_allocator);
922	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
923	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
924	ev.* = undefined;
925	}
926
927	fn findReadyFiber(ev: *Evented, thread: *Thread) ?*Fiber {
928	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) |ready_fiber| {
929	assert(ready_fiber != Fiber.finished);
930	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
931	ready_fiber.status.queue_next = null;
932	return ready_fiber;
933	}
934	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
935	for (0..@min(max_steal_ready_search, active_threads)) |_| {
936	defer thread.steal_ready_search_index += 1;
937	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
938	const steal_ready_search_thread =
939	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
940	if (steal_ready_search_thread == thread) continue;
941	const ready_fiber =
942	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .monotonic) orelse continue;
943	if (ready_fiber == Fiber.finished) continue;
944	if (@cmpxchgWeak(
945	?*Fiber,
946	&steal_ready_search_thread.ready_queue,
947	ready_fiber,
948	null,
949	.acquire,
950	.monotonic,
951	)) |_| continue;
952	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
953	ready_fiber.status.queue_next = null;
954	return ready_fiber;
955	}
956	// couldn't find anything to do, so we are now open for business
957	@atomicStore(?*Fiber, &thread.ready_queue, null, .monotonic);
958	return null;
959	}
960
961	fn yield(ev: *Evented, maybe_ready_fiber: ?*Fiber, pending_task: SwitchMessage.PendingTask) void {
962	const thread: *Thread = .current();
963	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) |ready_fiber|
964	&ready_fiber.context
965	else
966	&thread.idle_context;
967	const message: SwitchMessage = .{
968	.contexts = .{
969	.old = thread.current_context,
970	.new = ready_context,
971	},
972	.pending_task = pending_task,
973	};
974	contextSwitch(&message).handle(ev);
975	}
976
977	fn schedule(ev: *Evented, thread: *Thread, ready_queue: Fiber.Queue) bool {
978	// shared fields of previous `Thread` must be initialized before later ones are marked as active
979	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
980	for (0..@min(max_idle_search, new_thread_index)) |_| {
981	defer thread.idle_search_index += 1;
982	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
983	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
984	if (idle_search_thread == thread) continue;
985	if (@cmpxchgWeak(
986	?*Fiber,
987	&idle_search_thread.ready_queue,
988	null,
989	ready_queue.head,
990	.release,
991	.monotonic,
992	)) |_| continue;
993	thread.enqueue().* = .{
994	.opcode = .MSG_RING,
995	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
996	.ioprio = 0,
997	.fd = idle_search_thread.io_uring.fd,
998	.off = @backingInt(Completion.Userdata.wakeup),
999	.addr = @backingInt(linux.IORING_MSG_RING_COMMAND.DATA),
1000	.len = 0,
1001	.rw_flags = 0,
1002	.user_data = @backingInt(Completion.Userdata.wakeup),
1003	.buf_index = 0,
1004	.personality = 0,
1005	.splice_fd_in = 0,
1006	.addr3 = 0,
1007	.resv = 0,
1008	};
1009	return true;
1010	}
1011	spawn_thread: {
1012	// previous failed reservations must have completed before retrying
1013	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
1014	u32,
1015	&ev.threads.reserved,
1016	new_thread_index,
1017	new_thread_index + 1,
1018	.acquire,
1019	.monotonic,
1020	) != null) break :spawn_thread;
1021	const new_thread = &ev.threads.allocated[new_thread_index];
1022	const next_thread_index = new_thread_index + 1;
1023	var params = std.mem.zeroInit(linux.io_uring_params, .{
1024	.flags = linux.IORING_SETUP_ATTACH_WQ |
1025	linux.IORING_SETUP_R_DISABLED |
1026	linux.IORING_SETUP_COOP_TASKRUN |
1027	linux.IORING_SETUP_SINGLE_ISSUER,
1028	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
1029	});
1030	new_thread.* = .{
1031	.required_align = {},
1032	.thread = undefined,
1033	.idle_context = undefined,
1034	.current_context = &new_thread.idle_context,
1035	.ready_queue = ready_queue.head,
1036	.free_queue = null,
1037	.io_uring = IoUring.init_params(@as(u16, 1) << ev.log2_ring_entries, &params) catch |err| {
1038	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1039	// no more access to `thread` after giving up reservation
1040	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
1041	@errorName(err),
1042	});
1043	break :spawn_thread;
1044	},
1045	.idle_search_index = 0,
1046	.steal_ready_search_index = 0,
1047	.steal_free_search_index = 0,
1048	.name_arena = .{},
1049	.csprng = .uninitialized,
1050	};
1051	new_thread.thread = std.Thread.spawn(.{
1052	.stack_size = idle_stack_size,
1053	.allocator = ev.allocator(),
1054	}, threadEntry, .{ ev, new_thread_index }) catch |err| {
1055	new_thread.io_uring.deinit();
1056	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
1057	// no more access to `thread` after giving up reservation
1058	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
1059	break :spawn_thread;
1060	};
1061	// shared fields of `Thread` must be initialized before being marked active
1062	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
1063	return false;
1064	}
1065	// nobody wanted it, so just queue it on ourselves
1066	while (true) ready_queue.tail.status.queue_next = @cmpxchgWeak(
1067	?*Fiber,
1068	&thread.ready_queue,
1069	ready_queue.tail.status.queue_next,
1070	ready_queue.head,
1071	.acq_rel,
1072	.acquire,
1073	) orelse break;
1074	return false;
1075	}
1076
1077	fn threadEntry(ev: *Evented, index: u32) void {
1078	const thread: *Thread = &ev.threads.allocated[index];
1079	Thread.self = thread;
1080	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
1081	.SUCCESS => ev.idle(thread),
1082	else => |err| @panic(@tagName(err)),
1083	}
1084	}
1085
1086	const Completion = struct {
1087	result: i32,
1088	flags: u32,
1089
1090	const Userdata = enum(usize) {
1091	unused,
1092	wakeup,
1093	futex_wake,
1094	close,
1095	cleanup,
1096	exit,
1097	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
1098	/// If bits 0 and 1 are 0, a `*Fiber`.
1099	_,
1100	};
1101
1102	fn errno(completion: Completion) linux.E {
1103	return linux.errno(@bitCast(@as(isize, completion.result)));
1104	}
1105	};
1106
1107	fn mainIdleEntry() callconv(.naked) void {
1108	switch (builtin.cpu.arch) {
1109	.aarch64 => asm volatile (
1110	\\ mov x0, fp
1111	\\ mov fp, #0
1112	\\ b %[mainIdle]
1113	:
1114	: [mainIdle] "X" (&mainIdle),
1115	),
1116	.riscv64 => asm volatile (
1117	\\ mv a0, fp
1118	\\ mv fp, zero
1119	\\ tail %[mainIdle]@plt
1120	:
1121	: [mainIdle] "X" (&mainIdle),
1122	),
1123	.x86_64 => asm volatile (
1124	\\ movq %%rbp, %%rdi
1125	\\ xor %%ebp, %%ebp
1126	\\ jmp %[mainIdle:P]
1127	:
1128	: [mainIdle] "X" (&mainIdle),
1129	),
1130	else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1131	}
1132	}
1133
1134	fn mainIdle(
1135	ev: *Evented,
1136	contexts: *const Io.fiber.Switch,
1137	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Io.fiber.Context)))) noreturn {
1138	const message: *const SwitchMessage = @fieldParentPtr("contexts", contexts);
1139	message.handle(ev);
1140	ev.idle(&ev.threads.allocated[0]);
1141	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
1142	unreachable; // switched to dead fiber
1143	}
1144
1145	fn idle(ev: *Evented, thread: *Thread) void {
1146	var maybe_ready_fiber: ?*Fiber = null;
1147	while (true) {
1148	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) |ready_fiber| {
1149	ev.yield(ready_fiber, .nothing);
1150	maybe_ready_fiber = null;
1151	}
1152	_ = thread.io_uring.submit_and_wait(1) catch |err| switch (err) {
1153	error.SignalInterrupt => {},
1154	else => |e| @panic(@errorName(e)),
1155	};
1156	var maybe_ready_queue: ?Fiber.Queue = null;
1157	while (true) {
1158	var cqes_buffer: [1 << 8]linux.io_uring_cqe = undefined;
1159	const cqes = cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch |err| switch (err) {
1160	error.SignalInterrupt => 0,
1161	else => |e| @panic(@errorName(e)),
1162	}];
1163	if (cqes.len == 0) break;
1164	for (cqes) |cqe| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
1165	Completion.Userdata,
1166	@fromBackingInt(@intCast(cqe.user_data)),
1167	)) {
1168	.unused => unreachable, // bad submission queued?
1169	.wakeup => {},
1170	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1171	.SUCCESS => recoverableOsBugDetected(), // success is skipped
1172	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
1173	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.Userdata.futex_wake` is not cancelable
1174	.FAULT => {}, // pointer became invalid while doing the wake
1175	else => recoverableOsBugDetected(), // deadlock due to operating system bug
1176	},
1177	.close => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1178	.BADF => recoverableOsBugDetected(), // Always a race condition.
1179	.INTR => {}, // This is still a success. See https://github.com/ziglang/zig/issues/2425
1180	else => {},
1181	},
1182	.cleanup => @panic("failed to notify other threads that we are exiting"),
1183	.exit => {
1184	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
1185	return;
1186	},
1187	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
1188	0b00 => {
1189	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1190	ready_fiber.resultPointer(Completion).* = .{
1191	.result = cqe.res,
1192	.flags = cqe.flags,
1193	};
1194	break :ready_fiber ready_fiber;
1195	},
1196	0b01 => {
1197	thread.enqueue().* = .{
1198	.opcode = .ASYNC_CANCEL,
1199	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1200	.ioprio = 0,
1201	.fd = 0,
1202	.off = 0,
1203	.addr = cqe.user_data & ~@as(usize, 0b11),
1204	.len = 0,
1205	.rw_flags = 0,
1206	.user_data = @backingInt(Completion.Userdata.wakeup),
1207	.buf_index = 0,
1208	.personality = 0,
1209	.splice_fd_in = 0,
1210	.addr3 = 0,
1211	.resv = 0,
1212	};
1213	break :ready_fiber null;
1214	},
1215	0b10 => {
1216	const batch_userdata: *Io.Operation.Storage.Pending.Userdata =
1217	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1218	const batch: *Io.Batch = @ptrFromInt(batch_userdata[0]);
1219	var next: usize = 0b00;
1220	batch_userdata[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
1221	while (true) {
1222	next = @cmpxchgWeak(
1223	usize,
1224	@as(*usize, @ptrCast(&batch.userdata)),
1225	next,
1226	cqe.user_data,
1227	.release,
1228	.acquire,
1229	) orelse break;
1230	batch_userdata[0] = next;
1231	}
1232	break :ready_fiber switch (@as(u2, @truncate(next))) {
1233	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
1234	0b10, 0b11 => null,
1235	};
1236	},
1237	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
1238	.SUCCESS => unreachable, // no event count specified
1239	.TIME => {
1240	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
1241	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
1242	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
1243	else => unreachable, // timeout completed multiple times
1244	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
1245	0b10 => null,
1246	};
1247	},
1248	.CANCELED => null, // user data may have been invalidated
1249	else => |err| unexpectedErrno(err) catch null,
1250	},
1251	})) |ready_fiber| {
1252	assert(ready_fiber.status.queue_next == null);
1253	if (maybe_ready_fiber == null) {
1254	maybe_ready_fiber = ready_fiber;
1255	} else if (maybe_ready_queue) |*ready_queue| {
1256	ready_queue.tail.status.queue_next = ready_fiber;
1257	ready_queue.tail = ready_fiber;
1258	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
1259	},
1260	};
1261	}
1262	if (maybe_ready_queue) |ready_queue| _ = ev.schedule(thread, ready_queue);
1263	}
1264	}
1265
1266	const SwitchMessage = struct {
1267	contexts: Io.fiber.Switch,
1268	pending_task: PendingTask,
1269
1270	const PendingTask = union(enum) {
1271	nothing,
1272	reschedule,
1273	await: *Fiber,
1274	group_await: Group,
1275	group_cancel: Group,
1276	batch_await: *Io.Batch,
1277	destroy,
1278	exit,
1279	};
1280
1281	fn handle(message: *const SwitchMessage, ev: *Evented) void {
1282	const thread: *Thread = .current();
1283	thread.current_context = message.contexts.new;
1284	if (tracy.enable) {
1285	if (message.contexts.new != &thread.idle_context) {
1286	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.new));
1287	tracy.fiberEnter(fiber.name);
1288	} else tracy.fiberLeave();
1289	}
1290	switch (message.pending_task) {
1291	.nothing => {},
1292	.reschedule => if (message.contexts.old != &thread.idle_context) {
1293	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1294	assert(fiber.status.queue_next == null);
1295	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1296	},
1297	.await => |awaiting| {
1298	const awaiter: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1299	assert(awaiter.status.queue_next == null);
1300	if (@atomicRmw(?*Fiber, &awaiting.link.awaiter, .Xchg, awaiter, .acq_rel) ==
1301	Fiber.finished) _ = ev.schedule(thread, .{ .head = awaiter, .tail = awaiter });
1302	},
1303	.group_await => |group| {
1304	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1305	if (group.await(ev, fiber))
1306	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1307	},
1308	.group_cancel => |group| {
1309	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1310	if (group.cancel(ev, fiber))
1311	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1312	},
1313	.batch_await => |batch| {
1314	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1315	if (@cmpxchgStrong(
1316	?*anyopaque,
1317	&batch.userdata,
1318	null,
1319	fiber,
1320	.release,
1321	.monotonic,
1322	)) |head| {
1323	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
1324	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
1325	}
1326	},
1327	.destroy => {
1328	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.old));
1329	fiber.destroy();
1330	},
1331	.exit => for (
1332	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
1333	) |*each_thread| {
1334	thread.enqueue().* = .{
1335	.opcode = .MSG_RING,
1336	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1337	.ioprio = 0,
1338	.fd = each_thread.io_uring.fd,
1339	.off = @backingInt(Completion.Userdata.exit),
1340	.addr = @backingInt(linux.IORING_MSG_RING_COMMAND.DATA),
1341	.len = 0,
1342	.rw_flags = 0,
1343	.user_data = @backingInt(Completion.Userdata.cleanup),
1344	.buf_index = 0,
1345	.personality = 0,
1346	.splice_fd_in = 0,
1347	.addr3 = 0,
1348	.resv = 0,
1349	};
1350	},
1351	}
1352	}
1353	};
1354
1355	inline fn contextSwitch(message: *const SwitchMessage) *const SwitchMessage {
1356	return @fieldParentPtr("contexts", Io.fiber.contextSwitch(&message.contexts));
1357	}
1358
1359	fn crashHandler(userdata: ?*anyopaque) void {
1360	const ev: *Evented = @ptrCast(@alignCast(userdata));
1361	_ = ev;
1362	const thread = Thread.self orelse std.process.abort();
1363	if (thread.current_context == &thread.idle_context) std.process.abort();
1364	const fiber = thread.currentFiber();
1365	@atomicStore(
1366	Fiber.CancelStatus,
1367	&fiber.cancel_status,
1368	.{ .requested = true, .awaiting = .nothing },
1369	.monotonic,
1370	);
1371	fiber.cancel_protection = .{ .user = .blocked, .acknowledged = true };
1372	}
1373
1374	const AsyncClosure = struct {
1375	evented: *Evented,
1376	fiber: *Fiber,
1377	start: *const fn (context: *const anyopaque, result: *anyopaque) void,
1378	result_align: Alignment,
1379
1380	fn fromFiber(fiber: *Fiber) *AsyncClosure {
1381	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
1382	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1383	) - @sizeOf(AsyncClosure));
1384	}
1385
1386	fn contextPointer(closure: *AsyncClosure) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
1387	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
1388	}
1389
1390	fn entry() callconv(.naked) void {
1391	switch (builtin.cpu.arch) {
1392	.aarch64 => asm volatile (
1393	\\ mov x0, sp
1394	\\ b %[call]
1395	:
1396	: [call] "X" (&call),
1397	),
1398	.riscv64 => asm volatile (
1399	\\ mv a0, sp
1400	\\ tail %[call]@plt
1401	:
1402	: [call] "X" (&call),
1403	),
1404	.x86_64 => asm volatile (
1405	\\ leaq 8(%%rsp), %%rdi
1406	\\ jmp %[call:P]
1407	:
1408	: [call] "X" (&call),
1409	),
1410	else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1411	}
1412	}
1413
1414	fn call(
1415	closure: *AsyncClosure,
1416	contexts: *const Io.fiber.Switch,
1417	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
1418	const message: *const SwitchMessage = @fieldParentPtr("contexts", contexts);
1419	const ev = closure.evented;
1420	const fiber = closure.fiber;
1421	message.handle(ev);
1422	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
1423	ev.yield(@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel), .nothing);
1424	unreachable; // switched to dead fiber
1425	}
1426	};
1427
1428	fn async(
1429	userdata: ?*anyopaque,
1430	result: []u8,
1431	result_alignment: Alignment,
1432	context: []const u8,
1433	context_alignment: Alignment,
1434	start: *const fn (context: *const anyopaque, result: *anyopaque) void,
1435	) ?*std.Io.AnyFuture {
1436	const ev: *Evented = @ptrCast(@alignCast(userdata));
1437	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
1438	start(context.ptr, result.ptr);
1439	return null;
1440	};
1441	}
1442
1443	fn concurrent(
1444	userdata: ?*anyopaque,
1445	result_len: usize,
1446	result_alignment: Alignment,
1447	context: []const u8,
1448	context_alignment: Alignment,
1449	start: *const fn (context: *const anyopaque, result: *anyopaque) void,
1450	) Io.ConcurrentError!*std.Io.AnyFuture {
1451	assert(result_alignment.compare(.lte, Fiber.max_result_align)); // TODO
1452	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1453	assert(result_len <= Fiber.max_result_size); // TODO
1454	assert(context.len <= Fiber.max_context_size); // TODO
1455
1456	const ev: *Evented = @ptrCast(@alignCast(userdata));
1457	const fiber = Fiber.create(ev) catch |err| switch (err) {
1458	error.OutOfMemory => return error.ConcurrencyUnavailable,
1459	};
1460
1461	const closure: *AsyncClosure = .fromFiber(fiber);
1462	fiber.* = .{
1463	.required_align = {},
1464	.context = switch (builtin.cpu.arch) {
1465	.aarch64 => .{
1466	.sp = @intFromPtr(closure),
1467	.fp = 0,
1468	.pc = @intFromPtr(&AsyncClosure.entry),
1469	},
1470	.riscv64 => .{
1471	.sp = @intFromPtr(closure),
1472	.fp = 0,
1473	.pc = @intFromPtr(&AsyncClosure.entry),
1474	},
1475	.x86_64 => .{
1476	.rsp = @intFromPtr(closure) - 8,
1477	.rbp = 0,
1478	.rip = @intFromPtr(&AsyncClosure.entry),
1479	},
1480	else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1481	},
1482	.link = .{ .awaiter = null },
1483	.status = .{ .queue_next = null },
1484	.cancel_status = .unrequested,
1485	.cancel_protection = .unblocked,
1486	.name = if (tracy.enable) name: {
1487	const thread: *Thread = .current();
1488	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
1489	defer thread.name_arena = name_arena.state;
1490	break :name std.fmt.allocPrintSentinel(
1491	name_arena.allocator(),
1492	"task {d}",
1493	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
1494	0,
1495	) catch return error.ConcurrencyUnavailable;
1496	},
1497	};
1498	closure.* = .{
1499	.evented = ev,
1500	.fiber = fiber,
1501	.start = start,
1502	.result_align = result_alignment,
1503	};
1504	@memcpy(closure.contextPointer(), context);
1505
1506	const thread: *Thread = .current();
1507	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
1508	return @ptrCast(fiber);
1509	}
1510
1511	fn await(
1512	userdata: ?*anyopaque,
1513	future: *std.Io.AnyFuture,
1514	result: []u8,
1515	result_alignment: Alignment,
1516	) void {
1517	const ev: *Evented = @ptrCast(@alignCast(userdata));
1518	const awaiting: *Fiber = @ptrCast(@alignCast(future));
1519	if (@atomicLoad(?*Fiber, &awaiting.link.awaiter, .acquire) != Fiber.finished)
1520	ev.yield(null, .{ .await = awaiting });
1521	@memcpy(result, awaiting.resultBytes(result_alignment));
1522	awaiting.destroy();
1523	}
1524
1525	fn cancel(
1526	userdata: ?*anyopaque,
1527	future: *std.Io.AnyFuture,
1528	result: []u8,
1529	result_alignment: Alignment,
1530	) void {
1531	const ev: *Evented = @ptrCast(@alignCast(userdata));
1532	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
1533	future_fiber.requestCancel(ev);
1534	await(ev, future, result, result_alignment);
1535	}
1536
1537	const Group = struct {
1538	ptr: *Io.Group,
1539
1540	const List = packed struct(usize) {
1541	cancel_requested: bool,
1542	awaiter_delayed: bool,
1543	fibers: Fiber.PackedPtr,
1544	};
1545	fn listPtr(group: Group) *List {
1546	return @ptrCast(&group.ptr.token);
1547	}
1548
1549	const Mutex = packed struct(u32) {
1550	locked: bool,
1551	contended: bool,
1552	shared2: u30,
1553	};
1554	fn mutexPtr(group: Group) *Mutex {
1555	return switch (comptime builtin.cpu.arch.endian()) {
1556	.little => @ptrCast(&group.ptr.state),
1557	.big => @ptrCast(@alignCast(
1558	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
1559	)),
1560	};
1561	}
1562
1563	const Awaiter = packed struct(usize) {
1564	locked: bool,
1565	contended: bool,
1566	awaiter: Fiber.PackedPtr,
1567	};
1568	fn awaiterPtr(group: Group) *Awaiter {
1569	return @ptrCast(&group.ptr.state);
1570	}
1571
1572	fn lock(group: Group, ev: *Evented) void {
1573	const mutex = group.mutexPtr();
1574	{
1575	const old_state = @atomicRmw(
1576	Mutex,
1577	mutex,
1578	.Or,
1579	.{ .locked = true, .contended = false, .shared2 = 0 },
1580	.acquire,
1581	);
1582	if (!old_state.locked) {
1583	@branchHint(.likely);
1584	return;
1585	}
1586	if (old_state.contended) {
1587	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1588	}
1589	}
1590	while (true) {
1591	var old_state = @atomicRmw(
1592	Mutex,
1593	mutex,
1594	.Or,
1595	.{ .locked = true, .contended = true, .shared2 = 0 },
1596	.acquire,
1597	);
1598	if (!old_state.locked) {
1599	@branchHint(.likely);
1600	return;
1601	}
1602	old_state.contended = true;
1603	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
1604	}
1605	}
1606
1607	fn unlock(group: Group, ev: *Evented) void {
1608	const mutex = group.mutexPtr();
1609	const old_state = @atomicRmw(
1610	Mutex,
1611	mutex,
1612	.And,
1613	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
1614	.release,
1615	);
1616	assert(old_state.locked);
1617	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
1618	}
1619
1620	fn addFiber(group: Group, ev: *Evented, fiber: *Fiber) void {
1621	group.lock(ev);
1622	defer group.unlock(ev);
1623	const list_ptr = group.listPtr();
1624	const list = @atomicLoad(List, list_ptr, .monotonic);
1625	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
1626	const old_head = list.fibers.unpack();
1627	if (old_head) |head| head.link.group.prev = fiber;
1628	fiber.link.group.next = old_head;
1629	@atomicStore(List, list_ptr, .{
1630	.cancel_requested = list.cancel_requested,
1631	.awaiter_delayed = list.awaiter_delayed,
1632	.fibers = .pack(fiber),
1633	}, .monotonic);
1634	}
1635
1636	fn removeFiber(group: Group, ev: *Evented, fiber: *Fiber) ?*Fiber {
1637	group.lock(ev);
1638	defer group.unlock(ev);
1639	const list_ptr = group.listPtr();
1640	const list = @atomicLoad(List, list_ptr, .monotonic);
1641	if (fiber.link.group.next) |next| next.link.group.prev = fiber.link.group.prev;
1642	if (fiber.link.group.prev) |prev| {
1643	prev.link.group.next = fiber.link.group.next;
1644	} else if (fiber.link.group.next) |new_head| {
1645	@atomicStore(List, list_ptr, .{
1646	.cancel_requested = list.cancel_requested,
1647	.awaiter_delayed = list.awaiter_delayed,
1648	.fibers = .pack(new_head),
1649	}, .monotonic);
1650	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) |awaiter| {
1651	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
1652	@atomicStore(List, list_ptr, .{
1653	.cancel_requested = false,
1654	.awaiter_delayed = false,
1655	.fibers = .null,
1656	}, .release);
1657	assert(awaiter.status.awaiting_group.ptr == group.ptr);
1658	awaiter.status = .{ .queue_next = null };
1659	return awaiter;
1660	}
1661	// Race with `Fiber.requestCancel`
1662	@atomicStore(List, list_ptr, .{
1663	.cancel_requested = false,
1664	.awaiter_delayed = true,
1665	.fibers = .null,
1666	}, .monotonic);
1667	} else @atomicStore(List, list_ptr, .{
1668	.cancel_requested = false,
1669	.awaiter_delayed = false,
1670	.fibers = .null,
1671	}, .release);
1672	return null;
1673	}
1674
1675	fn await(group: Group, ev: *Evented, awaiter: *Fiber) bool {
1676	group.lock(ev);
1677	defer group.unlock(ev);
1678	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) |_| {
1679	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
1680	// The awaiter already had an unacknowledged cancelation request before
1681	// attempting to await a group, so propagate the cancelation to the group.
1682	assert(!group.cancelLocked(ev, null));
1683	}
1684	return false;
1685	}
1686	return true;
1687	}
1688
1689	fn cancel(group: Group, ev: *Evented, maybe_awaiter: ?*Fiber) bool {
1690	group.lock(ev);
1691	defer group.unlock(ev);
1692	return group.cancelLocked(ev, maybe_awaiter);
1693	}
1694
1695	/// Assumes the mutex is held.
1696	fn cancelLocked(group: Group, ev: *Evented, maybe_awaiter: ?*Fiber) bool {
1697	const list_ptr = group.listPtr();
1698	const list = @atomicRmw(
1699	List,
1700	list_ptr,
1701	.Add,
1702	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
1703	.monotonic,
1704	);
1705	assert(!list.cancel_requested);
1706	if (list.fibers.unpack()) |head| {
1707	var maybe_fiber: ?*Fiber = head;
1708	while (maybe_fiber) |fiber| {
1709	fiber.requestCancel(ev);
1710	maybe_fiber = fiber.link.group.next;
1711	}
1712	if (maybe_awaiter) |awaiter| _ = group.registerAwaiter(awaiter);
1713	return false;
1714	}
1715	@atomicStore(
1716	List,
1717	list_ptr,
1718	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
1719	.release,
1720	);
1721	return if (maybe_awaiter) |_| true else list.awaiter_delayed;
1722	}
1723
1724	/// Assumes the mutex is held.
1725	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
1726	assert(awaiter.status.queue_next == null);
1727	awaiter.status = .{ .awaiting_group = group };
1728	assert(@atomicRmw(
1729	Awaiter,
1730	group.awaiterPtr(),
1731	.Add,
1732	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
1733	.monotonic,
1734	).awaiter == .null);
1735	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
1736	}
1737
1738	const AsyncClosure = struct {
1739	evented: *Evented,
1740	group: Group,
1741	fiber: *Fiber,
1742	start: *const fn (context: *const anyopaque) void,
1743
1744	fn fromFiber(fiber: *Fiber) *Group.AsyncClosure {
1745	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
1746	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
1747	) - @sizeOf(Group.AsyncClosure));
1748	}
1749
1750	fn contextPointer(
1751	closure: *Group.AsyncClosure,
1752	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
1753	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
1754	}
1755
1756	fn entry() callconv(.naked) void {
1757	switch (builtin.cpu.arch) {
1758	.aarch64 => asm volatile (
1759	\\ mov x0, sp
1760	\\ b %[call]
1761	:
1762	: [call] "X" (&call),
1763	),
1764	.riscv64 => asm volatile (
1765	\\ mv a0, sp
1766	\\ tail %[call]@plt
1767	:
1768	: [call] "X" (&call),
1769	),
1770	.x86_64 => asm volatile (
1771	\\ leaq 8(%%rsp), %%rdi
1772	\\ jmp %[call:P]
1773	:
1774	: [call] "X" (&call),
1775	),
1776	else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1777	}
1778	}
1779
1780	fn call(
1781	closure: *Group.AsyncClosure,
1782	contexts: *const Io.fiber.Switch,
1783	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
1784	const message: *const SwitchMessage = @fieldParentPtr("contexts", contexts);
1785	const ev = closure.evented;
1786	const fiber = closure.fiber;
1787	message.handle(ev);
1788	assert(fiber.status.queue_next == null);
1789	closure.start(closure.contextPointer());
1790	ev.yield(closure.group.removeFiber(ev, fiber), .destroy);
1791	unreachable; // switched to dead fiber
1792	}
1793	};
1794	};
1795
1796	fn groupAsync(
1797	userdata: ?*anyopaque,
1798	type_erased: *Io.Group,
1799	context: []const u8,
1800	context_alignment: Alignment,
1801	start: *const fn (context: *const anyopaque) void,
1802	) void {
1803	const ev: *Evented = @ptrCast(@alignCast(userdata));
1804	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
1805	start(context.ptr);
1806	};
1807	}
1808
1809	fn groupConcurrent(
1810	userdata: ?*anyopaque,
1811	type_erased: *Io.Group,
1812	context: []const u8,
1813	context_alignment: Alignment,
1814	start: *const fn (context: *const anyopaque) void,
1815	) Io.ConcurrentError!void {
1816	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
1817	assert(context.len <= Fiber.max_context_size); // TODO
1818
1819	const ev: *Evented = @ptrCast(@alignCast(userdata));
1820	const group: Group = .{ .ptr = type_erased };
1821	const fiber = Fiber.create(ev) catch |err| switch (err) {
1822	error.OutOfMemory => return error.ConcurrencyUnavailable,
1823	};
1824
1825	const closure: *Group.AsyncClosure = .fromFiber(fiber);
1826	fiber.* = .{
1827	.required_align = {},
1828	.context = switch (builtin.cpu.arch) {
1829	.aarch64 => .{
1830	.sp = @intFromPtr(closure),
1831	.fp = 0,
1832	.pc = @intFromPtr(&Group.AsyncClosure.entry),
1833	},
1834	.riscv64 => .{
1835	.sp = @intFromPtr(closure),
1836	.fp = 0,
1837	.pc = @intFromPtr(&Group.AsyncClosure.entry),
1838	},
1839	.x86_64 => .{
1840	.rsp = @intFromPtr(closure) - 8,
1841	.rbp = 0,
1842	.rip = @intFromPtr(&Group.AsyncClosure.entry),
1843	},
1844	else => |arch| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1845	},
1846	.link = .{ .group = .{ .prev = null, .next = null } },
1847	.status = .{ .queue_next = null },
1848	.cancel_status = .unrequested,
1849	.cancel_protection = .unblocked,
1850	.name = if (tracy.enable) name: {
1851	const thread: *Thread = .current();
1852	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
1853	defer thread.name_arena = name_arena.state;
1854	break :name std.fmt.allocPrintSentinel(
1855	name_arena.allocator(),
1856	"group task {d}",
1857	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
1858	0,
1859	) catch return error.ConcurrencyUnavailable;
1860	},
1861	};
1862	closure.* = .{
1863	.evented = ev,
1864	.group = group,
1865	.fiber = fiber,
1866	.start = start,
1867	};
1868	@memcpy(closure.contextPointer(), context);
1869	group.addFiber(ev, fiber);
1870	const thread: *Thread = .current();
1871	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
1872	}
1873
1874	fn groupAwait(
1875	userdata: ?*anyopaque,
1876	type_erased: *Io.Group,
1877	initial_token: *anyopaque,
1878	) Io.Cancelable!void {
1879	const ev: *Evented = @ptrCast(@alignCast(userdata));
1880	_ = initial_token;
1881	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
1882	}
1883
1884	fn groupCancel(userdata: ?*anyopaque, type_erased: *Io.Group, initial_token: *anyopaque) void {
1885	const ev: *Evented = @ptrCast(@alignCast(userdata));
1886	_ = initial_token;
1887	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
1888	}
1889
1890	fn recancel(userdata: ?*anyopaque) void {
1891	const ev: *Evented = @ptrCast(@alignCast(userdata));
1892	_ = ev;
1893	Thread.current().currentFiber().cancel_protection.recancel();
1894	}
1895
1896	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
1897	const ev: *Evented = @ptrCast(@alignCast(userdata));
1898	_ = ev;
1899	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
1900	defer cancel_protection.user = new;
1901	return cancel_protection.user;
1902	}
1903
1904	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
1905	const ev: *Evented = @ptrCast(@alignCast(userdata));
1906	_ = ev;
1907	const fiber = Thread.current().currentFiber();
1908	switch (fiber.cancel_protection.check()) {
1909	.unblocked => {
1910	const cancel_status = @atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic);
1911	assert(cancel_status.awaiting == .nothing);
1912	if (cancel_status.requested) {
1913	@branchHint(.unlikely);
1914	fiber.cancel_protection.acknowledge();
1915	return error.Canceled;
1916	}
1917	},
1918	.blocked => {},
1919	}
1920	}
1921
1922	fn futexWait(
1923	userdata: ?*anyopaque,
1924	ptr: *const u32,
1925	expected: u32,
1926	timeout: Io.Timeout,
1927	) Io.Cancelable!void {
1928	const ev: *Evented = @ptrCast(@alignCast(userdata));
1929	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
1930	.none => .{
1931	null,
1932	.awake,
1933	linux.IORING_TIMEOUT_ABS,
1934	},
1935	.duration => |duration| {
1936	const ns = duration.raw.toNanoseconds();
1937	break :timespec .{
1938	.{
1939	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
1940	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
1941	},
1942	duration.clock,
1943	0,
1944	};
1945	},
1946	.deadline => |deadline| {
1947	const ns = deadline.raw.toNanoseconds();
1948	break :timespec .{
1949	.{
1950	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
1951	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
1952	},
1953	deadline.clock,
1954	linux.IORING_TIMEOUT_ABS,
1955	};
1956	},
1957	};
1958	var cancel_region: CancelRegion = .init();
1959	defer cancel_region.deinit();
1960	const thread = try cancel_region.awaitIoUring();
1961	thread.enqueue().* = .{
1962	.opcode = .FUTEX_WAIT,
1963	.flags = if (timespec) |_| linux.IOSQE_IO_LINK else 0,
1964	.ioprio = 0,
1965	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
1966	.off = expected,
1967	.addr = @intFromPtr(ptr),
1968	.len = 0,
1969	.rw_flags = 0,
1970	.user_data = @intFromPtr(cancel_region.fiber),
1971	.buf_index = 0,
1972	.personality = 0,
1973	.splice_fd_in = 0,
1974	.addr3 = std.math.maxInt(u32),
1975	.resv = 0,
1976	};
1977	if (timespec) |*timespec_ptr| thread.enqueue().* = .{
1978	.opcode = .LINK_TIMEOUT,
1979	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1980	.ioprio = 0,
1981	.fd = 0,
1982	.off = 0,
1983	.addr = @intFromPtr(timespec_ptr),
1984	.len = 1,
1985	.rw_flags = timeout_flags | @as(u32, switch (clock) {
1986	.real => linux.IORING_TIMEOUT_REALTIME,
1987	else => 0,
1988	.boot => linux.IORING_TIMEOUT_BOOTTIME,
1989	}),
1990	.user_data = @backingInt(Completion.Userdata.wakeup),
1991	.buf_index = 0,
1992	.personality = 0,
1993	.splice_fd_in = 0,
1994	.addr3 = 0,
1995	.resv = 0,
1996	};
1997	ev.yield(null, .nothing);
1998	switch (cancel_region.errno()) {
1999	.SUCCESS => {}, // notified by `wake()`
2000	.INTR, .CANCELED => {}, // caller's responsibility to retry
2001	.AGAIN => {}, // ptr.* != expect
2002	.INVAL => {}, // possibly timeout overflow
2003	.TIMEDOUT => unreachable,
2004	.FAULT => recoverableOsBugDetected(), // ptr was invalid
2005	else => recoverableOsBugDetected(),
2006	}
2007	}
2008
2009	fn futexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void {
2010	const ev: *Evented = @ptrCast(@alignCast(userdata));
2011	var cancel_region: CancelRegion = .initBlocked();
2012	defer cancel_region.deinit();
2013	const thread = cancel_region.awaitIoUring() catch |err| switch (err) {
2014	error.Canceled => unreachable, // blocked
2015	};
2016	thread.enqueue().* = .{
2017	.opcode = .FUTEX_WAIT,
2018	.flags = 0,
2019	.ioprio = 0,
2020	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2021	.off = expected,
2022	.addr = @intFromPtr(ptr),
2023	.len = 0,
2024	.rw_flags = 0,
2025	.user_data = @intFromPtr(cancel_region.fiber),
2026	.buf_index = 0,
2027	.personality = 0,
2028	.splice_fd_in = 0,
2029	.addr3 = std.math.maxInt(u32),
2030	.resv = 0,
2031	};
2032	ev.yield(null, .nothing);
2033	switch (cancel_region.errno()) {
2034	.SUCCESS => {}, // notified by `wake()`
2035	.INTR, .CANCELED => {}, // caller's responsibility to retry
2036	.AGAIN => {}, // ptr.* != expect
2037	.INVAL => {}, // possibly timeout overflow
2038	.FAULT => recoverableOsBugDetected(), // ptr was invalid
2039	else => recoverableOsBugDetected(),
2040	}
2041	}
2042
2043	fn futexWake(userdata: ?*anyopaque, ptr: *const u32, max_waiters: u32) void {
2044	const ev: *Evented = @ptrCast(@alignCast(userdata));
2045	_ = ev;
2046	const thread: *Thread = .current();
2047	thread.enqueue().* = .{
2048	.opcode = .FUTEX_WAKE,
2049	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2050	.ioprio = 0,
2051	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
2052	.off = max_waiters,
2053	.addr = @intFromPtr(ptr),
2054	.len = 0,
2055	.rw_flags = 0,
2056	.user_data = @backingInt(Completion.Userdata.futex_wake),
2057	.buf_index = 0,
2058	.personality = 0,
2059	.splice_fd_in = 0,
2060	.addr3 = std.math.maxInt(u32),
2061	.resv = 0,
2062	};
2063	thread.submit();
2064	}
2065
2066	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
2067	const ev: *Evented = @ptrCast(@alignCast(userdata));
2068	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2069	defer maybe_sync.deinit(ev);
2070	return switch (operation) {
2071	.file_read_streaming => |o| .{
2072	.file_read_streaming = ev.fileReadStreaming(
2073	&maybe_sync.cancel_region,
2074	o.file,
2075	o.data,
2076	) catch |err| switch (err) {
2077	error.Canceled => |e| return e,
2078	else => |e| e,
2079	},
2080	},
2081	.file_write_streaming => |o| .{
2082	.file_write_streaming = ev.fileWriteStreaming(
2083	&maybe_sync.cancel_region,
2084	o.file,
2085	o.header,
2086	o.data,
2087	o.splat,
2088	) catch |err| switch (err) {
2089	error.Canceled => |e| return e,
2090	else => |e| e,
2091	},
2092	},
2093	.device_io_control => |o| .{
2094	.device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o),
2095	},
2096	.net_receive => |o| .{
2097	.net_receive = r: {
2098	const opt_err, const n = ev.netReceive(&maybe_sync.cancel_region, o.socket_handle, o.message_buffer, o.data_buffer, o.flags);
2099	break :r .{
2100	if (opt_err) |err| switch (err) {
2101	error.Canceled => |e| return e,
2102	else => |e| e,
2103	} else null,
2104	n,
2105	};
2106	},
2107	},
2108	.net_send => |o| .{
2109	.net_send = r: {
2110	_ = o;
2111	break :r .{ error.NetworkDown, 0 }; // TODO
2112	},
2113	},
2114	.net_read => |o| .{
2115	.net_read = r: {
2116	_ = o;
2117	break :r error.NetworkDown; // TODO
2118	},
2119	},
2120	.net_write => @panic("TODO implement net_write operation"),
2121	};
2122	}
2123
2124	fn fileReadStreaming(
2125	ev: *Evented,
2126	cancel_region: *CancelRegion,
2127	file: File,
2128	data: []const []u8,
2129	) File.ReadStreamingError!usize {
2130	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
2131	var i: usize = 0;
2132	for (data) |buf| {
2133	if (iovecs_buffer.len - i == 0) break;
2134	if (buf.len > 0) {
2135	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
2136	i += 1;
2137	}
2138	}
2139	const dest = iovecs_buffer[0..i];
2140	assert(dest[0].len > 0);
2141
2142	const n = try ev.preadv(cancel_region, file.handle, dest, null);
2143	return if (n == 0) error.EndOfStream else n;
2144	}
2145
2146	fn fileWriteStreaming(
2147	ev: *Evented,
2148	cancel_region: *CancelRegion,
2149	file: File,
2150	header: []const u8,
2151	data: []const []const u8,
2152	splat: usize,
2153	) File.Writer.Error!usize {
2154	var iovecs: [max_iovecs_len]iovec_const = undefined;
2155	var iovlen: iovlen_t = 0;
2156	addBuf(&iovecs, &iovlen, header);
2157	for (data[0 .. data.len - 1]) |bytes| addBuf(&iovecs, &iovlen, bytes);
2158	const pattern = data[data.len - 1];
2159	var backup_buffer: [splat_buffer_size]u8 = undefined;
2160	if (iovecs.len - iovlen != 0) switch (splat) {
2161	0 => {},
2162	1 => addBuf(&iovecs, &iovlen, pattern),
2163	else => switch (pattern.len) {
2164	0 => {},
2165	1 => {
2166	const splat_buffer = &backup_buffer;
2167	const memset_len = @min(splat_buffer.len, splat);
2168	const buf = splat_buffer[0..memset_len];
2169	@memset(buf, pattern[0]);
2170	addBuf(&iovecs, &iovlen, buf);
2171	var remaining_splat = splat - buf.len;
2172	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
2173	assert(buf.len == splat_buffer.len);
2174	addBuf(&iovecs, &iovlen, splat_buffer);
2175	remaining_splat -= splat_buffer.len;
2176	}
2177	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
2178	},
2179	else => for (0..@min(splat, iovecs.len - iovlen)) |_| {
2180	addBuf(&iovecs, &iovlen, pattern);
2181	},
2182	},
2183	};
2184	return ev.pwritev(cancel_region, file.handle, iovecs[0..iovlen], null);
2185	}
2186
2187	fn deviceIoControl(
2188	ev: *Evented,
2189	sync: *CancelRegion.Sync,
2190	o: Io.Operation.DeviceIoControl,
2191	) Io.Cancelable!i32 {
2192	_ = ev;
2193	while (true) {
2194	try sync.cancel_region.await(.nothing);
2195	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
2196	switch (linux.errno(rc)) {
2197	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
2198	.INTR => {},
2199	else => |err| return -@as(i32, @backingInt(err)),
2200	}
2201	}
2202	}
2203
2204	fn batchAwaitAsync(userdata: ?*anyopaque, batch: *Io.Batch) Io.Cancelable!void {
2205	const ev: *Evented = @ptrCast(@alignCast(userdata));
2206	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2207	defer maybe_sync.deinit(ev);
2208	ev.batchDrainSubmitted(&maybe_sync, batch, false) catch |err| switch (err) {
2209	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2210	error.Canceled => |e| return e,
2211	};
2212	maybe_sync.leaveSync(ev);
2213	while (true) {
2214	batchDrainReady(batch) catch |err| switch (err) {
2215	error.Timeout => unreachable, // no timeout
2216	};
2217	if (batch.completed.head != .none or batch.pending.head == .none) return;
2218	ev.yield(null, .{ .batch_await = batch });
2219	}
2220	}
2221
2222	fn batchAwaitConcurrent(
2223	userdata: ?*anyopaque,
2224	batch: *Io.Batch,
2225	timeout: Io.Timeout,
2226	) Io.Batch.AwaitConcurrentError!void {
2227	const ev: *Evented = @ptrCast(@alignCast(userdata));
2228	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2229	defer maybe_sync.deinit(ev);
2230	try ev.batchDrainSubmitted(&maybe_sync, batch, true);
2231	maybe_sync.leaveSync(ev);
2232	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
2233	batchDrainReady(batch) catch |err| switch (err) {
2234	error.Timeout => unreachable, // no timeout
2235	};
2236	if (batch.completed.head != .none or batch.pending.head == .none) return;
2237	switch (timeout) {
2238	.none => ev.yield(null, .{ .batch_await = batch }),
2239	.duration => |duration| {
2240	const ns = duration.raw.toNanoseconds();
2241	break .{
2242	.{
2243	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2244	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2245	},
2246	duration.clock,
2247	0,
2248	};
2249	},
2250	.deadline => |deadline| {
2251	const ns = deadline.raw.toNanoseconds();
2252	break .{
2253	.{
2254	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
2255	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
2256	},
2257	deadline.clock,
2258	linux.IORING_TIMEOUT_ABS,
2259	};
2260	},
2261	}
2262	};
2263	{
2264	const thread = try maybe_sync.cancel_region.awaitIoUring();
2265	thread.enqueue().* = .{
2266	.opcode = .TIMEOUT,
2267	.flags = 0,
2268	.ioprio = 0,
2269	.fd = 0,
2270	.off = 0,
2271	.addr = @intFromPtr(&timespec),
2272	.len = 1,
2273	.rw_flags = timeout_flags | @as(u32, switch (clock) {
2274	.real => linux.IORING_TIMEOUT_REALTIME,
2275	else => 0,
2276	.boot => linux.IORING_TIMEOUT_BOOTTIME,
2277	}),
2278	.user_data = @intFromPtr(&batch.userdata) | 0b11,
2279	.buf_index = 0,
2280	.personality = 0,
2281	.splice_fd_in = 0,
2282	.addr3 = 0,
2283	.resv = 0,
2284	};
2285	}
2286	while (batch.completed.head == .none and batch.pending.head != .none) {
2287	ev.yield(null, .{ .batch_await = batch });
2288	batchDrainReady(batch) catch |err| switch (err) {
2289	error.Timeout => |e| return if (batch.completed.head == .none and
2290	batch.pending.head != .none) e,
2291	};
2292	}
2293	const thread = try maybe_sync.cancel_region.awaitIoUring();
2294	thread.enqueue().* = .{
2295	.opcode = .TIMEOUT_REMOVE,
2296	.flags = 0,
2297	.ioprio = 0,
2298	.fd = 0,
2299	.off = 0,
2300	.addr = @intFromPtr(&batch.userdata) | 0b11,
2301	.len = 0,
2302	.rw_flags = 0,
2303	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
2304	.buf_index = 0,
2305	.personality = 0,
2306	.splice_fd_in = 0,
2307	.addr3 = 0,
2308	.resv = 0,
2309	};
2310	ev.yield(null, .nothing);
2311	switch (maybe_sync.cancel_region.errno()) {
2312	.SUCCESS => return,
2313	.BUSY, .NOENT => {},
2314	else => |err| unexpectedErrno(err) catch {},
2315	}
2316	while (true) {
2317	batchDrainReady(batch) catch |err| switch (err) {
2318	error.Timeout => return,
2319	};
2320	ev.yield(null, .{ .batch_await = batch });
2321	}
2322	}
2323
2324	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2325	fn batchDrainSubmitted(
2326	ev: *Evented,
2327	maybe_sync: *CancelRegion.Sync.Maybe,
2328	batch: *Io.Batch,
2329	concurrency: bool,
2330	) (Io.ConcurrentError || Io.Cancelable)!void {
2331	var index = batch.submitted.head;
2332	if (index == .none) return;
2333	const thread = try maybe_sync.cancelRegion().awaitIoUring();
2334	errdefer batch.submitted.head = index;
2335	while (index != .none) {
2336	const storage = &batch.storage[index.toIndex()];
2337	const next_index = storage.submission.node.next;
2338	if (@as(?Io.Operation.Result, result: switch (storage.submission.operation) {
2339	.file_read_streaming => |o| {
2340	const buffer = for (o.data) |buffer| {
2341	if (buffer.len > 0) break buffer;
2342	} else break :result .{ .file_read_streaming = 0 };
2343	const fd = o.file.handle;
2344	storage.* = .{ .pending = .{
2345	.node = .{ .prev = batch.pending.tail, .next = .none },
2346	.tag = .file_read_streaming,
2347	.userdata = undefined,
2348	} };
2349	thread.enqueue().* = .{
2350	.opcode = .READ,
2351	.flags = 0,
2352	.ioprio = 0,
2353	.fd = fd,
2354	.off = std.math.maxInt(u64),
2355	.addr = @intFromPtr(buffer.ptr),
2356	.len = @min(buffer.len, 0xfffff000),
2357	.rw_flags = 0,
2358	.user_data = @intFromPtr(&storage.pending.userdata) | 0b10,
2359	.buf_index = 0,
2360	.personality = 0,
2361	.splice_fd_in = 0,
2362	.addr3 = 0,
2363	.resv = 0,
2364	};
2365	break :result null;
2366	},
2367	.file_write_streaming => |o| {
2368	const buffer = buffer: {
2369	if (o.header.len != 0) break :buffer o.header;
2370	for (o.data[0 .. o.data.len - 1]) |buffer| {
2371	if (buffer.len > 0) break :buffer buffer;
2372	}
2373	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
2374	break :result .{ .file_write_streaming = 0 };
2375	};
2376	const fd = o.file.handle;
2377	storage.* = .{ .pending = .{
2378	.node = .{ .prev = batch.pending.tail, .next = .none },
2379	.tag = .file_write_streaming,
2380	.userdata = undefined,
2381	} };
2382	thread.enqueue().* = .{
2383	.opcode = .WRITE,
2384	.flags = 0,
2385	.ioprio = 0,
2386	.fd = fd,
2387	.off = std.math.maxInt(u64),
2388	.addr = @intFromPtr(buffer.ptr),
2389	.len = @min(buffer.len, 0xfffff000),
2390	.rw_flags = 0,
2391	.user_data = @intFromPtr(&storage.pending.userdata) | 0b10,
2392	.buf_index = 0,
2393	.personality = 0,
2394	.splice_fd_in = 0,
2395	.addr3 = 0,
2396	.resv = 0,
2397	};
2398	break :result null;
2399	},
2400	.device_io_control => |o| if (concurrency)
2401	return error.ConcurrencyUnavailable
2402	else
2403	.{ .device_io_control = try ev.deviceIoControl(try maybe_sync.enterSync(ev), o) },
2404	.net_receive => |o| {
2405	_ = o;
2406	@panic("TODO implement batchDrainSubmitted for net_receive");
2407	},
2408	.net_send => |o| {
2409	_ = o;
2410	@panic("TODO implement batchDrainSubmitted for net_send");
2411	},
2412	.net_read => |o| {
2413	_ = o;
2414	@panic("TODO implement batchDrainSubmitted for net_read");
2415	},
2416	.net_write => |o| {
2417	_ = o;
2418	@panic("TODO implement batchDrainSubmitted for net_write");
2419	},
2420	})) |result| {
2421	switch (batch.completed.tail) {
2422	.none => batch.completed.head = index,
2423	else => |tail_index| batch.storage[tail_index.toIndex()].completion.node.next = index,
2424	}
2425	batch.completed.tail = index;
2426	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2427	} else {
2428	switch (batch.pending.tail) {
2429	.none => batch.pending.head = index,
2430	else => |tail_index| batch.storage[tail_index.toIndex()].pending.node.next = index,
2431	}
2432	batch.pending.tail = index;
2433	storage.pending.userdata[0] = @intFromPtr(batch);
2434	}
2435	index = next_index;
2436	}
2437	batch.submitted = .{ .head = .none, .tail = .none };
2438	}
2439
2440	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
2441	while (@atomicRmw(?*anyopaque, &batch.userdata, .Xchg, null, .acquire)) |head| {
2442	var next: usize = @intFromPtr(head);
2443	var timeout = false;
2444	while (cond: switch (@as(u2, @truncate(next))) {
2445	0b00 => if (timeout) return error.Timeout else false,
2446	0b01 => {
2447	assert(!timeout);
2448	return error.Timeout;
2449	},
2450	0b10 => true,
2451	0b11 => {
2452	assert(!timeout);
2453	timeout = true;
2454	break :cond true;
2455	},
2456	}) {
2457	const operation_userdata: *Io.Operation.Storage.Pending.Userdata =
2458	@ptrFromInt(next & ~@as(usize, 0b11));
2459	next = operation_userdata[0];
2460	const completion: Completion = .{
2461	.result = @bitCast(@as(u32, @intCast(operation_userdata[1]))),
2462	.flags = @intCast(operation_userdata[2]),
2463	};
2464	const pending: *Io.Operation.Storage.Pending =
2465	@fieldParentPtr("userdata", operation_userdata);
2466	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
2467	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
2468	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
2469	switch (pending.node.prev) {
2470	.none => batch.pending.head = pending.node.next,
2471	else => |prev_index| batch.storage[prev_index.toIndex()].pending.node.next =
2472	pending.node.next,
2473	}
2474	switch (pending.node.next) {
2475	.none => batch.pending.tail = pending.node.prev,
2476	else => |prev_index| batch.storage[prev_index.toIndex()].pending.node.prev =
2477	pending.node.prev,
2478	}
2479	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
2480	.file_read_streaming => .{
2481	.file_read_streaming = switch (completion.errno()) {
2482	.SUCCESS => @as(u32, @bitCast(completion.result)),
2483	.INTR => 0,
2484	.CANCELED => break :result null,
2485	.INVAL => |err| errnoBug(err),
2486	.FAULT => |err| errnoBug(err),
2487	.AGAIN => error.WouldBlock,
2488	.BADF => |err| errnoBug(err), // File descriptor used after closed
2489	.IO => error.InputOutput,
2490	.ISDIR => error.IsDir,
2491	.NOBUFS => error.SystemResources,
2492	.NOMEM => error.SystemResources,
2493	.NOTCONN => error.SocketUnconnected,
2494	.CONNRESET => error.ConnectionResetByPeer,
2495	else => |err| unexpectedErrno(err),
2496	},
2497	},
2498	.file_write_streaming => .{
2499	.file_write_streaming = switch (completion.errno()) {
2500	.SUCCESS => @as(u32, @bitCast(completion.result)),
2501	.INTR => 0,
2502	.CANCELED => break :result null,
2503	.INVAL => |err| errnoBug(err),
2504	.FAULT => |err| errnoBug(err),
2505	.AGAIN => error.WouldBlock,
2506	.BADF => error.NotOpenForWriting, // Can be a race condition.
2507	.DESTADDRREQ => |err| errnoBug(err), // `connect` was never called.
2508	.DQUOT => error.DiskQuota,
2509	.FBIG => error.FileTooBig,
2510	.IO => error.InputOutput,
2511	.NOSPC => error.NoSpaceLeft,
2512	.PERM => error.PermissionDenied,
2513	.PIPE => error.BrokenPipe,
2514	.CONNRESET => |err| errnoBug(err), // Not a socket handle.
2515	.BUSY => error.DeviceBusy,
2516	else => |err| unexpectedErrno(err),
2517	},
2518	},
2519	.device_io_control => unreachable,
2520	.net_receive => @panic("TODO"),
2521	.net_send => @panic("TODO"),
2522	.net_read => @panic("TODO"),
2523	.net_write => @panic("TODO"),
2524	})) |result| {
2525	switch (batch.completed.tail) {
2526	.none => batch.completed.head = index,
2527	else => |tail_index| batch.storage[tail_index.toIndex()].completion.node.next =
2528	index,
2529	}
2530	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2531	batch.completed.tail = index;
2532	} else {
2533	switch (batch.unused.tail) {
2534	.none => batch.unused.head = index,
2535	else => |tail_index| batch.storage[tail_index.toIndex()].unused.next = index,
2536	}
2537	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
2538	batch.unused.tail = index;
2539	}
2540	}
2541	}
2542	}
2543
2544	fn batchCancel(userdata: ?*anyopaque, batch: *Io.Batch) void {
2545	const ev: *Evented = @ptrCast(@alignCast(userdata));
2546	_ = ev;
2547	batchDrainReady(batch) catch |err| switch (err) {
2548	error.Timeout => unreachable, // no timeout
2549	};
2550	var index = batch.pending.head;
2551	if (index == .none) return;
2552	var cancel_region: CancelRegion = .initBlocked();
2553	defer cancel_region.deinit();
2554	const thread = cancel_region.awaitIoUring() catch |err| switch (err) {
2555	error.Canceled => unreachable, // blocked
2556	};
2557	while (index != .none) {
2558	const pending = &batch.storage[index.toIndex()].pending;
2559	thread.enqueue().* = .{
2560	.opcode = .ASYNC_CANCEL,
2561	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
2562	.ioprio = 0,
2563	.fd = 0,
2564	.off = 0,
2565	.addr = @intFromPtr(&pending.userdata) | 0b10,
2566	.len = 0,
2567	.rw_flags = 0,
2568	.user_data = @backingInt(Completion.Userdata.wakeup),
2569	.buf_index = 0,
2570	.personality = 0,
2571	.splice_fd_in = 0,
2572	.addr3 = 0,
2573	.resv = 0,
2574	};
2575	index = pending.node.next;
2576	}
2577	while (batch.pending.head != .none) batchDrainReady(batch) catch |err| switch (err) {
2578	error.Timeout => unreachable, // no timeout
2579	};
2580	}
2581
2582	fn dirCreateDir(
2583	userdata: ?*anyopaque,
2584	dir: Dir,
2585	sub_path: []const u8,
2586	permissions: Dir.Permissions,
2587	) Dir.CreateDirError!void {
2588	const ev: *Evented = @ptrCast(@alignCast(userdata));
2589
2590	var path_buffer: [PATH_MAX]u8 = undefined;
2591	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2592
2593	var cancel_region: CancelRegion = .init();
2594	defer cancel_region.deinit();
2595	while (true) {
2596	const thread = try cancel_region.awaitIoUring();
2597	thread.enqueue().* = .{
2598	.opcode = .MKDIRAT,
2599	.flags = 0,
2600	.ioprio = 0,
2601	.fd = dir.handle,
2602	.off = 0,
2603	.addr = @intFromPtr(sub_path_posix.ptr),
2604	.len = permissions.toMode(),
2605	.rw_flags = 0,
2606	.user_data = @intFromPtr(cancel_region.fiber),
2607	.buf_index = 0,
2608	.personality = 0,
2609	.splice_fd_in = 0,
2610	.addr3 = 0,
2611	.resv = 0,
2612	};
2613	ev.yield(null, .nothing);
2614	switch (cancel_region.errno()) {
2615	.SUCCESS => return,
2616	.INTR, .CANCELED => {},
2617	.ACCES => return error.AccessDenied,
2618	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
2619	.PERM => return error.PermissionDenied,
2620	.DQUOT => return error.DiskQuota,
2621	.EXIST => return error.PathAlreadyExists,
2622	.FAULT => |err| return errnoBug(err),
2623	.LOOP => return error.SymLinkLoop,
2624	.MLINK => return error.LinkQuotaExceeded,
2625	.NAMETOOLONG => return error.NameTooLong,
2626	.NOENT => return error.FileNotFound,
2627	.NOMEM => return error.SystemResources,
2628	.NOSPC => return error.NoSpaceLeft,
2629	.NOTDIR => return error.NotDir,
2630	.ROFS => return error.ReadOnlyFileSystem,
2631	.ILSEQ => return error.BadPathName,
2632	else => |err| return unexpectedErrno(err),
2633	}
2634	}
2635	}
2636
2637	fn dirCreateDirPath(
2638	userdata: ?*anyopaque,
2639	dir: Dir,
2640	sub_path: []const u8,
2641	permissions: Dir.Permissions,
2642	) Dir.CreateDirPathError!Dir.CreatePathStatus {
2643	const ev: *Evented = @ptrCast(@alignCast(userdata));
2644
2645	var it = Dir.path.componentIterator(sub_path);
2646	var status: Dir.CreatePathStatus = .existed;
2647	var component = it.last() orelse return error.BadPathName;
2648	while (true) {
2649	if (dirCreateDir(ev, dir, component.path, permissions)) |_| {
2650	status = .created;
2651	} else |err| switch (err) {
2652	error.PathAlreadyExists => {
2653	// stat the file and return an error if it's not a directory
2654	// this is important because otherwise a dangling symlink
2655	// could cause an infinite loop
2656	const kind = try ev.filePathKind(dir, component.path);
2657	if (kind != .directory) return error.NotDir;
2658	},
2659	error.FileNotFound => |e| {
2660	component = it.previous() orelse return e;
2661	continue;
2662	},
2663	else => |e| return e,
2664	}
2665	component = it.next() orelse return status;
2666	}
2667	}
2668
2669	fn filePathKind(ev: *Evented, dir: Dir, sub_path: []const u8) !File.Kind {
2670	var path_buffer: [PATH_MAX]u8 = undefined;
2671	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2672	var cancel_region: CancelRegion = .init();
2673	defer cancel_region.deinit();
2674	while (true) {
2675	var statx_buf = std.mem.zeroes(linux.Statx);
2676	const thread = try cancel_region.awaitIoUring();
2677	thread.enqueue().* = .{
2678	.opcode = .STATX,
2679	.flags = 0,
2680	.ioprio = 0,
2681	.fd = dir.handle,
2682	.off = @intFromPtr(&statx_buf),
2683	.addr = @intFromPtr(sub_path_posix.ptr),
2684	.len = @bitCast(linux.STATX{ .TYPE = true }),
2685	.rw_flags = linux.AT.NO_AUTOMOUNT | linux.AT.SYMLINK_NOFOLLOW,
2686	.user_data = @intFromPtr(cancel_region.fiber),
2687	.buf_index = 0,
2688	.personality = 0,
2689	.splice_fd_in = 0,
2690	.addr3 = 0,
2691	.resv = 0,
2692	};
2693	ev.yield(null, .nothing);
2694	switch (cancel_region.errno()) {
2695	.SUCCESS => {
2696	if (!statx_buf.mask.TYPE) return error.Unexpected;
2697	return statxKind(statx_buf.mode);
2698	},
2699	.INTR, .CANCELED => {},
2700	.ACCES => |err| return errnoBug(err),
2701	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
2702	.FAULT => |err| return errnoBug(err),
2703	.INVAL => |err| return errnoBug(err),
2704	.LOOP => |err| return errnoBug(err),
2705	.NAMETOOLONG => |err| return errnoBug(err),
2706	.NOENT => |err| return errnoBug(err),
2707	.NOMEM => return error.SystemResources,
2708	.NOTDIR => |err| return errnoBug(err),
2709	else => |err| return unexpectedErrno(err),
2710	}
2711	}
2712	}
2713
2714	fn dirCreateDirPathOpen(
2715	userdata: ?*anyopaque,
2716	dir: Dir,
2717	sub_path: []const u8,
2718	permissions: Dir.Permissions,
2719	options: Dir.OpenOptions,
2720	) Dir.CreateDirPathOpenError!Dir {
2721	const ev: *Evented = @ptrCast(@alignCast(userdata));
2722	return dirOpenDir(ev, dir, sub_path, options) catch |err| switch (err) {
2723	error.FileNotFound => {
2724	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
2725	return dirOpenDir(ev, dir, sub_path, options);
2726	},
2727	else => |e| return e,
2728	};
2729	}
2730
2731	fn dirOpenDir(
2732	userdata: ?*anyopaque,
2733	dir: Dir,
2734	sub_path: []const u8,
2735	options: Dir.OpenOptions,
2736	) Dir.OpenError!Dir {
2737	const ev: *Evented = @ptrCast(@alignCast(userdata));
2738
2739	var path_buffer: [PATH_MAX]u8 = undefined;
2740	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2741
2742	var cancel_region: CancelRegion = .init();
2743	defer cancel_region.deinit();
2744	return .{
2745	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
2746	.ACCMODE = .RDONLY,
2747	.DIRECTORY = true,
2748	.NOFOLLOW = !options.follow_symlinks,
2749	.CLOEXEC = true,
2750	.PATH = !options.iterate,
2751	}, 0) catch |err| switch (err) {
2752	error.IsDir => return errnoBug(.ISDIR),
2753	error.WouldBlock => return errnoBug(.AGAIN),
2754	error.FileTooBig => return errnoBug(.FBIG),
2755	error.NoSpaceLeft => return errnoBug(.NOSPC),
2756	error.DeviceBusy => return errnoBug(.BUSY), // EXCL unset.
2757	error.FileBusy => return errnoBug(.TXTBSY),
2758	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
2759	error.OperationUnsupported => return errnoBug(.OPNOTSUPP), // No TMPFILE, no locks.
2760	error.ReadOnlyFileSystem => return errnoBug(.ROFS), // Not creating.
2761	else => |e| return e,
2762	},
2763	};
2764	}
2765
2766	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
2767	const ev: *Evented = @ptrCast(@alignCast(userdata));
2768	var cancel_region: CancelRegion = .init();
2769	defer cancel_region.deinit();
2770	return ev.stat(&cancel_region, dir.handle);
2771	}
2772
2773	fn dirStatFile(
2774	userdata: ?*anyopaque,
2775	dir: Dir,
2776	sub_path: []const u8,
2777	options: Dir.StatFileOptions,
2778	) Dir.StatFileError!File.Stat {
2779	const ev: *Evented = @ptrCast(@alignCast(userdata));
2780	var path_buffer: [PATH_MAX]u8 = undefined;
2781	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2782	var cancel_region: CancelRegion = .init();
2783	defer cancel_region.deinit();
2784	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT |
2785	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
2786	}
2787
2788	fn dirAccess(
2789	userdata: ?*anyopaque,
2790	dir: Dir,
2791	sub_path: []const u8,
2792	options: Dir.AccessOptions,
2793	) Dir.AccessError!void {
2794	const ev: *Evented = @ptrCast(@alignCast(userdata));
2795
2796	var path_buffer: [PATH_MAX]u8 = undefined;
2797	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2798
2799	const mode: u32 =
2800	@as(u32, if (options.read) linux.R_OK else 0) |
2801	@as(u32, if (options.write) linux.W_OK else 0) |
2802	@as(u32, if (options.execute) linux.X_OK else 0);
2803	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
2804
2805	var sync: CancelRegion.Sync = try .init(ev);
2806	defer sync.deinit(ev);
2807	while (true) {
2808	try sync.cancel_region.await(.nothing);
2809	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
2810	.SUCCESS => return,
2811	.INTR => {},
2812	.ACCES => return error.AccessDenied,
2813	.PERM => return error.PermissionDenied,
2814	.ROFS => return error.ReadOnlyFileSystem,
2815	.LOOP => return error.SymLinkLoop,
2816	.TXTBSY => return error.FileBusy,
2817	.NOTDIR => return error.FileNotFound,
2818	.NOENT => return error.FileNotFound,
2819	.NAMETOOLONG => return error.NameTooLong,
2820	.INVAL => |err| return errnoBug(err),
2821	.FAULT => |err| return errnoBug(err),
2822	.IO => return error.InputOutput,
2823	.NOMEM => return error.SystemResources,
2824	.ILSEQ => return error.BadPathName,
2825	else => |err| return unexpectedErrno(err),
2826	}
2827	}
2828	}
2829
2830	fn dirCreateFile(
2831	userdata: ?*anyopaque,
2832	dir: Dir,
2833	sub_path: []const u8,
2834	flags: Dir.CreateFileOptions,
2835	) File.OpenError!File {
2836	const ev: *Evented = @ptrCast(@alignCast(userdata));
2837
2838	var path_buffer: [PATH_MAX]u8 = undefined;
2839	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
2840
2841	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
2842	defer maybe_sync.deinit(ev);
2843	const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
2844	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
2845	.CREAT = true,
2846	.TRUNC = flags.truncate,
2847	.EXCL = flags.exclusive,
2848	.CLOEXEC = true,
2849	}, flags.permissions.toMode()) catch |err| switch (err) {
2850	error.OperationUnsupported => return error.Unexpected, // TMPFILE unset.
2851	else => |e| return e,
2852	};
2853	errdefer ev.closeAsync(fd);
2854
2855	switch (flags.lock) {
2856	.none => {},
2857	.shared, .exclusive => try ev.flock(
2858	try maybe_sync.enterSync(ev),
2859	fd,
2860	flags.lock,
2861	if (flags.lock_nonblocking) .nonblocking else .blocking,
2862	),
2863	}
2864
2865	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
2866	}
2867
2868	fn dirCreateFileAtomic(
2869	userdata: ?*anyopaque,
2870	dir: Dir,
2871	dest_path: []const u8,
2872	options: Dir.CreateFileAtomicOptions,
2873	) Dir.CreateFileAtomicError!File.Atomic {
2874	const ev: *Evented = @ptrCast(@alignCast(userdata));
2875	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
2876	// useless when we have to make up a bogus path name to do the rename()
2877	// anyway.
2878	if (!options.replace) tmpfile: {
2879	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
2880	.ACCMODE = .RDWR,
2881	.TMPFILE = true,
2882	.DIRECTORY = true,
2883	.CLOEXEC = true,
2884	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
2885	.ACCMODE = .RDWR,
2886	.TMPFILE0 = true,
2887	.TMPFILE1 = true,
2888	.DIRECTORY = true,
2889	.CLOEXEC = true,
2890	} else break :tmpfile;
2891
2892	const dest_dirname = Dir.path.dirname(dest_path);
2893	if (dest_dirname) |dirname| {
2894	// This has a nice side effect of preemptively triggering EISDIR or
2895	// ENOENT, avoiding the ambiguity below.
2896	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch |err| switch (err) {
2897	// None of these make sense in this context.
2898	error.IsDir,
2899	error.Streaming,
2900	error.DiskQuota,
2901	error.PathAlreadyExists,
2902	error.LinkQuotaExceeded,
2903	error.PipeBusy,
2904	error.FileTooBig,
2905	error.DeviceBusy,
2906	error.FileLocksUnsupported,
2907	error.FileBusy,
2908	=> return error.Unexpected,
2909
2910	else => |e| return e,
2911	};
2912	}
2913
2914	var path_buffer: [PATH_MAX]u8 = undefined;
2915	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
2916
2917	var cancel_region: CancelRegion = .init();
2918	defer cancel_region.deinit();
2919	return .{
2920	.file = .{
2921	.handle = ev.openat(
2922	&cancel_region,
2923	dir.handle,
2924	sub_path_posix,
2925	flags,
2926	options.permissions.toMode(),
2927	) catch |err| switch (err) {
2928	error.IsDir, error.FileNotFound, error.OperationUnsupported => {
2929	// Ambiguous error code. It might mean the file system
2930	// does not support O_TMPFILE. Therefore, we must fall
2931	// back to not using O_TMPFILE.
2932	break :tmpfile;
2933	},
2934	error.FileTooBig => return errnoBug(.FBIG),
2935	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
2936	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
2937	else => |e| return e,
2938	},
2939	.flags = .{ .nonblocking = false },
2940	},
2941	.file_basename_hex = 0,
2942	.dest_sub_path = dest_path,
2943	.file_open = true,
2944	.file_exists = false,
2945	.close_dir_on_deinit = false,
2946	.dir = dir,
2947	};
2948	}
2949
2950	if (Dir.path.dirname(dest_path)) |dirname| {
2951	const new_dir = if (options.make_path)
2952	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch |err| switch (err) {
2953	// None of these make sense in this context.
2954	error.IsDir,
2955	error.Streaming,
2956	error.DiskQuota,
2957	error.PathAlreadyExists,
2958	error.LinkQuotaExceeded,
2959	error.PipeBusy,
2960	error.FileTooBig,
2961	error.FileLocksUnsupported,
2962	error.DeviceBusy,
2963	=> return error.Unexpected,
2964
2965	else => |e| return e,
2966	}
2967	else
2968	try dirOpenDir(ev, dir, dirname, .{});
2969
2970	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
2971	}
2972
2973	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
2974	}
2975
2976	fn atomicFileInit(
2977	ev: *Evented,
2978	dest_basename: []const u8,
2979	permissions: File.Permissions,
2980	dir: Dir,
2981	close_dir_on_deinit: bool,
2982	) Dir.CreateFileAtomicError!File.Atomic {
2983	while (true) {
2984	var random_integer: u64 = undefined;
2985	random(ev, @ptrCast(&random_integer));
2986	const tmp_sub_path = std.fmt.hex(random_integer);
2987	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
2988	.permissions = permissions,
2989	.exclusive = true,
2990	}) catch |err| switch (err) {
2991	error.PathAlreadyExists => continue,
2992	error.DeviceBusy => continue,
2993	error.FileBusy => continue,
2994
2995	error.IsDir => return error.Unexpected, // No path components.
2996	error.FileTooBig => return error.Unexpected, // Creating, not opening.
2997	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
2998	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
2999
3000	else => |e| return e,
3001	};
3002	return .{
3003	.file = file,
3004	.file_basename_hex = random_integer,
3005	.dest_sub_path = dest_basename,
3006	.file_open = true,
3007	.file_exists = true,
3008	.close_dir_on_deinit = close_dir_on_deinit,
3009	.dir = dir,
3010	};
3011	}
3012	}
3013
3014	fn dirOpenFile(
3015	userdata: ?*anyopaque,
3016	dir: Dir,
3017	sub_path: []const u8,
3018	flags: Dir.OpenFileOptions,
3019	) File.OpenError!File {
3020	const ev: *Evented = @ptrCast(@alignCast(userdata));
3021
3022	var path_buffer: [PATH_MAX]u8 = undefined;
3023	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3024
3025	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3026	defer maybe_sync.deinit(ev);
3027	const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3028	.ACCMODE = switch (flags.mode) {
3029	.read_only => .RDONLY,
3030	.write_only => .WRONLY,
3031	.read_write => .RDWR,
3032	},
3033	.NOCTTY = !flags.allow_ctty,
3034	.NOFOLLOW = !flags.follow_symlinks,
3035	.CLOEXEC = true,
3036	.PATH = flags.path_only,
3037	}, 0) catch |err| switch (err) {
3038	error.OperationUnsupported => return error.Unexpected, // TMPFILE unset.
3039	else => |e| return e,
3040	};
3041	errdefer ev.closeAsync(fd);
3042
3043	if (!flags.allow_directory) {
3044	const is_dir = is_dir: {
3045	const s = ev.stat(&maybe_sync.cancel_region, fd) catch |err| switch (err) {
3046	// The directory-ness is either unknown or unknowable
3047	error.Streaming => break :is_dir false,
3048	else => |e| return e,
3049	};
3050	break :is_dir s.kind == .directory;
3051	};
3052	if (is_dir) return error.IsDir;
3053	}
3054
3055	switch (flags.lock) {
3056	.none => {},
3057	.shared, .exclusive => try ev.flock(
3058	try maybe_sync.enterSync(ev),
3059	fd,
3060	flags.lock,
3061	if (flags.lock_nonblocking) .nonblocking else .blocking,
3062	),
3063	}
3064
3065	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
3066	}
3067
3068	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
3069	const ev: *Evented = @ptrCast(@alignCast(userdata));
3070	for (dirs) |dir| ev.close(dir.handle);
3071	}
3072
3073	fn dirRead(userdata: ?*anyopaque, dr: *Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
3074	const ev: *Evented = @ptrCast(@alignCast(userdata));
3075	var buffer_index: usize = 0;
3076	while (buffer.len - buffer_index != 0) {
3077	if (dr.end - dr.index == 0) {
3078	// Refill the buffer, unless we've already created references to
3079	// buffered data.
3080	if (buffer_index != 0) break;
3081	var sync: CancelRegion.Sync = try .init(ev);
3082	defer sync.deinit(ev);
3083	if (dr.state == .reset) {
3084	ev.lseek(&sync, dr.dir.handle, 0, linux.SEEK.SET) catch |err| switch (err) {
3085	error.Unseekable => return error.Unexpected,
3086	else => |e| return e,
3087	};
3088	dr.state = .reading;
3089	}
3090	const n = while (true) {
3091	try sync.cancel_region.await(.nothing);
3092	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, @min(dr.buffer.len, std.math.maxInt(c_uint)));
3093	switch (linux.errno(rc)) {
3094	.SUCCESS => break rc,
3095	.INTR => {},
3096	.BADF => |err| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
3097	.FAULT => |err| return errnoBug(err),
3098	.NOTDIR => |err| return errnoBug(err),
3099	// To be consistent across platforms, iteration
3100	// ends if the directory being iterated is deleted
3101	// during iteration. This matches the behavior of
3102	// non-Linux, non-WASI UNIX platforms.
3103	.NOENT => {
3104	dr.state = .finished;
3105	return 0;
3106	},
3107	// This can occur when reading /proc/$PID/net, or
3108	// if the provided buffer is too small. Neither
3109	// scenario is intended to be handled by this API.
3110	.INVAL => return error.Unexpected,
3111	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
3112	else => |err| return unexpectedErrno(err),
3113	}
3114	};
3115	if (n == 0) {
3116	dr.state = .finished;
3117	return 0;
3118	}
3119	dr.index = 0;
3120	dr.end = n;
3121	}
3122	// Linux aligns the header by padding after the null byte of the name
3123	// to align the next entry. This means we can find the end of the name
3124	// by looking at only the 8 bytes before the next record. However since
3125	// file names are usually short it's better to keep the machine code
3126	// simpler.
3127	//
3128	// Furthermore, I observed qemu user mode to not align this struct, so
3129	// this code makes the conservative choice to not assume alignment.
3130	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
3131	const next_index = dr.index + linux_entry.reclen;
3132	dr.index = next_index;
3133	const name_ptr: [*]u8 = &linux_entry.name;
3134	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
3135	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
3136	const name = name_ptr[0..name_len :0];
3137
3138	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
3139
3140	const entry_kind: File.Kind = switch (linux_entry.type) {
3141	linux.DT.BLK => .block_device,
3142	linux.DT.CHR => .character_device,
3143	linux.DT.DIR => .directory,
3144	linux.DT.FIFO => .named_pipe,
3145	linux.DT.LNK => .sym_link,
3146	linux.DT.REG => .file,
3147	linux.DT.SOCK => .unix_domain_socket,
3148	else => .unknown,
3149	};
3150	buffer[buffer_index] = .{
3151	.name = name,
3152	.kind = entry_kind,
3153	.inode = linux_entry.ino,
3154	};
3155	buffer_index += 1;
3156	}
3157	return buffer_index;
3158	}
3159
3160	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
3161	const ev: *Evented = @ptrCast(@alignCast(userdata));
3162	var sync: CancelRegion.Sync = try .init(ev);
3163	defer sync.deinit(ev);
3164	return ev.realPath(&sync, dir.handle, out_buffer);
3165	}
3166
3167	fn dirRealPathFile(
3168	userdata: ?*anyopaque,
3169	dir: Dir,
3170	sub_path: []const u8,
3171	out_buffer: []u8,
3172	) Dir.RealPathFileError!usize {
3173	const ev: *Evented = @ptrCast(@alignCast(userdata));
3174
3175	var path_buffer: [PATH_MAX]u8 = undefined;
3176	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3177
3178	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
3179	defer maybe_sync.deinit(ev);
3180	const fd = ev.openat(&maybe_sync.cancel_region, dir.handle, sub_path_posix, .{
3181	.CLOEXEC = true,
3182	.PATH = true,
3183	}, 0) catch |err| switch (err) {
3184	error.WouldBlock => return errnoBug(.AGAIN),
3185	error.OperationUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
3186	error.ReadOnlyFileSystem => return errnoBug(.ROFS), // Not creating.
3187	else => |e| return e,
3188	};
3189	defer ev.closeAsync(fd);
3190	return ev.realPath(try maybe_sync.enterSync(ev), fd, out_buffer);
3191	}
3192
3193	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
3194	const ev: *Evented = @ptrCast(@alignCast(userdata));
3195
3196	var path_buffer: [PATH_MAX]u8 = undefined;
3197	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3198
3199	var cancel_region: CancelRegion = .init();
3200	defer cancel_region.deinit();
3201	while (true) {
3202	const thread = try cancel_region.awaitIoUring();
3203	thread.enqueue().* = .{
3204	.opcode = .UNLINKAT,
3205	.flags = 0,
3206	.ioprio = 0,
3207	.fd = dir.handle,
3208	.off = 0,
3209	.addr = @intFromPtr(sub_path_posix.ptr),
3210	.len = 0,
3211	.rw_flags = 0,
3212	.user_data = @intFromPtr(cancel_region.fiber),
3213	.buf_index = 0,
3214	.personality = 0,
3215	.splice_fd_in = 0,
3216	.addr3 = 0,
3217	.resv = 0,
3218	};
3219	ev.yield(null, .nothing);
3220	switch (cancel_region.errno()) {
3221	.SUCCESS => return,
3222	.INTR, .CANCELED => {},
3223	.PERM => return error.PermissionDenied,
3224	.ACCES => return error.AccessDenied,
3225	.BUSY => return error.FileBusy,
3226	.FAULT => |err| return errnoBug(err),
3227	.IO => return error.FileSystem,
3228	.ISDIR => return error.IsDir,
3229	.LOOP => return error.SymLinkLoop,
3230	.NAMETOOLONG => return error.NameTooLong,
3231	.NOENT => return error.FileNotFound,
3232	.NOTDIR => return error.NotDir,
3233	.NOMEM => return error.SystemResources,
3234	.ROFS => return error.ReadOnlyFileSystem,
3235	.EXIST => |err| return errnoBug(err),
3236	.NOTEMPTY => |err| return errnoBug(err), // Not passing AT.REMOVEDIR
3237	.ILSEQ => return error.BadPathName,
3238	.INVAL => |err| return errnoBug(err), // invalid flags, or pathname has . as last component
3239	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
3240	else => |err| return unexpectedErrno(err),
3241	}
3242	}
3243	}
3244
3245	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
3246	const ev: *Evented = @ptrCast(@alignCast(userdata));
3247
3248	var path_buffer: [PATH_MAX]u8 = undefined;
3249	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3250
3251	var cancel_region: CancelRegion = .init();
3252	defer cancel_region.deinit();
3253	while (true) {
3254	const thread = try cancel_region.awaitIoUring();
3255	thread.enqueue().* = .{
3256	.opcode = .UNLINKAT,
3257	.flags = 0,
3258	.ioprio = 0,
3259	.fd = dir.handle,
3260	.off = 0,
3261	.addr = @intFromPtr(sub_path_posix.ptr),
3262	.len = 0,
3263	.rw_flags = linux.AT.REMOVEDIR,
3264	.user_data = @intFromPtr(cancel_region.fiber),
3265	.buf_index = 0,
3266	.personality = 0,
3267	.splice_fd_in = 0,
3268	.addr3 = 0,
3269	.resv = 0,
3270	};
3271	ev.yield(null, .nothing);
3272	switch (cancel_region.errno()) {
3273	.SUCCESS => return,
3274	.INTR, .CANCELED => {},
3275	.ACCES => return error.AccessDenied,
3276	.PERM => return error.PermissionDenied,
3277	.BUSY => return error.FileBusy,
3278	.FAULT => |err| return errnoBug(err),
3279	.IO => return error.FileSystem,
3280	.ISDIR => |err| return errnoBug(err),
3281	.LOOP => return error.SymLinkLoop,
3282	.NAMETOOLONG => return error.NameTooLong,
3283	.NOENT => return error.FileNotFound,
3284	.NOTDIR => return error.NotDir,
3285	.NOMEM => return error.SystemResources,
3286	.ROFS => return error.ReadOnlyFileSystem,
3287	.EXIST => |err| return errnoBug(err),
3288	.NOTEMPTY => return error.DirNotEmpty,
3289	.ILSEQ => return error.BadPathName,
3290	.INVAL => |err| return errnoBug(err), // invalid flags, or pathname has . as last component
3291	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
3292	else => |err| return unexpectedErrno(err),
3293	}
3294	}
3295	}
3296
3297	fn dirRename(
3298	userdata: ?*anyopaque,
3299	old_dir: Dir,
3300	old_sub_path: []const u8,
3301	new_dir: Dir,
3302	new_sub_path: []const u8,
3303	) Dir.RenameError!void {
3304	const ev: *Evented = @ptrCast(@alignCast(userdata));
3305
3306	var old_path_buffer: [PATH_MAX]u8 = undefined;
3307	var new_path_buffer: [PATH_MAX]u8 = undefined;
3308
3309	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3310	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3311
3312	var cancel_region: CancelRegion = .init();
3313	defer cancel_region.deinit();
3314	return ev.renameat(
3315	&cancel_region,
3316	old_dir.handle,
3317	old_sub_path_posix,
3318	new_dir.handle,
3319	new_sub_path_posix,
3320	.{},
3321	);
3322	}
3323
3324	fn dirRenamePreserve(
3325	userdata: ?*anyopaque,
3326	old_dir: Dir,
3327	old_sub_path: []const u8,
3328	new_dir: Dir,
3329	new_sub_path: []const u8,
3330	) Dir.RenamePreserveError!void {
3331	const ev: *Evented = @ptrCast(@alignCast(userdata));
3332
3333	var old_path_buffer: [PATH_MAX]u8 = undefined;
3334	var new_path_buffer: [PATH_MAX]u8 = undefined;
3335
3336	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3337	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3338
3339	var cancel_region: CancelRegion = .init();
3340	defer cancel_region.deinit();
3341	return ev.renameat(
3342	&cancel_region,
3343	old_dir.handle,
3344	old_sub_path_posix,
3345	new_dir.handle,
3346	new_sub_path_posix,
3347	.{ .NOREPLACE = true },
3348	);
3349	}
3350
3351	fn dirSymLink(
3352	userdata: ?*anyopaque,
3353	dir: Dir,
3354	target_path: []const u8,
3355	sym_link_path: []const u8,
3356	flags: Dir.SymLinkFlags,
3357	) Dir.SymLinkError!void {
3358	const ev: *Evented = @ptrCast(@alignCast(userdata));
3359	_ = flags;
3360
3361	var target_path_buffer: [PATH_MAX]u8 = undefined;
3362	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
3363
3364	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
3365	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
3366
3367	var cancel_region: CancelRegion = .init();
3368	defer cancel_region.deinit();
3369	while (true) {
3370	const thread = try cancel_region.awaitIoUring();
3371	thread.enqueue().* = .{
3372	.opcode = .SYMLINKAT,
3373	.flags = 0,
3374	.ioprio = 0,
3375	.fd = dir.handle,
3376	.off = @intFromPtr(sym_link_path_posix.ptr),
3377	.addr = @intFromPtr(target_path_posix.ptr),
3378	.len = 0,
3379	.rw_flags = 0,
3380	.user_data = @intFromPtr(cancel_region.fiber),
3381	.buf_index = 0,
3382	.personality = 0,
3383	.splice_fd_in = 0,
3384	.addr3 = 0,
3385	.resv = 0,
3386	};
3387	ev.yield(null, .nothing);
3388	switch (cancel_region.errno()) {
3389	.SUCCESS => return,
3390	.INTR, .CANCELED => {},
3391	.FAULT => |err| return errnoBug(err),
3392	.INVAL => |err| return errnoBug(err),
3393	.ACCES => return error.AccessDenied,
3394	.PERM => return error.PermissionDenied,
3395	.DQUOT => return error.DiskQuota,
3396	.EXIST => return error.PathAlreadyExists,
3397	.IO => return error.FileSystem,
3398	.LOOP => return error.SymLinkLoop,
3399	.NAMETOOLONG => return error.NameTooLong,
3400	.NOENT => return error.FileNotFound,
3401	.NOTDIR => return error.NotDir,
3402	.NOMEM => return error.SystemResources,
3403	.NOSPC => return error.NoSpaceLeft,
3404	.ROFS => return error.ReadOnlyFileSystem,
3405	.ILSEQ => return error.BadPathName,
3406	else => |err| return unexpectedErrno(err),
3407	}
3408	}
3409	}
3410
3411	fn dirReadLink(
3412	userdata: ?*anyopaque,
3413	dir: Dir,
3414	sub_path: []const u8,
3415	buffer: []u8,
3416	) Dir.ReadLinkError!usize {
3417	const ev: *Evented = @ptrCast(@alignCast(userdata));
3418
3419	var sub_path_buffer: [PATH_MAX]u8 = undefined;
3420	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
3421
3422	var sync: CancelRegion.Sync = try .init(ev);
3423	defer sync.deinit(ev);
3424	while (true) {
3425	try sync.cancel_region.await(.nothing);
3426	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
3427	switch (linux.errno(rc)) {
3428	.SUCCESS => return @bitCast(rc),
3429	.INTR => {},
3430	.ACCES => return error.AccessDenied,
3431	.FAULT => |err| return errnoBug(err),
3432	.INVAL => return error.NotLink,
3433	.IO => return error.FileSystem,
3434	.LOOP => return error.SymLinkLoop,
3435	.NAMETOOLONG => return error.NameTooLong,
3436	.NOENT => return error.FileNotFound,
3437	.NOMEM => return error.SystemResources,
3438	.NOTDIR => return error.NotDir,
3439	.ILSEQ => return error.BadPathName,
3440	else => |err| return unexpectedErrno(err),
3441	}
3442	}
3443	}
3444
3445	fn dirSetOwner(
3446	userdata: ?*anyopaque,
3447	dir: Dir,
3448	owner: ?File.Uid,
3449	group: ?File.Gid,
3450	) Dir.SetOwnerError!void {
3451	const ev: *Evented = @ptrCast(@alignCast(userdata));
3452	var sync: CancelRegion.Sync = try .init(ev);
3453	defer sync.deinit(ev);
3454	try ev.fchownat(
3455	&sync,
3456	dir.handle,
3457	"",
3458	owner orelse std.math.maxInt(linux.uid_t),
3459	group orelse std.math.maxInt(linux.gid_t),
3460	linux.AT.EMPTY_PATH,
3461	);
3462	}
3463
3464	fn dirSetFileOwner(
3465	userdata: ?*anyopaque,
3466	dir: Dir,
3467	sub_path: []const u8,
3468	owner: ?File.Uid,
3469	group: ?File.Gid,
3470	options: Dir.SetFileOwnerOptions,
3471	) Dir.SetFileOwnerError!void {
3472	const ev: *Evented = @ptrCast(@alignCast(userdata));
3473	var path_buffer: [PATH_MAX]u8 = undefined;
3474	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3475	var sync: CancelRegion.Sync = try .init(ev);
3476	defer sync.deinit(ev);
3477	try ev.fchownat(
3478	&sync,
3479	dir.handle,
3480	sub_path_posix,
3481	owner orelse std.math.maxInt(linux.uid_t),
3482	group orelse std.math.maxInt(linux.gid_t),
3483	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3484	);
3485	}
3486
3487	fn dirSetPermissions(
3488	userdata: ?*anyopaque,
3489	dir: Dir,
3490	permissions: Dir.Permissions,
3491	) Dir.SetPermissionsError!void {
3492	const ev: *Evented = @ptrCast(@alignCast(userdata));
3493	var sync: CancelRegion.Sync = try .init(ev);
3494	defer sync.deinit(ev);
3495	ev.fchmodat(
3496	&sync,
3497	dir.handle,
3498	"",
3499	permissions.toMode(),
3500	linux.AT.EMPTY_PATH,
3501	) catch |err| switch (err) {
3502	error.NameTooLong => return errnoBug(.NAMETOOLONG),
3503	error.BadPathName => return errnoBug(.ILSEQ),
3504	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
3505	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
3506	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
3507	else => |e| return e,
3508	};
3509	}
3510
3511	fn dirSetFilePermissions(
3512	userdata: ?*anyopaque,
3513	dir: Dir,
3514	sub_path: []const u8,
3515	permissions: Dir.Permissions,
3516	options: Dir.SetFilePermissionsOptions,
3517	) Dir.SetFilePermissionsError!void {
3518	const ev: *Evented = @ptrCast(@alignCast(userdata));
3519	var path_buffer: [PATH_MAX]u8 = undefined;
3520	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3521	var sync: CancelRegion.Sync = try .init(ev);
3522	defer sync.deinit(ev);
3523	try ev.fchmodat(
3524	&sync,
3525	dir.handle,
3526	sub_path_posix,
3527	permissions.toMode(),
3528	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3529	);
3530	}
3531
3532	fn dirSetTimestamps(
3533	userdata: ?*anyopaque,
3534	dir: Dir,
3535	sub_path: []const u8,
3536	options: Dir.SetTimestampsOptions,
3537	) Dir.SetTimestampsError!void {
3538	const ev: *Evented = @ptrCast(@alignCast(userdata));
3539	var path_buffer: [PATH_MAX]u8 = undefined;
3540	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
3541	var cancel_region: CancelRegion.Sync = try .init(ev);
3542	defer cancel_region.deinit(ev);
3543	try ev.utimensat(
3544	&cancel_region,
3545	dir.handle,
3546	sub_path_posix,
3547	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
3548	setTimestampToPosix(options.access_timestamp),
3549	setTimestampToPosix(options.modify_timestamp),
3550	} else null,
3551	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
3552	);
3553	}
3554
3555	fn dirHardLink(
3556	userdata: ?*anyopaque,
3557	old_dir: Dir,
3558	old_sub_path: []const u8,
3559	new_dir: Dir,
3560	new_sub_path: []const u8,
3561	options: Dir.HardLinkOptions,
3562	) Dir.HardLinkError!void {
3563	const ev: *Evented = @ptrCast(@alignCast(userdata));
3564
3565	var old_path_buffer: [PATH_MAX]u8 = undefined;
3566	var new_path_buffer: [PATH_MAX]u8 = undefined;
3567
3568	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
3569	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
3570
3571	var cancel_region: CancelRegion = .init();
3572	defer cancel_region.deinit();
3573	return ev.linkat(
3574	&cancel_region,
3575	old_dir.handle,
3576	old_sub_path_posix,
3577	new_dir.handle,
3578	new_sub_path_posix,
3579	if (options.follow_symlinks) linux.AT.SYMLINK_FOLLOW else 0,
3580	);
3581	}
3582
3583	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
3584	const ev: *Evented = @ptrCast(@alignCast(userdata));
3585	var cancel_region: CancelRegion = .init();
3586	defer cancel_region.deinit();
3587	return ev.stat(&cancel_region, file.handle);
3588	}
3589
3590	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
3591	const ev: *Evented = @ptrCast(@alignCast(userdata));
3592	var cancel_region: CancelRegion = .init();
3593	defer cancel_region.deinit();
3594	while (true) {
3595	var statx_buf = std.mem.zeroes(linux.Statx);
3596	const thread = try cancel_region.awaitIoUring();
3597	thread.enqueue().* = .{
3598	.opcode = .STATX,
3599	.flags = 0,
3600	.ioprio = 0,
3601	.fd = file.handle,
3602	.off = @intFromPtr(&statx_buf),
3603	.addr = @intFromPtr(""),
3604	.len = @bitCast(linux.STATX{ .SIZE = true }),
3605	.rw_flags = linux.AT.EMPTY_PATH,
3606	.user_data = @intFromPtr(cancel_region.fiber),
3607	.buf_index = 0,
3608	.personality = 0,
3609	.splice_fd_in = 0,
3610	.addr3 = 0,
3611	.resv = 0,
3612	};
3613	ev.yield(null, .nothing);
3614	switch (cancel_region.errno()) {
3615	.SUCCESS => {
3616	if (!statx_buf.mask.SIZE) return error.Unexpected;
3617	return statx_buf.size;
3618	},
3619	.INTR, .CANCELED => {},
3620	.ACCES => |err| return errnoBug(err),
3621	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
3622	.FAULT => |err| return errnoBug(err),
3623	.INVAL => |err| return errnoBug(err),
3624	.LOOP => |err| return errnoBug(err),
3625	.NAMETOOLONG => |err| return errnoBug(err),
3626	.NOENT => |err| return errnoBug(err),
3627	.NOMEM => return error.SystemResources,
3628	.NOTDIR => |err| return errnoBug(err),
3629	else => |err| return unexpectedErrno(err),
3630	}
3631	}
3632	}
3633
3634	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
3635	const ev: *Evented = @ptrCast(@alignCast(userdata));
3636	var cancel_region: CancelRegion = .init();
3637	defer cancel_region.deinit();
3638	for (files) |file| ev.close(file.handle);
3639	}
3640
3641	fn fileWritePositional(
3642	userdata: ?*anyopaque,
3643	file: File,
3644	header: []const u8,
3645	data: []const []const u8,
3646	splat: usize,
3647	offset: u64,
3648	) File.WritePositionalError!usize {
3649	const ev: *Evented = @ptrCast(@alignCast(userdata));
3650
3651	var iovecs: [max_iovecs_len]iovec_const = undefined;
3652	var iovlen: iovlen_t = 0;
3653	addBuf(&iovecs, &iovlen, header);
3654	for (data[0 .. data.len - 1]) |bytes| addBuf(&iovecs, &iovlen, bytes);
3655	const pattern = data[data.len - 1];
3656	var backup_buffer: [splat_buffer_size]u8 = undefined;
3657	if (iovecs.len - iovlen != 0) switch (splat) {
3658	0 => {},
3659	1 => addBuf(&iovecs, &iovlen, pattern),
3660	else => switch (pattern.len) {
3661	0 => {},
3662	1 => {
3663	const splat_buffer = &backup_buffer;
3664	const memset_len = @min(splat_buffer.len, splat);
3665	const buf = splat_buffer[0..memset_len];
3666	@memset(buf, pattern[0]);
3667	addBuf(&iovecs, &iovlen, buf);
3668	var remaining_splat = splat - buf.len;
3669	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
3670	assert(buf.len == splat_buffer.len);
3671	addBuf(&iovecs, &iovlen, splat_buffer);
3672	remaining_splat -= splat_buffer.len;
3673	}
3674	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
3675	},
3676	else => for (0..@min(splat, iovecs.len - iovlen)) |_| {
3677	addBuf(&iovecs, &iovlen, pattern);
3678	},
3679	},
3680	};
3681
3682	var cancel_region: CancelRegion = .init();
3683	defer cancel_region.deinit();
3684	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
3685	}
3686
3687	/// This is either usize or u32. Since, either is fine, let's use the same
3688	/// `addBuf` function for both writing to a file and sending network messages.
3689	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
3690
3691	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
3692	// OS checks ptr addr before length so zero length vectors must be omitted.
3693	if (bytes.len == 0) return;
3694	if (v.len - i.* == 0) return;
3695	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
3696	i.* += 1;
3697	}
3698
3699	fn fileWriteFileStreaming(
3700	userdata: ?*anyopaque,
3701	file: File,
3702	header: []const u8,
3703	file_reader: *File.Reader,
3704	limit: Io.Limit,
3705	) File.Writer.WriteFileError!usize {
3706	const ev: *Evented = @ptrCast(@alignCast(userdata));
3707	_ = ev;
3708	_ = file;
3709	_ = header;
3710	_ = file_reader;
3711	_ = limit;
3712	return error.Unimplemented;
3713	}
3714
3715	fn fileWriteFilePositional(
3716	userdata: ?*anyopaque,
3717	file: File,
3718	header: []const u8,
3719	file_reader: *File.Reader,
3720	limit: Io.Limit,
3721	offset: u64,
3722	) File.WriteFilePositionalError!usize {
3723	const ev: *Evented = @ptrCast(@alignCast(userdata));
3724	_ = ev;
3725	_ = file;
3726	_ = header;
3727	_ = file_reader;
3728	_ = limit;
3729	_ = offset;
3730	return error.Unimplemented;
3731	}
3732
3733	fn fileReadPositional(
3734	userdata: ?*anyopaque,
3735	file: File,
3736	data: []const []u8,
3737	offset: u64,
3738	) File.ReadPositionalError!usize {
3739	const ev: *Evented = @ptrCast(@alignCast(userdata));
3740
3741	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
3742	var i: usize = 0;
3743	for (data) |buf| {
3744	if (iovecs_buffer.len - i == 0) break;
3745	if (buf.len > 0) {
3746	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
3747	i += 1;
3748	}
3749	}
3750	if (i == 0) return 0;
3751	const dest = iovecs_buffer[0..i];
3752	assert(dest[0].len > 0);
3753
3754	var cancel_region: CancelRegion = .init();
3755	defer cancel_region.deinit();
3756	return ev.preadv(&cancel_region, file.handle, dest, offset) catch |err| switch (err) {
3757	error.SocketUnconnected => return errnoBug(.NOTCONN), // not a socket
3758	error.ConnectionResetByPeer => return errnoBug(.CONNRESET), // not a socket
3759	else => |e| return e,
3760	};
3761	}
3762
3763	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
3764	const ev: *Evented = @ptrCast(@alignCast(userdata));
3765	var sync: CancelRegion.Sync = try .init(ev);
3766	defer sync.deinit(ev);
3767	try ev.lseek(&sync, file.handle, @bitCast(offset), linux.SEEK.CUR);
3768	}
3769
3770	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
3771	const ev: *Evented = @ptrCast(@alignCast(userdata));
3772	var sync: CancelRegion.Sync = try .init(ev);
3773	defer sync.deinit(ev);
3774	try ev.lseek(&sync, file.handle, offset, linux.SEEK.SET);
3775	}
3776
3777	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
3778	const ev: *Evented = @ptrCast(@alignCast(userdata));
3779	var cancel_region: CancelRegion = .init();
3780	defer cancel_region.deinit();
3781	while (true) {
3782	const thread = try cancel_region.awaitIoUring();
3783	thread.enqueue().* = .{
3784	.opcode = .FSYNC,
3785	.flags = 0,
3786	.ioprio = 0,
3787	.fd = file.handle,
3788	.off = 0,
3789	.addr = 0,
3790	.len = 0,
3791	.rw_flags = 0,
3792	.user_data = @intFromPtr(cancel_region.fiber),
3793	.buf_index = 0,
3794	.personality = 0,
3795	.splice_fd_in = 0,
3796	.addr3 = 0,
3797	.resv = 0,
3798	};
3799	ev.yield(null, .nothing);
3800	switch (cancel_region.errno()) {
3801	.SUCCESS => return,
3802	.INTR, .CANCELED => {},
3803	.BADF => |err| return errnoBug(err),
3804	.INVAL => |err| return errnoBug(err),
3805	.ROFS => |err| return errnoBug(err),
3806	.IO => return error.InputOutput,
3807	.NOSPC => return error.NoSpaceLeft,
3808	.DQUOT => return error.DiskQuota,
3809	else => |err| return unexpectedErrno(err),
3810	}
3811	}
3812	}
3813
3814	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
3815	const ev: *Evented = @ptrCast(@alignCast(userdata));
3816	var sync: CancelRegion.Sync = try .init(ev);
3817	defer sync.deinit(ev);
3818	while (true) {
3819	try sync.cancel_region.await(.nothing);
3820	var wsz: winsize = undefined;
3821	const rc = linux.ioctl(file.handle, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
3822	switch (linux.errno(rc)) {
3823	.SUCCESS => return true,
3824	.INTR => {},
3825	else => return false,
3826	}
3827	}
3828	}
3829
3830	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
3831	const ev: *Evented = @ptrCast(@alignCast(userdata));
3832	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
3833	}
3834
3835	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
3836	const ev: *Evented = @ptrCast(@alignCast(userdata));
3837	var cancel_region: CancelRegion = .init();
3838	defer cancel_region.deinit();
3839	while (true) {
3840	const thread = try cancel_region.awaitIoUring();
3841	thread.enqueue().* = .{
3842	.opcode = .FTRUNCATE,
3843	.flags = 0,
3844	.ioprio = 0,
3845	.fd = file.handle,
3846	.off = length,
3847	.addr = 0,
3848	.len = 0,
3849	.rw_flags = 0,
3850	.user_data = @intFromPtr(cancel_region.fiber),
3851	.buf_index = 0,
3852	.personality = 0,
3853	.splice_fd_in = 0,
3854	.addr3 = 0,
3855	.resv = 0,
3856	};
3857	ev.yield(null, .nothing);
3858	switch (cancel_region.errno()) {
3859	.SUCCESS => return,
3860	.INTR, .CANCELED => {},
3861	.FBIG => return error.FileTooBig,
3862	.IO => return error.InputOutput,
3863	.PERM => return error.PermissionDenied,
3864	.TXTBSY => return error.FileBusy,
3865	.BADF => |err| return errnoBug(err), // Handle not open for writing.
3866	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
3867	else => |err| return unexpectedErrno(err),
3868	}
3869	}
3870	}
3871
3872	fn fileSetOwner(
3873	userdata: ?*anyopaque,
3874	file: File,
3875	owner: ?File.Uid,
3876	group: ?File.Gid,
3877	) File.SetOwnerError!void {
3878	const ev: *Evented = @ptrCast(@alignCast(userdata));
3879	var sync: CancelRegion.Sync = try .init(ev);
3880	defer sync.deinit(ev);
3881	try ev.fchownat(
3882	&sync,
3883	file.handle,
3884	"",
3885	owner orelse std.math.maxInt(linux.uid_t),
3886	group orelse std.math.maxInt(linux.gid_t),
3887	linux.AT.EMPTY_PATH,
3888	);
3889	}
3890
3891	fn fileSetPermissions(
3892	userdata: ?*anyopaque,
3893	file: File,
3894	permissions: File.Permissions,
3895	) File.SetPermissionsError!void {
3896	const ev: *Evented = @ptrCast(@alignCast(userdata));
3897	var sync: CancelRegion.Sync = try .init(ev);
3898	defer sync.deinit(ev);
3899	ev.fchmodat(
3900	&sync,
3901	file.handle,
3902	"",
3903	permissions.toMode(),
3904	linux.AT.EMPTY_PATH,
3905	) catch |err| switch (err) {
3906	error.NameTooLong => return errnoBug(.NAMETOOLONG),
3907	error.BadPathName => return errnoBug(.ILSEQ),
3908	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
3909	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
3910	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
3911	else => |e| return e,
3912	};
3913	}
3914
3915	fn fileSetTimestamps(
3916	userdata: ?*anyopaque,
3917	file: File,
3918	options: File.SetTimestampsOptions,
3919	) File.SetTimestampsError!void {
3920	const ev: *Evented = @ptrCast(@alignCast(userdata));
3921	var sync: CancelRegion.Sync = try .init(ev);
3922	defer sync.deinit(ev);
3923	try ev.utimensat(
3924	&sync,
3925	file.handle,
3926	"",
3927	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
3928	setTimestampToPosix(options.access_timestamp),
3929	setTimestampToPosix(options.modify_timestamp),
3930	} else null,
3931	linux.AT.EMPTY_PATH,
3932	);
3933	}
3934
3935	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
3936	const ev: *Evented = @ptrCast(@alignCast(userdata));
3937	var sync: CancelRegion.Sync = try .init(ev);
3938	defer sync.deinit(ev);
3939	ev.flock(&sync, file.handle, lock, .blocking) catch |err| switch (err) {
3940	error.WouldBlock => unreachable, // blocking
3941	else => |e| return e,
3942	};
3943	}
3944
3945	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
3946	const ev: *Evented = @ptrCast(@alignCast(userdata));
3947	var sync: CancelRegion.Sync = try .init(ev);
3948	defer sync.deinit(ev);
3949	ev.flock(&sync, file.handle, lock, switch (lock) {
3950	.none => .blocking,
3951	.shared, .exclusive => .nonblocking,
3952	}) catch |err| switch (err) {
3953	error.WouldBlock => return false,
3954	else => |e| return e,
3955	};
3956	return true;
3957	}
3958
3959	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
3960	const ev: *Evented = @ptrCast(@alignCast(userdata));
3961	var sync: CancelRegion.Sync = .initBlocked(ev);
3962	defer sync.deinit(ev);
3963	ev.flock(&sync, file.handle, .none, .blocking) catch |err| switch (err) {
3964	error.Canceled => unreachable, // blocked
3965	error.WouldBlock => unreachable, // blocking
3966	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
3967	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
3968	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
3969	};
3970	}
3971
3972	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
3973	const ev: *Evented = @ptrCast(@alignCast(userdata));
3974	var sync: CancelRegion.Sync = try .init(ev);
3975	defer sync.deinit(ev);
3976	ev.flock(&sync, file.handle, .shared, .nonblocking) catch |err| switch (err) {
3977	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
3978	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
3979	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
3980	else => |e| return e,
3981	};
3982	}
3983
3984	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
3985	const ev: *Evented = @ptrCast(@alignCast(userdata));
3986	var sync: CancelRegion.Sync = try .init(ev);
3987	defer sync.deinit(ev);
3988	return ev.realPath(&sync, file.handle, out_buffer);
3989	}
3990
3991	fn fileHardLink(
3992	userdata: ?*anyopaque,
3993	file: File,
3994	new_dir: Dir,
3995	new_sub_path: []const u8,
3996	options: File.HardLinkOptions,
3997	) File.HardLinkError!void {
3998	const ev: *Evented = @ptrCast(@alignCast(userdata));
3999
4000	var new_path_buffer: [PATH_MAX]u8 = undefined;
4001	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
4002
4003	var cancel_region: CancelRegion = .init();
4004	defer cancel_region.deinit();
4005	return ev.linkat(
4006	&cancel_region,
4007	file.handle,
4008	"",
4009	new_dir.handle,
4010	new_sub_path_posix,
4011	linux.AT.EMPTY_PATH | @as(u32, if (options.follow_symlinks) linux.AT.SYMLINK_FOLLOW else 0),
4012	);
4013	}
4014
4015	fn fileMemoryMapCreate(
4016	userdata: ?*anyopaque,
4017	file: File,
4018	options: File.MemoryMap.CreateOptions,
4019	) File.MemoryMap.CreateError!File.MemoryMap {
4020	const ev: *Evented = @ptrCast(@alignCast(userdata));
4021
4022	const prot: linux.PROT = .{
4023	.READ = options.protection.read,
4024	.WRITE = options.protection.write,
4025	.EXEC = options.protection.execute,
4026	};
4027	const flags: linux.MAP = .{
4028	.TYPE = .SHARED_VALIDATE,
4029	.POPULATE = options.populate,
4030	};
4031
4032	const page_align = std.heap.page_size_min;
4033
4034	var sync: CancelRegion.Sync = try .init(ev);
4035	defer sync.deinit(ev);
4036	const contents = while (true) {
4037	try sync.cancel_region.await(.nothing);
4038	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
4039	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
4040	switch (linux.errno(rc)) {
4041	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
4042	.INTR => {},
4043	.ACCES => return error.AccessDenied,
4044	.AGAIN => return error.LockedMemoryLimitExceeded,
4045	.MFILE => return error.ProcessFdQuotaExceeded,
4046	.NFILE => return error.SystemFdQuotaExceeded,
4047	.NOMEM => return error.OutOfMemory,
4048	.PERM => return error.PermissionDenied,
4049	.OVERFLOW => return error.Unseekable,
4050	.BADF => |err| return errnoBug(err), // Always a race condition.
4051	.INVAL => |err| return errnoBug(err), // Invalid parameters to mmap()
4052	.OPNOTSUPP => |err| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
4053	else => |err| return unexpectedErrno(err),
4054	}
4055	};
4056	return .{
4057	.file = file,
4058	.offset = options.offset,
4059	.memory = contents,
4060	.section = {},
4061	};
4062	}
4063
4064	fn fileMemoryMapDestroy(userdata: ?*anyopaque, mm: *File.MemoryMap) void {
4065	const ev: *Evented = @ptrCast(@alignCast(userdata));
4066	_ = ev;
4067	const memory = mm.memory;
4068	if (memory.len == 0) return;
4069	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
4070	.SUCCESS => {},
4071	else => |err| if (builtin.mode == .debug)
4072	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
4073	}
4074	mm.* = undefined;
4075	}
4076
4077	fn fileMemoryMapSetLength(
4078	userdata: ?*anyopaque,
4079	mm: *File.MemoryMap,
4080	new_len: usize,
4081	) File.MemoryMap.SetLengthError!void {
4082	const ev: *Evented = @ptrCast(@alignCast(userdata));
4083
4084	const page_size = std.heap.pageSize();
4085	const alignment: Alignment = .fromByteUnits(page_size);
4086	const page_align = std.heap.page_size_min;
4087	const old_memory = mm.memory;
4088
4089	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
4090	mm.memory.len = new_len;
4091	return;
4092	}
4093	const flags: linux.MREMAP = .{ .MAYMOVE = true };
4094	const addr_hint: ?[*]const u8 = null;
4095	var sync: CancelRegion.Sync = try .init(ev);
4096	defer sync.deinit(ev);
4097	const new_memory = while (true) {
4098	try sync.cancel_region.await(.nothing);
4099	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
4100	switch (linux.errno(rc)) {
4101	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
4102	.INTR => {},
4103	.AGAIN => return error.LockedMemoryLimitExceeded,
4104	.NOMEM => return error.OutOfMemory,
4105	.INVAL => |err| return errnoBug(err),
4106	.FAULT => |err| return errnoBug(err),
4107	else => |err| return unexpectedErrno(err),
4108	}
4109	};
4110	mm.memory = new_memory;
4111	}
4112
4113	fn fileMemoryMapRead(userdata: ?*anyopaque, mm: *File.MemoryMap) File.ReadPositionalError!void {
4114	const ev: *Evented = @ptrCast(@alignCast(userdata));
4115	_ = ev;
4116	_ = mm;
4117	}
4118
4119	fn fileMemoryMapWrite(userdata: ?*anyopaque, mm: *File.MemoryMap) File.WritePositionalError!void {
4120	const ev: *Evented = @ptrCast(@alignCast(userdata));
4121	_ = ev;
4122	_ = mm;
4123	}
4124
4125	fn processExecutableOpen(
4126	userdata: ?*anyopaque,
4127	flags: Dir.OpenFileOptions,
4128	) process.OpenExecutableError!File {
4129	const ev: *Evented = @ptrCast(@alignCast(userdata));
4130	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
4131	}
4132
4133	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
4134	const ev: *Evented = @ptrCast(@alignCast(userdata));
4135	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch |err| switch (err) {
4136	error.UnsupportedReparsePointType => unreachable, // Windows-only
4137	error.NetworkNotFound => unreachable, // Windows-only
4138	error.FileBusy => unreachable, // Windows-only
4139	else => |e| return e,
4140	};
4141	}
4142
4143	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4144	const ev: *Evented = @ptrCast(@alignCast(userdata));
4145	const ev_io = ev.io();
4146	ev.stderr_mutex.lockUncancelable(ev_io);
4147	errdefer ev.stderr_mutex.unlock(ev_io);
4148	return ev.initLockedStderr(terminal_mode);
4149	}
4150
4151	fn tryLockStderr(
4152	userdata: ?*anyopaque,
4153	terminal_mode: ?Io.Terminal.Mode,
4154	) Io.Cancelable!?Io.LockedStderr {
4155	const ev: *Evented = @ptrCast(@alignCast(userdata));
4156	const ev_io = ev.io();
4157	if (!ev.stderr_mutex.tryLock()) return null;
4158	errdefer ev.stderr_mutex.unlock(ev_io);
4159	return try ev.initLockedStderr(terminal_mode);
4160	}
4161
4162	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
4163	if (!ev.stderr_writer_initialized) {
4164	const ev_io = ev.io();
4165	const cancel_protection = swapCancelProtection(ev, .blocked);
4166	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4167	ev.scanEnviron() catch |err| switch (err) {
4168	error.Canceled => unreachable, // blocked
4169	};
4170	const NO_COLOR = ev.environ.exist.NO_COLOR;
4171	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
4172	ev.stderr_mode = Io.Terminal.Mode.detect(
4173	ev_io,
4174	ev.stderr_writer.file,
4175	NO_COLOR,
4176	CLICOLOR_FORCE,
4177	) catch |err| switch (err) {
4178	error.Canceled => unreachable, // blocked
4179	};
4180	ev.stderr_writer_initialized = true;
4181	}
4182	return .{
4183	.file_writer = &ev.stderr_writer,
4184	.terminal_mode = terminal_mode orelse ev.stderr_mode,
4185	};
4186	}
4187
4188	fn unlockStderr(userdata: ?*anyopaque) void {
4189	const ev: *Evented = @ptrCast(@alignCast(userdata));
4190	if (ev.stderr_writer.err == null) ev.stderr_writer.interface.flush() catch {};
4191	if (ev.stderr_writer.err) |err| {
4192	switch (err) {
4193	error.Canceled => Thread.current().currentFiber().cancel_protection.recancel(),
4194	else => {},
4195	}
4196	ev.stderr_writer.err = null;
4197	}
4198	ev.stderr_writer.interface.end = 0;
4199	ev.stderr_writer.interface.buffer = &.{};
4200	ev.stderr_mutex.unlock(ev.io());
4201	}
4202
4203	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
4204	const ev: *Evented = @ptrCast(@alignCast(userdata));
4205	var sync: CancelRegion.Sync = try .init(ev);
4206	defer sync.deinit(ev);
4207	while (true) {
4208	try sync.cancel_region.await(.nothing);
4209	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
4210	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
4211	.INTR => {},
4212	.NOENT => return error.CurrentDirUnlinked,
4213	.RANGE => return error.NameTooLong,
4214	.FAULT => |err| return errnoBug(err),
4215	.INVAL => |err| return errnoBug(err),
4216	else => |err| return unexpectedErrno(err),
4217	}
4218	}
4219	}
4220
4221	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
4222	const ev: *Evented = @ptrCast(@alignCast(userdata));
4223	if (dir.handle == linux.AT.FDCWD) return;
4224	var sync: CancelRegion.Sync = try .init(ev);
4225	defer sync.deinit(ev);
4226	return fchdir(&sync, dir.handle);
4227	}
4228
4229	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) process.SetCurrentPathError!void {
4230	const ev: *Evented = @ptrCast(@alignCast(userdata));
4231	var path_buffer: [PATH_MAX]u8 = undefined;
4232	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
4233	var sync: CancelRegion.Sync = try .init(ev);
4234	defer sync.deinit(ev);
4235	return chdir(&sync, dir_path_posix);
4236	}
4237
4238	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
4239	const ev: *Evented = @ptrCast(@alignCast(userdata));
4240
4241	try ev.scanEnviron(); // for PATH
4242	const PATH = ev.environ.string.PATH orelse default_PATH;
4243
4244	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4245	defer arena_allocator.deinit();
4246	const arena = arena_allocator.allocator();
4247
4248	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4249	for (options.argv, 0..) |arg, i| argv_buf[i] = (try arena.dupeSentinel(u8, arg, 0)).ptr;
4250
4251	const env_block = env_block: {
4252	const prog_fd: i32 = -1;
4253	if (options.environ_map) |environ_map| break :env_block try environ_map.createPosixBlock(arena, .{
4254	.zig_progress_fd = prog_fd,
4255	});
4256	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4257	.zig_progress_fd = prog_fd,
4258	});
4259	};
4260
4261	var sync: CancelRegion.Sync = try .init(ev);
4262	defer sync.deinit(ev);
4263	return execv(&sync, options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
4264	}
4265
4266	fn processReplacePath(
4267	userdata: ?*anyopaque,
4268	dir: Dir,
4269	options: process.ReplaceOptions,
4270	) process.ReplaceError {
4271	const ev: *Evented = @ptrCast(@alignCast(userdata));
4272	_ = ev;
4273	_ = dir;
4274	_ = options;
4275	@panic("TODO processReplacePath");
4276	}
4277
4278	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
4279	const ev: *Evented = @ptrCast(@alignCast(userdata));
4280	const spawned = try ev.spawn(options);
4281	var cancel_region: CancelRegion = .initBlocked();
4282	defer cancel_region.deinit();
4283	defer ev.closeAsync(spawned.err_fd);
4284
4285	// Wait for the child to report any errors in or before `execvpe`.
4286	var child_err: ForkBailError = undefined;
4287	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch |read_err| {
4288	switch (read_err) {
4289	error.Canceled => unreachable, // blocked
4290	error.EndOfStream => {
4291	// Write end closed by CLOEXEC at the time of the `execvpe` call,
4292	// indicating success.
4293	},
4294	else => {
4295	// Problem reading the error from the error reporting pipe. We
4296	// don't know if the child is alive or dead. Better to assume it is
4297	// alive so the resource does not risk being leaked.
4298	},
4299	}
4300	return .{
4301	.id = spawned.pid,
4302	.thread_handle = {},
4303	.stdin = spawned.stdin,
4304	.stdout = spawned.stdout,
4305	.stderr = spawned.stderr,
4306	.request_resource_usage_statistics = options.request_resource_usage_statistics,
4307	};
4308	};
4309	return child_err;
4310	}
4311
4312	fn processSpawnPath(
4313	userdata: ?*anyopaque,
4314	dir: Dir,
4315	options: process.SpawnOptions,
4316	) process.SpawnError!process.Child {
4317	const ev: *Evented = @ptrCast(@alignCast(userdata));
4318	_ = ev;
4319	_ = dir;
4320	_ = options;
4321	@panic("TODO processSpawnPath");
4322	}
4323
4324	const prog_fileno = @max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO);
4325
4326	const Spawned = struct {
4327	pid: pid_t,
4328	err_fd: fd_t,
4329	stdin: ?File,
4330	stdout: ?File,
4331	stderr: ?File,
4332	};
4333	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
4334	var cancel_region: CancelRegion = .init();
4335	defer cancel_region.deinit();
4336
4337	// The child process does need to access (one end of) these pipes. However,
4338	// we must initially set CLOEXEC to avoid a race condition. If another thread
4339	// is racing to spawn a different child process, we don't want it to inherit
4340	// these FDs in any scenario; that would mean that, for instance, calls to
4341	// `poll` from the parent would not report the child's stdout as closing when
4342	// expected, since the other child may retain a reference to the write end of
4343	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
4344	// need to do something in the new child to make sure we preserve the reference
4345	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
4346	// turns out, we `dup2` everything anyway, so there's no need!
4347	const pipe_flags: linux.O = .{ .CLOEXEC = true };
4348
4349	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
4350	errdefer if (options.stdin == .pipe) {
4351	ev.destroyPipe(stdin_pipe);
4352	};
4353
4354	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
4355	errdefer if (options.stdout == .pipe) {
4356	ev.destroyPipe(stdout_pipe);
4357	};
4358
4359	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
4360	errdefer if (options.stderr == .pipe) {
4361	ev.destroyPipe(stderr_pipe);
4362	};
4363
4364	const any_ignore =
4365	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
4366	const dev_null_fd = if (any_ignore) try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
4367	.ACCMODE = .RDWR,
4368	}) else undefined;
4369
4370	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
4371	// We use CLOEXEC for the same reason as in `pipe_flags`.
4372	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
4373	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
4374	break :pipe pipe;
4375	} else .{ -1, -1 };
4376	errdefer ev.destroyPipe(prog_pipe);
4377
4378	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
4379	defer arena_allocator.deinit();
4380	const arena = arena_allocator.allocator();
4381
4382	// The POSIX standard does not allow malloc() between fork() and execve(),
4383	// and this allocator may be a libc allocator.
4384	// I have personally observed the child process deadlocking when it tries
4385	// to call malloc() due to a heap allocation between fork() and execve(),
4386	// in musl v1.1.24.
4387	// Additionally, we want to reduce the number of possible ways things
4388	// can fail between fork() and execve().
4389	// Therefore, we do all the allocation for the execve() before the fork().
4390	// This means we must do the null-termination of argv and env vars here.
4391	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
4392	for (options.argv, 0..) |arg, i| argv_buf[i] = (try arena.dupeSentinel(u8, arg, 0)).ptr;
4393
4394	const env_block = env_block: {
4395	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
4396	if (options.environ_map) |environ_map| break :env_block try environ_map.createPosixBlock(arena, .{
4397	.zig_progress_fd = prog_fd,
4398	});
4399	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
4400	.zig_progress_fd = prog_fd,
4401	});
4402	};
4403
4404	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
4405	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
4406	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
4407	errdefer ev.destroyPipe(err_pipe);
4408
4409	try ev.scanEnviron(); // for PATH
4410	const PATH = ev.environ.string.PATH orelse default_PATH;
4411
4412	const pid_result: pid_t = fork: {
4413	const rc = linux.fork();
4414	switch (linux.errno(rc)) {
4415	.SUCCESS => break :fork @intCast(rc),
4416	.AGAIN => return error.SystemResources,
4417	.NOMEM => return error.SystemResources,
4418	.NOSYS => return error.OperationUnsupported,
4419	else => |err| return unexpectedErrno(err),
4420	}
4421	};
4422
4423	if (pid_result == 0) {
4424	defer comptime unreachable; // We are the child.
4425	// Note that the parent uring is no longer accessible, so we must no longer reference `ev`.
4426	var sync: CancelRegion.Sync = .{ .cancel_region = .initBlocked() };
4427	const err = setUpChild(&sync, .{
4428	.stdin_pipe = stdin_pipe[0],
4429	.stdout_pipe = stdout_pipe[1],
4430	.stderr_pipe = stderr_pipe[1],
4431	.dev_null_fd = dev_null_fd,
4432	.prog_pipe = prog_pipe[1],
4433	.argv_buf = argv_buf,
4434	.env_block = env_block,
4435	.PATH = PATH,
4436	.spawn = options,
4437	});
4438	writeAllSync(&sync, err_pipe[1], @ptrCast(&err)) catch {};
4439	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
4440	exit(1);
4441	}
4442
4443	const pid: pid_t = @intCast(pid_result); // We are the parent.
4444	errdefer comptime unreachable; // The child is forked; we must not error from now on
4445
4446	ev.closeAsync(err_pipe[1]); // make sure only the child holds the write end open
4447
4448	if (options.stdin == .pipe) ev.closeAsync(stdin_pipe[0]);
4449	if (options.stdout == .pipe) ev.closeAsync(stdout_pipe[1]);
4450	if (options.stderr == .pipe) ev.closeAsync(stderr_pipe[1]);
4451
4452	if (prog_pipe[1] != -1) ev.closeAsync(prog_pipe[1]);
4453
4454	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
4455
4456	return .{
4457	.pid = pid,
4458	.err_fd = err_pipe[0],
4459	.stdin = switch (options.stdin) {
4460	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
4461	else => null,
4462	},
4463	.stdout = switch (options.stdout) {
4464	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
4465	else => null,
4466	},
4467	.stderr = switch (options.stderr) {
4468	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
4469	else => null,
4470	},
4471	};
4472	}
4473
4474	pub const PipeError = error{
4475	SystemFdQuotaExceeded,
4476	ProcessFdQuotaExceeded,
4477	} || Io.UnexpectedError;
4478	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
4479	var fds: [2]fd_t = undefined;
4480	switch (linux.errno(linux.pipe2(&fds, flags))) {
4481	.SUCCESS => return fds,
4482	.INVAL => |err| return errnoBug(err), // Invalid flags
4483	.NFILE => return error.SystemFdQuotaExceeded,
4484	.MFILE => return error.ProcessFdQuotaExceeded,
4485	else => |err| return unexpectedErrno(err),
4486	}
4487	}
4488	fn destroyPipe(ev: *Evented, pipe: [2]fd_t) void {
4489	if (pipe[0] != -1) ev.closeAsync(pipe[0]);
4490	if (pipe[0] != pipe[1]) ev.closeAsync(pipe[1]);
4491	}
4492
4493	/// Errors that can occur between fork() and execv()
4494	const ForkBailError = process.SetCurrentDirError || ChdirError ||
4495	process.SpawnError || process.ReplaceError;
4496	fn setUpChild(sync: *CancelRegion.Sync, options: struct {
4497	stdin_pipe: fd_t,
4498	stdout_pipe: fd_t,
4499	stderr_pipe: fd_t,
4500	dev_null_fd: fd_t,
4501	prog_pipe: fd_t,
4502	argv_buf: [:null]?[*:0]const u8,
4503	env_block: process.Environ.Block,
4504	PATH: []const u8,
4505	spawn: process.SpawnOptions,
4506	}) ForkBailError {
4507	try setUpChildIo(
4508	sync,
4509	options.spawn.stdin,
4510	options.stdin_pipe,
4511	linux.STDIN_FILENO,
4512	options.dev_null_fd,
4513	);
4514	try setUpChildIo(
4515	sync,
4516	options.spawn.stdout,
4517	options.stdout_pipe,
4518	linux.STDOUT_FILENO,
4519	options.dev_null_fd,
4520	);
4521	try setUpChildIo(
4522	sync,
4523	options.spawn.stderr,
4524	options.stderr_pipe,
4525	linux.STDERR_FILENO,
4526	options.dev_null_fd,
4527	);
4528
4529	switch (options.spawn.cwd) {
4530	.inherit => {},
4531	.dir => |cwd_dir| try fchdir(sync, cwd_dir.handle),
4532	.path => |cwd_path| {
4533	var cwd_path_buffer: [PATH_MAX]u8 = undefined;
4534	const cwd_path_posix = try pathToPosix(cwd_path, &cwd_path_buffer);
4535	try chdir(sync, cwd_path_posix);
4536	},
4537	}
4538
4539	// Must happen after fchdir above, the cwd file descriptor might be
4540	// equal to prog_fileno and be clobbered by this dup2 call.
4541	if (options.prog_pipe != -1) try dup2(sync, options.prog_pipe, prog_fileno);
4542
4543	if (options.spawn.gid) |gid| {
4544	switch (linux.errno(linux.setregid(gid, gid))) {
4545	.SUCCESS => {},
4546	.AGAIN => return error.ResourceLimitReached,
4547	.INVAL => return error.InvalidUserId,
4548	.PERM => return error.PermissionDenied,
4549	else => return error.Unexpected,
4550	}
4551	}
4552
4553	if (options.spawn.uid) |uid| {
4554	switch (linux.errno(linux.setreuid(uid, uid))) {
4555	.SUCCESS => {},
4556	.AGAIN => return error.ResourceLimitReached,
4557	.INVAL => return error.InvalidUserId,
4558	.PERM => return error.PermissionDenied,
4559	else => return error.Unexpected,
4560	}
4561	}
4562
4563	if (options.spawn.pgid) |pid| {
4564	switch (linux.errno(linux.setpgid(0, pid))) {
4565	.SUCCESS => {},
4566	.ACCES => return error.ProcessAlreadyExec,
4567	.INVAL => return error.InvalidProcessGroupId,
4568	.PERM => return error.PermissionDenied,
4569	else => return error.Unexpected,
4570	}
4571	}
4572
4573	if (options.spawn.start_suspended) {
4574	switch (linux.errno(linux.kill(0, .STOP))) {
4575	.SUCCESS => {},
4576	.PERM => return error.PermissionDenied,
4577	else => return error.Unexpected,
4578	}
4579	}
4580
4581	return execv(
4582	sync,
4583	options.spawn.expand_arg0,
4584	options.argv_buf.ptr[0].?,
4585	options.argv_buf.ptr,
4586	options.env_block,
4587	options.PATH,
4588	);
4589	}
4590
4591	fn setUpChildIo(
4592	sync: *CancelRegion.Sync,
4593	stdio: process.SpawnOptions.StdIo,
4594	pipe_fd: fd_t,
4595	std_fileno: i32,
4596	dev_null_fd: fd_t,
4597	) !void {
4598	switch (stdio) {
4599	.pipe => try dup2(sync, pipe_fd, std_fileno),
4600	.close => _ = linux.close(std_fileno),
4601	.inherit => {},
4602	.ignore => try dup2(sync, dev_null_fd, std_fileno),
4603	.file => |file| try dup2(sync, file.handle, std_fileno),
4604	}
4605	}
4606
4607	pub const DupError = error{
4608	ProcessFdQuotaExceeded,
4609	SystemResources,
4610	} || Io.UnexpectedError || Io.Cancelable;
4611	pub fn dup2(sync: *CancelRegion.Sync, old_fd: fd_t, new_fd: fd_t) DupError!void {
4612	while (true) {
4613	try sync.cancel_region.await(.nothing);
4614	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
4615	.SUCCESS => return,
4616	.BUSY, .INTR => {},
4617	.INVAL => |err| return errnoBug(err), // invalid parameters
4618	.BADF => |err| return errnoBug(err), // use after free
4619	.MFILE => return error.ProcessFdQuotaExceeded,
4620	.NOMEM => return error.SystemResources,
4621	else => |err| return unexpectedErrno(err),
4622	}
4623	}
4624	}
4625
4626	fn execv(
4627	sync: *CancelRegion.Sync,
4628	arg0_expand: process.ArgExpansion,
4629	file: [*:0]const u8,
4630	child_argv: [*:null]?[*:0]const u8,
4631	env_block: process.Environ.PosixBlock,
4632	PATH: []const u8,
4633	) process.ReplaceError {
4634	const file_slice = std.mem.sliceTo(file, 0);
4635	if (std.mem.findScalar(u8, file_slice, '/') != null)
4636	return execvPath(sync, file, child_argv, env_block);
4637
4638	// Use of PATH_MAX here is valid as the path_buf will be passed
4639	// directly to the operating system in posixExecvPath.
4640	var path_buf: [PATH_MAX]u8 = undefined;
4641	var it = std.mem.tokenizeScalar(u8, PATH, ':');
4642	var seen_eacces = false;
4643	var err: process.ReplaceError = error.FileNotFound;
4644
4645	// In case of expanding arg0 we must put it back if we return with an error.
4646	const prev_arg0 = child_argv[0];
4647	defer switch (arg0_expand) {
4648	.expand => child_argv[0] = prev_arg0,
4649	.no_expand => {},
4650	};
4651
4652	while (it.next()) |search_path| {
4653	const path_len = search_path.len + file_slice.len + 1;
4654	if (path_buf.len < path_len + 1) return error.NameTooLong;
4655	@memcpy(path_buf[0..search_path.len], search_path);
4656	path_buf[search_path.len] = '/';
4657	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
4658	path_buf[path_len] = 0;
4659	const full_path = path_buf[0..path_len :0].ptr;
4660	switch (arg0_expand) {
4661	.expand => child_argv[0] = full_path,
4662	.no_expand => {},
4663	}
4664	err = execvPath(sync, full_path, child_argv, env_block);
4665	switch (err) {
4666	error.AccessDenied => seen_eacces = true,
4667	error.FileNotFound, error.NotDir => {},
4668	else => |e| return e,
4669	}
4670	}
4671	if (seen_eacces) return error.AccessDenied;
4672	return err;
4673	}
4674	/// This function ignores PATH environment variable.
4675	pub fn execvPath(
4676	sync: *CancelRegion.Sync,
4677	path: [*:0]const u8,
4678	child_argv: [*:null]const ?[*:0]const u8,
4679	env_block: process.Environ.PosixBlock,
4680	) process.ReplaceError {
4681	try sync.cancel_region.await(.nothing);
4682	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
4683	.FAULT => |err| return errnoBug(err), // Bad pointer parameter.
4684	.@"2BIG" => return error.SystemResources,
4685	.MFILE => return error.ProcessFdQuotaExceeded,
4686	.NAMETOOLONG => return error.NameTooLong,
4687	.NFILE => return error.SystemFdQuotaExceeded,
4688	.NOMEM => return error.SystemResources,
4689	.ACCES => return error.AccessDenied,
4690	.PERM => return error.PermissionDenied,
4691	.INVAL => return error.InvalidExe,
4692	.NOEXEC => return error.InvalidExe,
4693	.IO => return error.FileSystem,
4694	.LOOP => return error.FileSystem,
4695	.ISDIR => return error.IsDir,
4696	.NOENT => return error.FileNotFound,
4697	.NOTDIR => return error.NotDir,
4698	.TXTBSY => return error.FileBusy,
4699	.LIBBAD => return error.InvalidExe,
4700	else => |err| return unexpectedErrno(err),
4701	}
4702	}
4703
4704	fn childWait(userdata: ?*anyopaque, child: *process.Child) process.Child.WaitError!process.Child.Term {
4705	const ev: *Evented = @ptrCast(@alignCast(userdata));
4706
4707	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
4708	defer maybe_sync.deinit(ev);
4709	defer ev.childCleanup(child);
4710
4711	const pid = child.id.?;
4712	var info: linux.siginfo_t = undefined;
4713	while (true) {
4714	const thread = try maybe_sync.cancel_region.awaitIoUring();
4715	thread.enqueue().* = .{
4716	.opcode = .WAITID,
4717	.flags = 0,
4718	.ioprio = 0,
4719	.fd = pid,
4720	.off = @intFromPtr(&info),
4721	.addr = 0,
4722	.len = @backingInt(linux.P.PID),
4723	.rw_flags = 0,
4724	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4725	.buf_index = 0,
4726	.personality = 0,
4727	.splice_fd_in = linux.W.EXITED |
4728	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
4729	.addr3 = 0,
4730	.resv = 0,
4731	};
4732	ev.yield(null, .nothing);
4733	switch (maybe_sync.cancel_region.errno()) {
4734	.SUCCESS => {
4735	if (child.request_resource_usage_statistics) {
4736	const sync = try maybe_sync.enterSync(ev);
4737	while (true) {
4738	try sync.cancel_region.await(.nothing);
4739	var rusage: linux.rusage = undefined;
4740	switch (linux.errno(linux.waitid(
4741	.PID,
4742	pid,
4743	&info,
4744	linux.W.EXITED | linux.W.NOHANG,
4745	&rusage,
4746	))) {
4747	.SUCCESS => {
4748	child.resource_usage_statistics.rusage = rusage;
4749	break;
4750	},
4751	.INTR, .CANCELED => {},
4752	.CHILD => |err| return errnoBug(err), // Double-free.
4753	else => |err| return unexpectedErrno(err),
4754	}
4755	}
4756	}
4757	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
4758	const code: linux.CLD = @fromBackingInt(@intCast(info.code));
4759	return switch (code) {
4760	.EXITED => .{ .exited = @truncate(status) },
4761	.KILLED, .DUMPED => .{ .signal = @fromBackingInt(@intCast(status)) },
4762	.TRAPPED, .STOPPED => .{ .stopped = @fromBackingInt(@intCast(status)) },
4763	_, .CONTINUED => .{ .unknown = status },
4764	};
4765	},
4766	.INTR, .CANCELED => {},
4767	.CHILD => |err| return errnoBug(err), // Double-free.
4768	else => |err| return unexpectedErrno(err),
4769	}
4770	}
4771	}
4772
4773	fn childKill(userdata: ?*anyopaque, child: *process.Child) void {
4774	const ev: *Evented = @ptrCast(@alignCast(userdata));
4775
4776	var maybe_sync: CancelRegion.Sync.Maybe = .{ .sync = .initBlocked(ev) };
4777	defer maybe_sync.deinit(ev);
4778	defer ev.childCleanup(child);
4779
4780	const pid = child.id.?;
4781	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
4782	.SUCCESS => break,
4783	.INTR => {},
4784	.PERM => return,
4785	.INVAL => |err| return errnoBug(err) catch {},
4786	.SRCH => |err| return errnoBug(err) catch {},
4787	else => |err| return unexpectedErrno(err) catch {},
4788	};
4789	maybe_sync.leaveSync(ev);
4790
4791	var info: linux.siginfo_t = undefined;
4792	while (true) {
4793	const thread = maybe_sync.cancel_region.awaitIoUring() catch |err| switch (err) {
4794	error.Canceled => unreachable, // blocked
4795	};
4796	thread.enqueue().* = .{
4797	.opcode = .WAITID,
4798	.flags = 0,
4799	.ioprio = 0,
4800	.fd = pid,
4801	.off = @intFromPtr(&info),
4802	.addr = 0,
4803	.len = @backingInt(linux.P.PID),
4804	.rw_flags = 0,
4805	.user_data = @intFromPtr(maybe_sync.cancel_region.fiber),
4806	.buf_index = 0,
4807	.personality = 0,
4808	.splice_fd_in = linux.W.EXITED,
4809	.addr3 = 0,
4810	.resv = 0,
4811	};
4812	ev.yield(null, .nothing);
4813	switch (maybe_sync.cancel_region.errno()) {
4814	.SUCCESS => return,
4815	.INTR, .CANCELED => {},
4816	.CHILD => |err| return errnoBug(err) catch {}, // Double-free.
4817	else => |err| return unexpectedErrno(err) catch {},
4818	}
4819	}
4820	}
4821
4822	fn childCleanup(ev: *Evented, child: *process.Child) void {
4823	if (child.stdin) |*stdin| {
4824	ev.closeAsync(stdin.handle);
4825	child.stdin = null;
4826	}
4827	if (child.stdout) |*stdout| {
4828	ev.closeAsync(stdout.handle);
4829	child.stdout = null;
4830	}
4831	if (child.stderr) |*stderr| {
4832	ev.closeAsync(stderr.handle);
4833	child.stderr = null;
4834	}
4835	child.id = null;
4836	}
4837
4838	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
4839	const ev: *Evented = @ptrCast(@alignCast(userdata));
4840	const cancel_protection = swapCancelProtection(ev, .blocked);
4841	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
4842	ev.scanEnviron() catch |err| switch (err) {
4843	error.Canceled => unreachable, // blocked
4844	};
4845	return ev.environ.zig_progress_file;
4846	}
4847
4848	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
4849	const ev_io = ev.io();
4850	try ev.environ_mutex.lock(ev_io);
4851	defer ev.environ_mutex.unlock(ev_io);
4852	if (ev.environ_initialized) return;
4853	ev.environ.scan(ev.allocator());
4854	ev.environ_initialized = true;
4855	}
4856
4857	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
4858	const ev: *Evented = @ptrCast(@alignCast(userdata));
4859	_ = ev;
4860	const clock_id = clockToPosix(clock);
4861	var timespec: linux.timespec = undefined;
4862	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
4863	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
4864	.INVAL => return error.ClockUnavailable,
4865	else => |err| return unexpectedErrno(err),
4866	};
4867	}
4868
4869	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
4870	const ev: *Evented = @ptrCast(@alignCast(userdata));
4871	_ = ev;
4872	var tp: linux.timespec = undefined;
4873	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
4874	.SUCCESS => return timestampFromPosix(&tp),
4875	else => return .zero,
4876	}
4877	}
4878
4879	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
4880	const ev: *Evented = @ptrCast(@alignCast(userdata));
4881
4882	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
4883	.none => .{
4884	.{
4885	.sec = std.math.maxInt(i64),
4886	.nsec = std.time.ns_per_s - 1,
4887	},
4888	.awake,
4889	linux.IORING_TIMEOUT_ABS,
4890	},
4891	.duration => |duration| {
4892	const ns = duration.raw.toNanoseconds();
4893	break :timespec .{
4894	.{
4895	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
4896	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
4897	},
4898	duration.clock,
4899	0,
4900	};
4901	},
4902	.deadline => |deadline| {
4903	const ns = deadline.raw.toNanoseconds();
4904	break :timespec .{
4905	.{
4906	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
4907	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
4908	},
4909	deadline.clock,
4910	linux.IORING_TIMEOUT_ABS,
4911	};
4912	},
4913	};
4914	var cancel_region: CancelRegion = .init();
4915	defer cancel_region.deinit();
4916	const thread = try cancel_region.awaitIoUring();
4917	thread.enqueue().* = .{
4918	.opcode = .TIMEOUT,
4919	.flags = 0,
4920	.ioprio = 0,
4921	.fd = 0,
4922	.off = 0,
4923	.addr = @intFromPtr(&timespec),
4924	.len = 1,
4925	.rw_flags = timeout_flags | @as(u32, switch (clock) {
4926	.real => linux.IORING_TIMEOUT_REALTIME,
4927	else => 0,
4928	.boot => linux.IORING_TIMEOUT_BOOTTIME,
4929	}),
4930	.user_data = @intFromPtr(cancel_region.fiber),
4931	.buf_index = 0,
4932	.personality = 0,
4933	.splice_fd_in = 0,
4934	.addr3 = 0,
4935	.resv = 0,
4936	};
4937	ev.yield(null, .nothing);
4938	// Handles SUCCESS as well as clock not available and unexpected
4939	// errors. The user had a chance to check clock resolution before
4940	// getting here, which would have reported 0, making this a legal
4941	// amount of time to sleep.
4942	}
4943
4944	fn random(userdata: ?*anyopaque, buffer: []u8) void {
4945	const ev: *Evented = @ptrCast(@alignCast(userdata));
4946	var thread: *Thread = .current();
4947	if (!thread.csprng.isInitialized()) {
4948	@branchHint(.unlikely);
4949	var seed: [Csprng.seed_len]u8 = undefined;
4950	{
4951	const ev_io = ev.io();
4952	ev.csprng_mutex.lockUncancelable(ev_io);
4953	defer ev.csprng_mutex.unlock(ev_io);
4954	if (!ev.csprng.isInitialized()) {
4955	@branchHint(.unlikely);
4956	var cancel_region: CancelRegion = .initBlocked();
4957	defer cancel_region.deinit();
4958	ev.urandomReadAll(&cancel_region, &seed) catch |err| switch (err) {
4959	error.Canceled => unreachable, // blocked
4960	else => fallbackSeed(ev, &seed),
4961	};
4962	ev.csprng.rng = .init(seed);
4963	thread = .current();
4964	}
4965	ev.csprng.rng.fill(&seed);
4966	}
4967	if (!thread.csprng.isInitialized()) {
4968	@branchHint(.likely);
4969	thread.csprng.rng = .init(seed);
4970	} else thread.csprng.rng.addEntropy(&seed);
4971	}
4972	thread.csprng.rng.fill(buffer);
4973	}
4974
4975	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
4976	const ev: *Evented = @ptrCast(@alignCast(userdata));
4977	if (buffer.len == 0) return;
4978	var cancel_region: CancelRegion = .init();
4979	defer cancel_region.deinit();
4980	ev.urandomReadAll(&cancel_region, buffer) catch |err| switch (err) {
4981	error.Canceled => |e| return e,
4982	else => return error.EntropyUnavailable,
4983	};
4984	}
4985
4986	fn netListenIpUnavailable(
4987	userdata: ?*anyopaque,
4988	address: *const net.IpAddress,
4989	options: net.IpAddress.ListenOptions,
4990	) net.IpAddress.ListenError!net.Socket {
4991	const ev: *Evented = @ptrCast(@alignCast(userdata));
4992	_ = ev;
4993	_ = address;
4994	_ = options;
4995	return error.NetworkDown;
4996	}
4997
4998	fn netAcceptUnavailable(
4999	userdata: ?*anyopaque,
5000	listen_handle: net.Socket.Handle,
5001	options: net.Server.AcceptOptions,
5002	) net.Server.AcceptError!net.Socket {
5003	const ev: *Evented = @ptrCast(@alignCast(userdata));
5004	_ = ev;
5005	_ = listen_handle;
5006	_ = options;
5007	return error.NetworkDown;
5008	}
5009
5010	fn netBindIp(
5011	userdata: ?*anyopaque,
5012	address: *const net.IpAddress,
5013	options: net.IpAddress.BindOptions,
5014	) net.IpAddress.BindError!net.Socket {
5015	const ev: *Evented = @ptrCast(@alignCast(userdata));
5016	const family = posixAddressFamily(address);
5017	var maybe_sync: CancelRegion.Sync.Maybe = .{ .cancel_region = .init() };
5018	defer maybe_sync.deinit(ev);
5019	const socket_fd = try ev.socket(&maybe_sync.cancel_region, family, options);
5020	errdefer ev.closeAsync(socket_fd);
5021	var storage: PosixAddress = undefined;
5022	var addr_len = addressToPosix(address, &storage);
5023	try ev.bind(&maybe_sync.cancel_region, socket_fd, &storage.any, addr_len);
5024	if (options.allow_broadcast) try ev.setsockopt(&maybe_sync.cancel_region, socket_fd, linux.SOL.SOCKET, linux.SO.BROADCAST, 1);
5025	try ev.getsockname(try maybe_sync.enterSync(ev), socket_fd, &storage.any, &addr_len);
5026	return .{ .handle = socket_fd, .address = addressFromPosix(&storage) };
5027	}
5028
5029	fn netConnectIpUnavailable(
5030	userdata: ?*anyopaque,
5031	address: *const net.IpAddress,
5032	options: net.IpAddress.ConnectOptions,
5033	) net.IpAddress.ConnectError!net.Socket {
5034	const ev: *Evented = @ptrCast(@alignCast(userdata));
5035	_ = ev;
5036	_ = address;
5037	_ = options;
5038	return error.NetworkDown;
5039	}
5040
5041	fn netListenUnixUnavailable(
5042	userdata: ?*anyopaque,
5043	address: *const net.UnixAddress,
5044	options: net.UnixAddress.ListenOptions,
5045	) net.UnixAddress.ListenError!net.Socket.Handle {
5046	const ev: *Evented = @ptrCast(@alignCast(userdata));
5047	_ = ev;
5048	_ = address;
5049	_ = options;
5050	return error.AddressFamilyUnsupported;
5051	}
5052
5053	fn netConnectUnixUnavailable(
5054	userdata: ?*anyopaque,
5055	address: *const net.UnixAddress,
5056	) net.UnixAddress.ConnectError!net.Socket.Handle {
5057	const ev: *Evented = @ptrCast(@alignCast(userdata));
5058	_ = ev;
5059	_ = address;
5060	return error.AddressFamilyUnsupported;
5061	}
5062
5063	fn netSocketCreatePairUnavailable(
5064	userdata: ?*anyopaque,
5065	options: net.Socket.CreatePairOptions,
5066	) net.Socket.CreatePairError![2]net.Socket {
5067	_ = userdata;
5068	_ = options;
5069	return error.OperationUnsupported;
5070	}
5071
5072	fn netReceive(
5073	ev: *Evented,
5074	cancel_region: *CancelRegion,
5075	handle: net.Socket.Handle,
5076	message_buffer: []net.IncomingMessage,
5077	data_buffer: []u8,
5078	flags: net.ReceiveFlags,
5079	) struct { ?net.Socket.ReceiveError, usize } {
5080	var message_i: usize = 0;
5081	var data_i: usize = 0;
5082
5083	while (true) {
5084	if (message_buffer.len - message_i == 0) return .{ null, message_i };
5085	const message = &message_buffer[message_i];
5086	const remaining_data_buffer = data_buffer[data_i..];
5087	var storage: PosixAddress = undefined;
5088	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
5089	var msg: linux.msghdr = .{
5090	.name = &storage.any,
5091	.namelen = @sizeOf(PosixAddress),
5092	.iov = (&iov)[0..1],
5093	.iovlen = 1,
5094	.control = message.control.ptr,
5095	.controllen = @intCast(message.control.len),
5096	.flags = undefined,
5097	};
5098
5099	const thread = cancel_region.awaitIoUring() catch |err| return .{ err, message_i };
5100	thread.enqueue().* = .{
5101	.opcode = .RECVMSG,
5102	.flags = 0,
5103	.ioprio = 0,
5104	.fd = handle,
5105	.off = 0,
5106	.addr = @intFromPtr(&msg),
5107	.len = 0,
5108	.rw_flags = linux.MSG.NOSIGNAL |
5109	@as(u32, if (flags.oob) linux.MSG.OOB else 0) |
5110	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) |
5111	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
5112	.user_data = @intFromPtr(cancel_region.fiber),
5113	.buf_index = 0,
5114	.personality = 0,
5115	.splice_fd_in = 0,
5116	.addr3 = 0,
5117	.resv = 0,
5118	};
5119	ev.yield(null, .nothing);
5120	const completion = cancel_region.completion();
5121	switch (completion.errno()) {
5122	.SUCCESS => {
5123	const data = remaining_data_buffer[0..@intCast(completion.result)];
5124	data_i += data.len;
5125	message.* = .{
5126	.from = addressFromPosix(&storage),
5127	.data = data,
5128	.control = if (msg.control) |ptr| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
5129	.flags = .{
5130	.eor = msg.flags & linux.MSG.EOR != 0,
5131	.trunc = msg.flags & linux.MSG.TRUNC != 0,
5132	.ctrunc = msg.flags & linux.MSG.CTRUNC != 0,
5133	.oob = msg.flags & linux.MSG.OOB != 0,
5134	.errqueue = msg.flags & linux.MSG.ERRQUEUE != 0,
5135	},
5136	};
5137	message_i += 1;
5138	continue;
5139	},
5140	.AGAIN => unreachable,
5141	.INTR, .CANCELED => {},
5142	.BADF => |err| return .{ errnoBug(err), message_i },
5143	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
5144	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
5145	.FAULT => |err| return .{ errnoBug(err), message_i },
5146	.INVAL => |err| return .{ errnoBug(err), message_i },
5147	.NOBUFS => return .{ error.SystemResources, message_i },
5148	.NOMEM => return .{ error.SystemResources, message_i },
5149	.NOTCONN => return .{ error.SocketUnconnected, message_i },
5150	.NOTSOCK => |err| return .{ errnoBug(err), message_i },
5151	.MSGSIZE => return .{ error.MessageOversize, message_i },
5152	.PIPE => return .{ error.SocketUnconnected, message_i },
5153	.OPNOTSUPP => |err| return .{ errnoBug(err), message_i },
5154	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
5155	.TIMEDOUT => return .{ error.ConnectionTimedOut, message_i },
5156	.NETDOWN => return .{ error.NetworkDown, message_i },
5157	else => |err| return .{ unexpectedErrno(err), message_i },
5158	}
5159	}
5160	}
5161
5162	fn netWriteFileUnavailable(
5163	userdata: ?*anyopaque,
5164	socket_handle: net.Socket.Handle,
5165	header: []const u8,
5166	file_reader: *File.Reader,
5167	limit: Io.Limit,
5168	) net.Stream.Writer.WriteFileError!usize {
5169	const ev: *Evented = @ptrCast(@alignCast(userdata));
5170	_ = ev;
5171	_ = socket_handle;
5172	_ = header;
5173	_ = file_reader;
5174	_ = limit;
5175	return error.Unimplemented;
5176	}
5177
5178	fn netClose(userdata: ?*anyopaque, sockets: []const net.Socket) void {
5179	const ev: *Evented = @ptrCast(@alignCast(userdata));
5180	for (sockets) |sock| ev.close(sock.handle);
5181	}
5182
5183	fn netShutdown(
5184	userdata: ?*anyopaque,
5185	handle: net.Socket.Handle,
5186	how: net.ShutdownHow,
5187	) net.ShutdownError!void {
5188	const ev: *Evented = @ptrCast(@alignCast(userdata));
5189	var cancel_region: CancelRegion = .init();
5190	defer cancel_region.deinit();
5191	while (true) {
5192	const thread = try cancel_region.awaitIoUring();
5193	thread.enqueue().* = .{
5194	.opcode = .SHUTDOWN,
5195	.flags = 0,
5196	.ioprio = 0,
5197	.fd = handle,
5198	.off = 0,
5199	.addr = 0,
5200	.len = switch (how) {
5201	.recv => linux.SHUT.RD,
5202	.send => linux.SHUT.WR,
5203	.both => linux.SHUT.RDWR,
5204	},
5205	.rw_flags = 0,
5206	.user_data = @intFromPtr(cancel_region.fiber),
5207	.buf_index = 0,
5208	.personality = 0,
5209	.splice_fd_in = 0,
5210	.addr3 = 0,
5211	.resv = 0,
5212	};
5213	ev.yield(null, .nothing);
5214	switch (cancel_region.errno()) {
5215	.SUCCESS => return,
5216	.INTR, .CANCELED => {},
5217	.BADF, .NOTSOCK, .INVAL => |err| return errnoBug(err),
5218	.NOTCONN => return error.SocketUnconnected,
5219	.NOBUFS => return error.SystemResources,
5220	else => |err| return unexpectedErrno(err),
5221	}
5222	}
5223	}
5224
5225	fn netInterfaceNameResolveUnavailable(
5226	userdata: ?*anyopaque,
5227	name: *const net.Interface.Name,
5228	) net.Interface.Name.ResolveError!net.Interface {
5229	const ev: *Evented = @ptrCast(@alignCast(userdata));
5230	_ = ev;
5231	_ = name;
5232	return error.InterfaceNotFound;
5233	}
5234
5235	fn netInterfaceNameUnavailable(
5236	userdata: ?*anyopaque,
5237	interface: net.Interface,
5238	) net.Interface.NameError!net.Interface.Name {
5239	const ev: *Evented = @ptrCast(@alignCast(userdata));
5240	_ = ev;
5241	_ = interface;
5242	return error.Unexpected;
5243	}
5244
5245	fn netLookupUnavailable(
5246	userdata: ?*anyopaque,
5247	host_name: net.HostName,
5248	resolved: *Io.Queue(net.HostName.LookupResult),
5249	options: net.HostName.LookupOptions,
5250	) net.HostName.LookupError!void {
5251	const ev: *Evented = @ptrCast(@alignCast(userdata));
5252	_ = host_name;
5253	_ = options;
5254	resolved.close(ev.io());
5255	return error.NetworkDown;
5256	}
5257
5258	fn bind(
5259	ev: *Evented,
5260	cancel_region: *CancelRegion,
5261	socket_fd: fd_t,
5262	addr: *const linux.sockaddr,
5263	addr_len: linux.socklen_t,
5264	) !void {
5265	while (true) {
5266	const thread = try cancel_region.awaitIoUring();
5267	thread.enqueue().* = .{
5268	.opcode = .BIND,
5269	.flags = 0,
5270	.ioprio = 0,
5271	.fd = socket_fd,
5272	.off = addr_len,
5273	.addr = @intFromPtr(addr),
5274	.len = 0,
5275	.rw_flags = 0,
5276	.user_data = @intFromPtr(cancel_region.fiber),
5277	.buf_index = 0,
5278	.personality = 0,
5279	.splice_fd_in = 0,
5280	.addr3 = 0,
5281	.resv = 0,
5282	};
5283	ev.yield(null, .nothing);
5284	switch (cancel_region.errno()) {
5285	.SUCCESS => return,
5286	.INTR, .CANCELED => {},
5287	.ACCES => return error.AccessDenied,
5288	.ADDRINUSE => return error.AddressInUse,
5289	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
5290	.INVAL => |err| return errnoBug(err), // invalid parameters
5291	.NOTSOCK => |err| return errnoBug(err), // invalid `sockfd`
5292	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
5293	.ADDRNOTAVAIL => return error.AddressUnavailable,
5294	.FAULT => |err| return errnoBug(err), // invalid `addr` pointer
5295	.NOMEM => return error.SystemResources,
5296	else => |err| return unexpectedErrno(err),
5297	}
5298	}
5299	}
5300
5301	fn chdir(sync: *CancelRegion.Sync, path: [*:0]const u8) ChdirError!void {
5302	while (true) {
5303	try sync.cancel_region.await(.nothing);
5304	switch (linux.errno(linux.chdir(path))) {
5305	.SUCCESS => return,
5306	.INTR => {},
5307	.ACCES => return error.AccessDenied,
5308	.IO => return error.FileSystem,
5309	.LOOP => return error.SymLinkLoop,
5310	.NAMETOOLONG => return error.NameTooLong,
5311	.NOENT => return error.FileNotFound,
5312	.NOMEM => return error.SystemResources,
5313	.NOTDIR => return error.NotDir,
5314	.ILSEQ => return error.BadPathName,
5315	.FAULT => |err| return errnoBug(err),
5316	else => |err| return unexpectedErrno(err),
5317	}
5318	}
5319	}
5320
5321	fn close(ev: *Evented, fd: fd_t) void {
5322	var cancel_region: CancelRegion = .initBlocked();
5323	defer cancel_region.deinit();
5324	const thread = cancel_region.awaitIoUring() catch |err| switch (err) {
5325	error.Canceled => unreachable, // blocked
5326	};
5327	thread.enqueue().* = .{
5328	.opcode = .CLOSE,
5329	.flags = 0,
5330	.ioprio = 0,
5331	.fd = fd,
5332	.off = 0,
5333	.addr = 0,
5334	.len = 0,
5335	.rw_flags = 0,
5336	.user_data = @intFromPtr(cancel_region.fiber),
5337	.buf_index = 0,
5338	.personality = 0,
5339	.splice_fd_in = 0,
5340	.addr3 = 0,
5341	.resv = 0,
5342	};
5343	ev.yield(null, .nothing);
5344	switch (cancel_region.errno()) {
5345	.BADF => recoverableOsBugDetected(), // Always a race condition.
5346	.INTR => {}, // This is still a success. See https://github.com/ziglang/zig/issues/2425
5347	else => {},
5348	}
5349	}
5350
5351	fn closeAsync(ev: *Evented, fd: fd_t) void {
5352	_ = ev;
5353	const thread: *Thread = .current();
5354	thread.enqueue().* = .{
5355	.opcode = .CLOSE,
5356	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
5357	.ioprio = 0,
5358	.fd = fd,
5359	.off = 0,
5360	.addr = 0,
5361	.len = 0,
5362	.rw_flags = 0,
5363	.user_data = @backingInt(Completion.Userdata.close),
5364	.buf_index = 0,
5365	.personality = 0,
5366	.splice_fd_in = 0,
5367	.addr3 = 0,
5368	.resv = 0,
5369	};
5370	}
5371
5372	fn fchdir(sync: *CancelRegion.Sync, dir: fd_t) process.SetCurrentDirError!void {
5373	if (dir == linux.AT.FDCWD) return;
5374	while (true) {
5375	try sync.cancel_region.await(.nothing);
5376	switch (linux.errno(linux.fchdir(dir))) {
5377	.SUCCESS => return,
5378	.INTR => {},
5379	.ACCES => return error.AccessDenied,
5380	.NOTDIR => return error.NotDir,
5381	.IO => return error.FileSystem,
5382	.BADF => |err| return errnoBug(err),
5383	else => |err| return unexpectedErrno(err),
5384	}
5385	}
5386	}
5387
5388	fn fchmodat(
5389	ev: *Evented,
5390	sync: *CancelRegion.Sync,
5391	dir: fd_t,
5392	path: [*:0]const u8,
5393	mode: linux.mode_t,
5394	flags: u32,
5395	) Dir.SetFilePermissionsError!void {
5396	_ = ev;
5397	while (true) {
5398	try sync.cancel_region.await(.nothing);
5399	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
5400	.SUCCESS => return,
5401	.INTR => {},
5402	.BADF => |err| return errnoBug(err),
5403	.FAULT => |err| return errnoBug(err),
5404	.INVAL => |err| return errnoBug(err),
5405	.ACCES => return error.AccessDenied,
5406	.IO => return error.InputOutput,
5407	.LOOP => return error.SymLinkLoop,
5408	.NOENT => return error.FileNotFound,
5409	.NOMEM => return error.SystemResources,
5410	.NOTDIR => return error.FileNotFound,
5411	.OPNOTSUPP => return error.OperationUnsupported,
5412	.PERM => return error.PermissionDenied,
5413	.ROFS => return error.ReadOnlyFileSystem,
5414	else => |err| return unexpectedErrno(err),
5415	}
5416	}
5417	}
5418
5419	fn fchownat(
5420	ev: *Evented,
5421	sync: *CancelRegion.Sync,
5422	dir: fd_t,
5423	path: [*:0]const u8,
5424	owner: linux.uid_t,
5425	group: linux.gid_t,
5426	flags: u32,
5427	) File.SetOwnerError!void {
5428	_ = ev;
5429	while (true) {
5430	try sync.cancel_region.await(.nothing);
5431	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
5432	.SUCCESS => return,
5433	.INTR => {},
5434	.BADF => |err| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
5435	.FAULT => |err| return errnoBug(err),
5436	.INVAL => |err| return errnoBug(err),
5437	.ACCES => return error.AccessDenied,
5438	.IO => return error.InputOutput,
5439	.LOOP => return error.SymLinkLoop,
5440	.NOENT => return error.FileNotFound,
5441	.NOMEM => return error.SystemResources,
5442	.NOTDIR => return error.FileNotFound,
5443	.PERM => return error.PermissionDenied,
5444	.ROFS => return error.ReadOnlyFileSystem,
5445	else => |err| return unexpectedErrno(err),
5446	}
5447	}
5448	}
5449
5450	fn flock(
5451	ev: *Evented,
5452	sync: *CancelRegion.Sync,
5453	fd: fd_t,
5454	op: File.Lock,
5455	blocking: enum { blocking, nonblocking },
5456	) (File.LockError || error{WouldBlock})!void {
5457	while (true) {
5458	try sync.cancel_region.await(.nothing);
5459	switch (linux.errno(linux.flock(fd, LOCK.NB | @as(i32, switch (op) {
5460	.none => LOCK.UN,
5461	.shared => LOCK.SH,
5462	.exclusive => LOCK.EX,
5463	})))) {
5464	.SUCCESS => return,
5465	.INTR => {},
5466	.BADF => |err| return errnoBug(err),
5467	.INVAL => |err| return errnoBug(err), // invalid parameters
5468	.NOLCK => return error.SystemResources,
5469	.AGAIN => {
5470	const thread = try sync.cancel_region.awaitIoUring();
5471	thread.enqueue().* = .{
5472	.opcode = .NOP,
5473	.flags = 0,
5474	.ioprio = 0,
5475	.fd = 0,
5476	.off = 0,
5477	.addr = 0,
5478	.len = 0,
5479	.rw_flags = 0,
5480	.user_data = @intFromPtr(sync.cancel_region.fiber),
5481	.buf_index = 0,
5482	.personality = 0,
5483	.splice_fd_in = 0,
5484	.addr3 = 0,
5485	.resv = 0,
5486	};
5487	ev.yield(null, .nothing);
5488	switch (sync.cancel_region.errno()) {
5489	.SUCCESS, .INTR, .CANCELED => {},
5490	else => unreachable,
5491	}
5492	switch (blocking) {
5493	.blocking => continue,
5494	.nonblocking => return error.WouldBlock,
5495	}
5496	},
5497	.OPNOTSUPP => return error.FileLocksUnsupported,
5498	else => |err| return unexpectedErrno(err),
5499	}
5500	}
5501	}
5502
5503	fn getsockname(
5504	ev: *Evented,
5505	sync: *CancelRegion.Sync,
5506	socket_fd: fd_t,
5507	addr: *linux.sockaddr,
5508	addr_len: *linux.socklen_t,
5509	) !void {
5510	_ = ev;
5511	while (true) {
5512	try sync.cancel_region.await(.nothing);
5513	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
5514	.SUCCESS => return,
5515	.INTR => {},
5516	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
5517	.FAULT => |err| return errnoBug(err),
5518	.INVAL => |err| return errnoBug(err), // invalid parameters
5519	.NOTSOCK => |err| return errnoBug(err), // always a race condition
5520	.NOBUFS => return error.SystemResources,
5521	else => |err| return unexpectedErrno(err),
5522	}
5523	}
5524	}
5525
5526	fn linkat(
5527	ev: *Evented,
5528	cancel_region: *CancelRegion,
5529	old_dir: fd_t,
5530	old_path: [*:0]const u8,
5531	new_dir: fd_t,
5532	new_path: [*:0]const u8,
5533	flags: u32,
5534	) File.HardLinkError!void {
5535	// allowed flags: https://man7.org/linux/man-pages/man2/linkat.2.html
5536	assert(flags & ~(@as(u32, linux.AT.SYMLINK_FOLLOW | linux.AT.EMPTY_PATH)) == 0);
5537	while (true) {
5538	const thread = try cancel_region.awaitIoUring();
5539	thread.enqueue().* = .{
5540	.opcode = .LINKAT,
5541	.flags = 0,
5542	.ioprio = 0,
5543	.fd = old_dir,
5544	.off = @intFromPtr(new_path),
5545	.addr = @intFromPtr(old_path),
5546	.len = @bitCast(new_dir),
5547	.rw_flags = flags,
5548	.user_data = @intFromPtr(cancel_region.fiber),
5549	.buf_index = 0,
5550	.personality = 0,
5551	.splice_fd_in = 0,
5552	.addr3 = 0,
5553	.resv = 0,
5554	};
5555	ev.yield(null, .nothing);
5556	switch (cancel_region.errno()) {
5557	.SUCCESS => return,
5558	.INTR, .CANCELED => {},
5559	.ACCES => return error.AccessDenied,
5560	.DQUOT => return error.DiskQuota,
5561	.EXIST => return error.PathAlreadyExists,
5562	.IO => return error.HardwareFailure,
5563	.LOOP => return error.SymLinkLoop,
5564	.MLINK => return error.LinkQuotaExceeded,
5565	.NAMETOOLONG => return error.NameTooLong,
5566	.NOENT => return error.FileNotFound,
5567	.NOMEM => return error.SystemResources,
5568	.NOSPC => return error.NoSpaceLeft,
5569	.NOTDIR => return error.NotDir,
5570	.PERM => return error.PermissionDenied,
5571	.ROFS => return error.ReadOnlyFileSystem,
5572	.XDEV => return error.CrossDevice,
5573	.ILSEQ => return error.BadPathName,
5574	.FAULT => |err| return errnoBug(err),
5575	.INVAL => |err| return errnoBug(err),
5576	else => |err| return unexpectedErrno(err),
5577	}
5578	}
5579	}
5580
5581	fn lseek(
5582	ev: *Evented,
5583	sync: *CancelRegion.Sync,
5584	fd: fd_t,
5585	offset: u64,
5586	whence: u32,
5587	) File.SeekError!void {
5588	_ = ev;
5589	while (true) {
5590	try sync.cancel_region.await(.nothing);
5591	var result: u64 = undefined;
5592	switch (linux.errno(switch (@sizeOf(usize)) {
5593	else => comptime unreachable,
5594	4 => linux.llseek(fd, offset, &result, whence),
5595	8 => linux.lseek(fd, @bitCast(offset), whence),
5596	})) {
5597	.SUCCESS => return,
5598	.INTR => {},
5599	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
5600	.INVAL => return error.Unseekable,
5601	.OVERFLOW => return error.Unseekable,
5602	.SPIPE => return error.Unseekable,
5603	.NXIO => return error.Unseekable,
5604	else => |err| return unexpectedErrno(err),
5605	}
5606	}
5607	}
5608
5609	fn openat(
5610	ev: *Evented,
5611	cancel_region: *CancelRegion,
5612	dir: fd_t,
5613	path: [*:0]const u8,
5614	flags: linux.O,
5615	mode: linux.mode_t,
5616	) !fd_t {
5617	var mut_flags = flags;
5618	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
5619	while (true) {
5620	const thread = try cancel_region.awaitIoUring();
5621	thread.enqueue().* = .{
5622	.opcode = .OPENAT,
5623	.flags = 0,
5624	.ioprio = 0,
5625	.fd = dir,
5626	.off = 0,
5627	.addr = @intFromPtr(path),
5628	.len = mode,
5629	.rw_flags = @bitCast(mut_flags),
5630	.user_data = @intFromPtr(cancel_region.fiber),
5631	.buf_index = 0,
5632	.personality = 0,
5633	.splice_fd_in = 0,
5634	.addr3 = 0,
5635	.resv = 0,
5636	};
5637	ev.yield(null, .nothing);
5638	const completion = cancel_region.completion();
5639	switch (completion.errno()) {
5640	.SUCCESS => return completion.result,
5641	.INTR, .CANCELED => {},
5642	.FAULT => |err| return errnoBug(err),
5643	.INVAL => return error.BadPathName,
5644	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
5645	.ACCES => return error.AccessDenied,
5646	.FBIG => return error.FileTooBig,
5647	.OVERFLOW => return error.FileTooBig,
5648	.ISDIR => return error.IsDir,
5649	.LOOP => return error.SymLinkLoop,
5650	.MFILE => return error.ProcessFdQuotaExceeded,
5651	.NAMETOOLONG => return error.NameTooLong,
5652	.NFILE => return error.SystemFdQuotaExceeded,
5653	.NODEV => return error.NoDevice,
5654	.NOENT => return error.FileNotFound,
5655	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
5656	.NOMEM => return error.SystemResources,
5657	.NOSPC => return error.NoSpaceLeft,
5658	.NOTDIR => return error.NotDir,
5659	.PERM => return error.PermissionDenied,
5660	.EXIST => return error.PathAlreadyExists,
5661	.BUSY => return error.DeviceBusy,
5662	// This can be triggered by file locking and TMPFILE, but those
5663	// flags are mutually exclusive.
5664	.OPNOTSUPP => return error.OperationUnsupported,
5665	.AGAIN => return error.WouldBlock,
5666	.TXTBSY => return error.FileBusy,
5667	.NXIO => return error.NoDevice,
5668	.ROFS => return error.ReadOnlyFileSystem,
5669	.ILSEQ => return error.BadPathName,
5670	else => |err| return unexpectedErrno(err),
5671	}
5672	}
5673	}
5674
5675	fn preadv(
5676	ev: *Evented,
5677	cancel_region: *CancelRegion,
5678	fd: fd_t,
5679	iov: []const iovec,
5680	offset: ?u64,
5681	) File.Reader.Error!usize {
5682	if (iov.len == 0) return 0;
5683	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
5684	while (true) {
5685	const thread = try cancel_region.awaitIoUring();
5686	thread.enqueue().* = .{
5687	.opcode = if (gather) .READV else .READ,
5688	.flags = 0,
5689	.ioprio = 0,
5690	.fd = fd,
5691	.off = offset orelse std.math.maxInt(u64),
5692	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5693	.len = @intCast(if (gather) iov.len else iov[0].len),
5694	.rw_flags = 0,
5695	.user_data = @intFromPtr(cancel_region.fiber),
5696	.buf_index = 0,
5697	.personality = 0,
5698	.splice_fd_in = 0,
5699	.addr3 = 0,
5700	.resv = 0,
5701	};
5702	ev.yield(null, .nothing);
5703	const completion = cancel_region.completion();
5704	switch (completion.errno()) {
5705	.SUCCESS => return @as(u32, @bitCast(completion.result)),
5706	.INTR, .CANCELED => {},
5707	.INVAL => |err| return errnoBug(err),
5708	.FAULT => |err| return errnoBug(err),
5709	.AGAIN => return error.WouldBlock,
5710	.BADF => |err| return errnoBug(err), // File descriptor used after closed
5711	.IO => return error.InputOutput,
5712	.ISDIR => return error.IsDir,
5713	.NOBUFS => return error.SystemResources,
5714	.NOMEM => return error.SystemResources,
5715	.NOTCONN => return error.SocketUnconnected,
5716	.CONNRESET => return error.ConnectionResetByPeer,
5717	else => |err| return unexpectedErrno(err),
5718	}
5719	}
5720	}
5721
5722	fn pwritev(
5723	ev: *Evented,
5724	cancel_region: *CancelRegion,
5725	fd: fd_t,
5726	iov: []const iovec_const,
5727	offset: ?u64,
5728	) File.Writer.Error!usize {
5729	if (iov.len == 0) return 0;
5730	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
5731	while (true) {
5732	const thread = try cancel_region.awaitIoUring();
5733	thread.enqueue().* = .{
5734	.opcode = if (scatter) .WRITEV else .WRITE,
5735	.flags = 0,
5736	.ioprio = 0,
5737	.fd = fd,
5738	.off = offset orelse std.math.maxInt(u64),
5739	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
5740	.len = @intCast(if (scatter) iov.len else iov[0].len),
5741	.rw_flags = 0,
5742	.user_data = @intFromPtr(cancel_region.fiber),
5743	.buf_index = 0,
5744	.personality = 0,
5745	.splice_fd_in = 0,
5746	.addr3 = 0,
5747	.resv = 0,
5748	};
5749	ev.yield(null, .nothing);
5750	const completion = cancel_region.completion();
5751	switch (completion.errno()) {
5752	.SUCCESS => return @as(u32, @bitCast(completion.result)),
5753	.INTR, .CANCELED => {},
5754	.INVAL => |err| return errnoBug(err),
5755	.FAULT => |err| return errnoBug(err),
5756	.AGAIN => return error.WouldBlock,
5757	.BADF => return error.NotOpenForWriting, // Can be a race condition.
5758	.DESTADDRREQ => |err| return errnoBug(err), // `connect` was never called.
5759	.DQUOT => return error.DiskQuota,
5760	.FBIG => return error.FileTooBig,
5761	.IO => return error.InputOutput,
5762	.NOSPC => return error.NoSpaceLeft,
5763	.PERM => return error.PermissionDenied,
5764	.PIPE => return error.BrokenPipe,
5765	.CONNRESET => |err| return errnoBug(err), // Not a socket handle.
5766	.BUSY => return error.DeviceBusy,
5767	else => |err| return unexpectedErrno(err),
5768	}
5769	}
5770	}
5771
5772	fn readAll(
5773	ev: *Evented,
5774	cancel_region: *CancelRegion,
5775	fd: fd_t,
5776	buffer: []u8,
5777	) (File.Reader.Error || error{EndOfStream})!void {
5778	var index: usize = 0;
5779	while (buffer.len - index != 0) {
5780	const len = try ev.preadv(cancel_region, fd, &.{
5781	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
5782	}, null);
5783	if (len == 0) return error.EndOfStream;
5784	index += len;
5785	}
5786	}
5787
5788	fn realPath(
5789	ev: *Evented,
5790	sync: *CancelRegion.Sync,
5791	fd: fd_t,
5792	out_buffer: []u8,
5793	) File.RealPathError!usize {
5794	_ = ev;
5795	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
5796	const proc_path = std.mem.printSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
5797	unreachable;
5798	while (true) {
5799	try sync.cancel_region.await(.nothing);
5800	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
5801	switch (linux.errno(rc)) {
5802	.SUCCESS => return rc,
5803	.INTR => {},
5804	.ACCES => return error.AccessDenied,
5805	.FAULT => |err| return errnoBug(err),
5806	.IO => return error.FileSystem,
5807	.LOOP => return error.SymLinkLoop,
5808	.NAMETOOLONG => return error.NameTooLong,
5809	.NOENT => return error.FileNotFound,
5810	.NOMEM => return error.SystemResources,
5811	.NOTDIR => return error.NotDir,
5812	.ILSEQ => |err| return errnoBug(err),
5813	else => |err| return unexpectedErrno(err),
5814	}
5815	}
5816	}
5817
5818	fn renameat(
5819	ev: *Evented,
5820	cancel_region: *CancelRegion,
5821	old_dir: fd_t,
5822	old_path: [*:0]const u8,
5823	new_dir: fd_t,
5824	new_path: [*:0]const u8,
5825	flags: linux.RENAME,
5826	) Dir.RenameError!void {
5827	while (true) {
5828	const thread = try cancel_region.awaitIoUring();
5829	thread.enqueue().* = .{
5830	.opcode = .RENAMEAT,
5831	.flags = 0,
5832	.ioprio = 0,
5833	.fd = old_dir,
5834	.off = @intFromPtr(new_path),
5835	.addr = @intFromPtr(old_path),
5836	.len = @bitCast(new_dir),
5837	.rw_flags = @bitCast(flags),
5838	.user_data = @intFromPtr(cancel_region.fiber),
5839	.buf_index = 0,
5840	.personality = 0,
5841	.splice_fd_in = 0,
5842	.addr3 = 0,
5843	.resv = 0,
5844	};
5845	ev.yield(null, .nothing);
5846	switch (cancel_region.errno()) {
5847	.SUCCESS => return,
5848	.INTR, .CANCELED => {},
5849	.ACCES => return error.AccessDenied,
5850	.PERM => return error.PermissionDenied,
5851	.BUSY => return error.FileBusy,
5852	.DQUOT => return error.DiskQuota,
5853	.ISDIR => return error.IsDir,
5854	.IO => return error.HardwareFailure,
5855	.LOOP => return error.SymLinkLoop,
5856	.MLINK => return error.LinkQuotaExceeded,
5857	.NAMETOOLONG => return error.NameTooLong,
5858	.NOENT => return error.FileNotFound,
5859	.NOTDIR => return error.NotDir,
5860	.NOMEM => return error.SystemResources,
5861	.NOSPC => return error.NoSpaceLeft,
5862	.EXIST => return error.DirNotEmpty,
5863	.NOTEMPTY => return error.DirNotEmpty,
5864	.ROFS => return error.ReadOnlyFileSystem,
5865	.XDEV => return error.CrossDevice,
5866	.ILSEQ => return error.BadPathName,
5867	.FAULT => |err| return errnoBug(err),
5868	.INVAL => |err| return errnoBug(err),
5869	else => |err| return unexpectedErrno(err),
5870	}
5871	}
5872	}
5873
5874	fn setsockopt(
5875	ev: *Evented,
5876	cancel_region: *CancelRegion,
5877	fd: fd_t,
5878	level: i32,
5879	opt_name: u32,
5880	option: u32,
5881	) !void {
5882	const o: []const u8 = @ptrCast(&option);
5883	while (true) {
5884	const off: extern struct {
5885	cmd_op: linux.IO_URING_SOCKET_OP,
5886	pad: u32,
5887	} align(@alignOf(u64)) = .{
5888	.cmd_op = .SETSOCKOPT,
5889	.pad = 0,
5890	};
5891	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
5892	.level = level,
5893	.opt_name = opt_name,
5894	};
5895	const thread = try cancel_region.awaitIoUring();
5896	thread.enqueue().* = .{
5897	.opcode = .URING_CMD,
5898	.flags = 0,
5899	.ioprio = 0,
5900	.fd = fd,
5901	.off = @as(*const u64, @ptrCast(&off)).*,
5902	.addr = @as(*const u64, @ptrCast(&addr)).*,
5903	.len = 0,
5904	.rw_flags = 0,
5905	.user_data = @intFromPtr(cancel_region.fiber),
5906	.buf_index = 0,
5907	.personality = 0,
5908	.splice_fd_in = @intCast(o.len),
5909	.addr3 = @intFromPtr(o.ptr),
5910	.resv = 0,
5911	};
5912	ev.yield(null, .nothing);
5913	switch (cancel_region.errno()) {
5914	.SUCCESS => return,
5915	.INTR, .CANCELED => {},
5916	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
5917	.NOTSOCK => |err| return errnoBug(err),
5918	.INVAL => |err| return errnoBug(err),
5919	.FAULT => |err| return errnoBug(err),
5920	else => |err| return unexpectedErrno(err),
5921	}
5922	}
5923	}
5924
5925	fn socket(
5926	ev: *Evented,
5927	cancel_region: *CancelRegion,
5928	family: linux.sa_family_t,
5929	options: net.IpAddress.BindOptions,
5930	) error{
5931	AddressFamilyUnsupported,
5932	ProtocolUnsupportedBySystem,
5933	ProcessFdQuotaExceeded,
5934	SystemFdQuotaExceeded,
5935	SystemResources,
5936	ProtocolUnsupportedByAddressFamily,
5937	SocketModeUnsupported,
5938	OptionUnsupported,
5939	Unexpected,
5940	Canceled,
5941	}!fd_t {
5942	const mode, const protocol = try posixSocketModeProtocol(family, options.mode, options.protocol);
5943	const socket_fd = while (true) {
5944	const thread = try cancel_region.awaitIoUring();
5945	thread.enqueue().* = .{
5946	.opcode = .SOCKET,
5947	.flags = 0,
5948	.ioprio = 0,
5949	.fd = family,
5950	.off = mode | linux.SOCK.CLOEXEC,
5951	.addr = 0,
5952	.len = protocol,
5953	.rw_flags = 0,
5954	.user_data = @intFromPtr(cancel_region.fiber),
5955	.buf_index = 0,
5956	.personality = 0,
5957	.splice_fd_in = 0,
5958	.addr3 = 0,
5959	.resv = 0,
5960	};
5961	ev.yield(null, .nothing);
5962	const completion = cancel_region.completion();
5963	switch (completion.errno()) {
5964	.SUCCESS => break completion.result,
5965	.INTR, .CANCELED => {},
5966	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
5967	.INVAL => return error.ProtocolUnsupportedBySystem,
5968	.MFILE => return error.ProcessFdQuotaExceeded,
5969	.NFILE => return error.SystemFdQuotaExceeded,
5970	.NOBUFS => return error.SystemResources,
5971	.NOMEM => return error.SystemResources,
5972	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
5973	.PROTOTYPE => return error.SocketModeUnsupported,
5974	else => |err| return unexpectedErrno(err),
5975	}
5976	};
5977	errdefer ev.closeAsync(socket_fd);
5978
5979	if (options.ip6_only) |ip6_only| {
5980	if (linux.IPV6 == void) return error.OptionUnsupported;
5981	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, @intFromBool(ip6_only));
5982	}
5983
5984	return socket_fd;
5985	}
5986
5987	fn stat(ev: *Evented, cancel_region: *CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
5988	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch |err| switch (err) {
5989	error.BadPathName, error.NameTooLong => unreachable, // path is empty
5990	error.AccessDenied => return errnoBug(.ACCES),
5991	error.SymLinkLoop => return errnoBug(.LOOP),
5992	error.FileNotFound => return errnoBug(.NOENT),
5993	error.NotDir => return errnoBug(.NOTDIR),
5994	else => |e| return e,
5995	};
5996	}
5997
5998	fn statx(
5999	ev: *Evented,
6000	cancel_region: *CancelRegion,
6001	dir: fd_t,
6002	path: [*:0]const u8,
6003	flags: u32,
6004	) (Dir.StatError || Dir.PathNameError || error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
6005	while (true) {
6006	var statx_buf = std.mem.zeroes(linux.Statx);
6007	const thread = try cancel_region.awaitIoUring();
6008	thread.enqueue().* = .{
6009	.opcode = .STATX,
6010	.flags = 0,
6011	.ioprio = 0,
6012	.fd = dir,
6013	.off = @intFromPtr(&statx_buf),
6014	.addr = @intFromPtr(path),
6015	.len = @bitCast(linux_statx_request),
6016	.rw_flags = flags,
6017	.user_data = @intFromPtr(cancel_region.fiber),
6018	.buf_index = 0,
6019	.personality = 0,
6020	.splice_fd_in = 0,
6021	.addr3 = 0,
6022	.resv = 0,
6023	};
6024	ev.yield(null, .nothing);
6025	switch (cancel_region.errno()) {
6026	.SUCCESS => return statFromLinux(&statx_buf),
6027	.INTR, .CANCELED => {},
6028	.ACCES => return error.AccessDenied,
6029	.BADF => |err| return errnoBug(err), // File descriptor used after closed.
6030	.FAULT => |err| return errnoBug(err),
6031	.INVAL => |err| return errnoBug(err),
6032	.LOOP => return error.SymLinkLoop,
6033	.NAMETOOLONG => |err| return errnoBug(err),
6034	.NOENT => return error.FileNotFound,
6035	.NOTDIR => return error.NotDir,
6036	.NOMEM => return error.SystemResources,
6037	else => |err| return unexpectedErrno(err),
6038	}
6039	}
6040	}
6041
6042	fn urandomReadAll(
6043	ev: *Evented,
6044	cancel_region: *CancelRegion,
6045	buffer: []u8,
6046	) (File.OpenError || File.Reader.Error || error{EndOfStream})!void {
6047	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
6048	.ACCMODE = .RDONLY,
6049	.CLOEXEC = true,
6050	}), buffer);
6051	}
6052
6053	fn utimensat(
6054	ev: *Evented,
6055	sync: *CancelRegion.Sync,
6056	dir: fd_t,
6057	path: [*:0]const u8,
6058	times: ?*const [2]linux.timespec,
6059	flags: u32,
6060	) File.SetTimestampsError!void {
6061	_ = ev;
6062	while (true) {
6063	try sync.cancel_region.await(.nothing);
6064	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
6065	.SUCCESS => return,
6066	.INTR => {},
6067	.BADF => |err| return errnoBug(err), // always a race condition
6068	.FAULT => |err| return errnoBug(err),
6069	.INVAL => |err| return errnoBug(err),
6070	.ACCES => return error.AccessDenied,
6071	.PERM => return error.PermissionDenied,
6072	.ROFS => return error.ReadOnlyFileSystem,
6073	else => |err| return unexpectedErrno(err),
6074	}
6075	}
6076	}
6077
6078	fn writeAllSync(sync: *CancelRegion.Sync, fd: fd_t, buffer: []const u8) File.Writer.Error!void {
6079	var index: usize = 0;
6080	while (buffer.len - index != 0) index += try writeSync(sync, fd, buffer[index..]);
6081	}
6082
6083	fn writeSync(sync: *CancelRegion.Sync, fd: fd_t, buffer: []const u8) File.Writer.Error!usize {
6084	while (true) {
6085	try sync.cancel_region.await(.nothing);
6086	const rc = linux.write(fd, buffer.ptr, buffer.len);
6087	switch (linux.errno(rc)) {
6088	.SUCCESS => return @intCast(rc),
6089	.INTR => {},
6090	.INVAL => |err| return errnoBug(err),
6091	.FAULT => |err| return errnoBug(err),
6092	.AGAIN => return error.WouldBlock,
6093	.BADF => return error.NotOpenForWriting, // Can be a race condition.
6094	.DESTADDRREQ => |err| return errnoBug(err), // `connect` was never called.
6095	.DQUOT => return error.DiskQuota,
6096	.FBIG => return error.FileTooBig,
6097	.IO => return error.InputOutput,
6098	.NOSPC => return error.NoSpaceLeft,
6099	.PERM => return error.PermissionDenied,
6100	.PIPE => return error.BrokenPipe,
6101	.CONNRESET => |err| return errnoBug(err), // Not a socket handle.
6102	.BUSY => return error.DeviceBusy,
6103	else => |err| return unexpectedErrno(err),
6104	}
6105	}
6106	}
6107
6108	test {
6109	_ = Fiber.CancelProtection;
6110	}