Commit a28d57292f86dfdacf88453509d0b7a1a49443eb | zig

author	Jacob Young <jacobly@ziglang.org> 2026-01-06 05:10:33-05:00
committer	Jacob Young <jacobly@ziglang.org> 2026-02-09 10:47:21-05:00
log	a28d57292f86dfdacf88453509d0b7a1a49443eb
tree	f67cce22318a0eeafa7e17b15911e3a2f05ed09d
parent	b48599c5492448241a5c37cf17ff6caffa3dd302

IoUring: update to new Io APIs

19 files changed, 5884 insertions(+), 1182 deletions(-)

bootstrap.c
build.zig
lib
- std
  - Io.zig
  - Io
    File.zig
    IoUring.zig
    Threaded.zig
  - os
    linux.zig
  - process.zig
  - tar.zig
src
stage1
- config.zig.in

bootstrap.c+1

...	...	@@ -143,6 +143,7 @@ int main(int argc, char **argv) {
143	143	"pub const skip_non_native = false;\n"
144	144	"pub const debug_gpa = false;\n"
145	145	"pub const dev = .core;\n"
	146	"pub const io_mode: enum { threaded, evented } = .threaded;\n"
146	147	"pub const value_interpret_mode = .direct;\n"
147	148	, zig_version);
148	149	if (written < 100)

build.zig+4

...	...	@@ -13,6 +13,7 @@ const DevEnv = @import("src/dev.zig").Env;
13	13	const zig_version: std.SemanticVersion = .{ .major = 0, .minor = 16, .patch = 0 };
14	14	const stack_size = 46 * 1024 * 1024;
15	15
	16	const IoMode = enum { threaded, evented };
16	17	const ValueInterpretMode = enum { direct, by_name };
17	18
18	19	pub fn build(b: *std.Build) !void {
...	...	@@ -188,6 +189,7 @@ pub fn build(b: *std.Build) !void {
188	189	const strip = b.option(bool, "strip", "Omit debug information");
189	190	const valgrind = b.option(bool, "valgrind", "Enable valgrind integration");
190	191	const pie = b.option(bool, "pie", "Produce a Position Independent Executable");
	192	const io_mode = b.option(IoMode, "io-mode", "How the compiler performs IO") orelse .threaded;
191	193	const value_interpret_mode = b.option(ValueInterpretMode, "value-interpret-mode", "How the compiler translates between 'std.builtin' types and its internal datastructures") orelse .direct;
192	194	const value_tracing = b.option(bool, "value-tracing", "Enable extra state tracking to help troubleshoot bugs in the compiler (using the std.debug.Trace API)") orelse false;
193	195
...	...	@@ -236,6 +238,7 @@ pub fn build(b: *std.Build) !void {
236	238	exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa);
237	239	exe_options.addOption(bool, "debug_gpa", debug_gpa);
238	240	exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full);
	241	exe_options.addOption(IoMode, "io_mode", io_mode);
239	242	exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode);
240	243
241	244	if (link_libc) {
...	...	@@ -710,6 +713,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
710	713	exe_options.addOption(u32, "tracy_callstack_depth", 0);
711	714	exe_options.addOption(bool, "value_tracing", false);
712	715	exe_options.addOption(DevEnv, "dev", .bootstrap);
	716	exe_options.addOption(IoMode, "io_mode", .threaded);
713	717
714	718	// zig1 chooses to interpret values by name. The tradeoff is as follows:
715	719	//

lib/std/Io.zig+41-20

...	...	@@ -378,7 +378,9 @@ pub const Operation = union(enum) {
378	378	pub const Pending = struct {
379	379	node: List.DoubleNode,
380	380	tag: Tag,
381		context: [3]usize,
	381	context: Context align(@max(@alignOf(usize), 4)),
	382
	383	pub const Context = [3]usize;
382	384	};
383	385
384	386	pub const Completion = struct {
...	...	@@ -426,10 +428,10 @@ pub fn operate(io: Io, operation: Operation) Cancelable!Operation.Result {
426	428	pub const Batch = struct {
427	429	storage: []Operation.Storage,
428	430	unused: Operation.List,
429		submissions: Operation.List,
	431	submitted: Operation.List,
430	432	pending: Operation.List,
431		completions: Operation.List,
432		context: ?*anyopaque,
	433	completed: Operation.List,
	434	context: ?anyopaque align(@max(@alignOf(?anyopaque), 4)),
433	435
434	436	/// After calling this, it is safe to unconditionally defer a call to
435	437	/// `cancel`. `storage` is a pre-allocated buffer of undefined memory that
...	...	@@ -448,9 +450,9 @@ pub const Batch = struct {
448	450	.head = .fromIndex(0),
449	451	.tail = .fromIndex(storage.len - 1),
450	452	},
451		.submissions = .empty,
	453	.submitted = .empty,
452	454	.pending = .empty,
453		.completions = .empty,
	455	.completed = .empty,
454	456	.context = null,
455	457	};
456	458	}
...	...	@@ -471,20 +473,20 @@ pub const Batch = struct {
471	473	const storage = &b.storage[index];
472	474	const unused = storage.unused;
473	475	switch (unused.prev) {
474		.none => b.unused.head = .none,
	476	.none => b.unused.head = unused.next,
475	477	else => \|prev_index\| b.storage[prev_index.toIndex()].unused.next = unused.next,
476	478	}
477	479	switch (unused.next) {
478		.none => b.unused.tail = .none,
	480	.none => b.unused.tail = unused.prev,
479	481	else => \|next_index\| b.storage[next_index.toIndex()].unused.prev = unused.prev,
480	482	}
481	483
482		switch (b.submissions.tail) {
483		.none => b.submissions.head = .fromIndex(index),
	484	switch (b.submitted.tail) {
	485	.none => b.submitted.head = .fromIndex(index),
484	486	else => \|tail_index\| b.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
485	487	}
486	488	storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };
487		b.submissions.tail = .fromIndex(index);
	489	b.submitted.tail = .fromIndex(index);
488	490	}
489	491
490	492	pub const Completion = struct {
...	...	@@ -501,13 +503,13 @@ pub const Batch = struct {
501	503	/// Each completion returned from this function dequeues from the `Batch`.
502	504	/// It is not required to dequeue all completions before awaiting again.
503	505	pub fn next(b: *Batch) ?Completion {
504		const index = b.completions.head;
	506	const index = b.completed.head;
505	507	if (index == .none) return null;
506	508	const storage = &b.storage[index.toIndex()];
507	509	const completion = storage.completion;
508	510	const next_index = completion.node.next;
509		b.completions.head = next_index;
510		if (next_index == .none) b.completions.tail = .none;
	511	b.completed.head = next_index;
	512	if (next_index == .none) b.completed.tail = .none;
511	513
512	514	const tail_index = b.unused.tail;
513	515	switch (tail_index) {
...	...	@@ -551,7 +553,27 @@ pub const Batch = struct {
551	553	/// may have successfully completed regardless of the cancel request and
552	554	/// will appear in the iteration.
553	555	pub fn cancel(b: *Batch, io: Io) void {
554		return io.vtable.batchCancel(io.userdata, b);
	556	{ // abort pending submissions
	557	var tail_index = b.unused.tail;
	558	defer b.unused.tail = tail_index;
	559	var index = b.submitted.head;
	560	errdefer b.submissions.head = index;
	561	while (index != .none) {
	562	const next_index = b.storage[index.toIndex()].submission.node.next;
	563	switch (tail_index) {
	564	.none => b.unused.head = index,
	565	else => b.storage[tail_index.toIndex()].unused.next = index,
	566	}
	567	b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
	568	tail_index = index;
	569	index = next_index;
	570	}
	571	b.submitted = .{ .head = .none, .tail = .none };
	572	}
	573	io.vtable.batchCancel(io.userdata, b);
	574	assert(b.submitted.head == .none and b.submitted.tail == .none);
	575	assert(b.pending.head == .none and b.pending.tail == .none);
	576	assert(b.context == null); // that was the last chance to deallocate resources
555	577	}
556	578	};
557	579
...	...	@@ -1117,13 +1139,13 @@ pub fn recancel(io: Io) void {
1117	1139	/// To modify a task's cancel protection state, see `swapCancelProtection`.
1118	1140	///
1119	1141	/// For a description of cancelation and cancelation points, see `Future.cancel`.
1120		pub const CancelProtection = enum {
	1142	pub const CancelProtection = enum(u1) {
1121	1143	/// Any call to an `Io` function with `error.Canceled` in its error set is a cancelation point.
1122	1144	///
1123	1145	/// This is the default state, which all tasks are created in.
1124		unblocked,
	1146	unblocked = 0,
1125	1147	/// No `Io` function introduces a cancelation point (`error.Canceled` will never be returned).
1126		blocked,
	1148	blocked = 1,
1127	1149	};
1128	1150	/// Updates the current task's cancel protection state (see `CancelProtection`).
1129	1151	///
...	...	@@ -1292,8 +1314,7 @@ pub fn futexWake(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, m
1292	1314	/// shared region of code known as the "critical section".
1293	1315	///
1294	1316	/// Mutex is an extern struct so that it may be used as a field inside another
1295		/// extern struct. Having a guaranteed memory layout including mutexes is
1296		/// important for IPC over shared memory (mmap).
	1317	/// extern struct.
1297	1318	pub const Mutex = extern struct {
1298	1319	state: std.atomic.Value(State),
1299	1320

lib/std/Io/File.zig+7-2

...	...	@@ -477,12 +477,17 @@ pub const Permissions = std.Options.FilePermissions orelse if (is_windows) enum(
477	477	/// libc implementations use `0o666` inside `fopen` and then rely on the
478	478	/// process-scoped "umask" setting to adjust this number for file creation.
479	479	default_file = 0o666,
480		default_dir = 0o755,
481		executable_file = 0o777,
	480	/// This is the default mode given to POSIX operating systems for creating
	481	/// directories. `0o777` is "-rwxrwxrwx" which is counter-intuitive at first,
	482	/// since most people would expect "-rwxr-xr-x", for example, when using
	483	/// the `touch` command, which would correspond to `0o755`.
	484	default_dir = 0o777,
482	485	_,
483	486
484	487	pub const has_executable_bit = native_os != .wasi;
485	488
	489	pub const executable_file: @This() = .default_dir;
	490
486	491	pub fn toMode(self: @This()) std.posix.mode_t {
487	492	return @intFromEnum(self);
488	493	}

lib/std/Io/IoUring.zig+5380-774

...	...	@@ -1,21 +1,80 @@
1		const EventLoop = @This();
	1	const addressFromPosix = Io.Threaded.addressFromPosix;
	2	const addressToPosix = Io.Threaded.addressToPosix;
	3	const Alignment = std.mem.Alignment;
	4	const Allocator = std.mem.Allocator;
	5	const Argv0 = Io.Threaded.Argv0;
	6	const assert = std.debug.assert;
2	7	const builtin = @import("builtin");
3
4		const std = @import("../std.zig");
	8	const ChdirError = Io.Threaded.ChdirError;
	9	const clockToPosix = Io.Threaded.clockToPosix;
	10	const Csprng = Io.Threaded.Csprng;
	11	const default_PATH = Io.Threaded.default_PATH;
	12	const Dir = Io.Dir;
	13	const Environ = Io.Threaded.Environ;
	14	const errnoBug = Io.Threaded.errnoBug;
	15	const Evented = @This();
	16	const fallbackSeed = Io.Threaded.fallbackSeed;
	17	const fd_t = linux.fd_t;
	18	const File = Io.File;
5	19	const Io = std.Io;
6		const assert = std.debug.assert;
7		const Allocator = std.mem.Allocator;
8		const Alignment = std.mem.Alignment;
9		const IoUring = std.os.linux.IoUring;
	20	const IoUring = linux.IoUring;
	21	const iovec = std.posix.iovec;
	22	const iovec_const = std.posix.iovec_const;
	23	const linux = std.os.linux;
	24	const linux_statx_request = Io.Threaded.linux_statx_request;
	25	const LOCK = std.posix.LOCK;
	26	const log = std.log.scoped(.@"io-uring");
	27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
	28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
	29	const net = Io.net;
	30	const PATH_MAX = linux.PATH_MAX;
	31	const pathToPosix = Io.Threaded.pathToPosix;
	32	const pid_t = linux.pid_t;
	33	const PosixAddress = Io.Threaded.PosixAddress;
	34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
	35	const posixProtocol = Io.Threaded.posixProtocol;
	36	const posixSocketMode = Io.Threaded.posixSocketMode;
	37	const process = std.process;
	38	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
	39	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
	40	const splat_buffer_size = Io.Threaded.splat_buffer_size;
	41	const statFromLinux = Io.Threaded.statFromLinux;
	42	const std = @import("../std.zig");
	43	const timestampFromPosix = Io.Threaded.timestampFromPosix;
	44	const unexpectedErrno = std.posix.unexpectedErrno;
	45	const winsize = std.posix.winsize;
10	46
11		/// Must be a thread-safe allocator.
12		gpa: Allocator,
13		mutex: Io.Mutex,
14		main_fiber_buffer: [@sizeOf(Fiber) + Fiber.max_result_size]u8 align(@alignOf(Fiber)),
	47	backing_allocator_needs_mutex: bool,
	48	backing_allocator_mutex: Io.Mutex,
	49	/// Does not need to be thread-safe if not used elsewhere.
	50	backing_allocator: Allocator,
	51	main_fiber_buffer: [
	52	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
	53	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
15	54	threads: Thread.List,
16	55
	56	stderr_mutex: Io.Mutex,
	57	stderr_writer: File.Writer = .{
	58	.io = undefined,
	59	.interface = Io.File.Writer.initInterface(&.{}),
	60	.file = .stderr(),
	61	.mode = .streaming,
	62	},
	63	stderr_mode: Io.Terminal.Mode = .no_color,
	64	stderr_writer_initialized: bool = false,
	65
	66	environ_mutex: Io.Mutex,
	67	environ: Environ,
	68
	69	null_fd: CachedFd,
	70	random_fd: CachedFd,
	71
	72	csprng_mutex: Io.Mutex,
	73	csprng: Csprng,
	74
17	75	/// Empirically saw >128KB being used by the self-hosted backend to panic.
18		const idle_stack_size = 256 * 1024;
	76	/// Empirically saw glibc complain about 256KB.
	77	const idle_stack_size = 512 * 1024;
19	78
20	79	const max_idle_search = 4;
21	80	const max_steal_ready_search = 4;
...	...	@@ -23,6 +82,7 @@ const max_steal_ready_search = 4;
23	82	const io_uring_entries = 64;
24	83
25	84	const Thread = struct {
	85	required_align: void align(4),
26	86	thread: std.Thread,
27	87	idle_context: Context,
28	88	current_context: *Context,
...	...	@@ -30,19 +90,33 @@ const Thread = struct {
30	90	io_uring: IoUring,
31	91	idle_search_index: u32,
32	92	steal_ready_search_index: u32,
	93	csprng: Csprng,
33	94
34		const canceling: ?*Thread = @ptrFromInt(@alignOf(Thread));
	95	threadlocal var self: ?*Thread = null;
35	96
36		threadlocal var self: *Thread = undefined;
37
38		fn current() *Thread {
39		return self;
	97	noinline fn current() *Thread {
	98	return self.?;
40	99	}
41	100
42	101	fn currentFiber(thread: Thread) Fiber {
	102	assert(thread.current_context != &thread.idle_context);
43	103	return @fieldParentPtr("context", thread.current_context);
44	104	}
45	105
	106	fn enqueue(thread: Thread) linux.io_uring_sqe {
	107	while (true) return thread.io_uring.get_sqe() catch {
	108	thread.submit();
	109	continue;
	110	};
	111	}
	112
	113	fn submit(thread: *Thread) void {
	114	_ = thread.io_uring.submit() catch \|err\| switch (err) {
	115	error.SignalInterrupt => {},
	116	else => \|e\| @panic(@errorName(e)),
	117	};
	118	}
	119
46	120	const List = struct {
47	121	allocated: []Thread,
48	122	reserved: u32,
...	...	@@ -53,18 +127,109 @@ const Thread = struct {
53	127	const Fiber = struct {
54	128	required_align: void align(4),
55	129	context: Context,
56		awaiter: ?*Fiber,
57		queue_next: ?*Fiber,
58		cancel_thread: ?*Thread,
59		awaiting_completions: std.StaticBitSet(3),
	130	await_count: i32,
	131	link: union {
	132	awaiter: ?*Fiber,
	133	group: struct { prev: ?Fiber, next: ?Fiber },
	134	},
	135	status: union(enum) {
	136	queue_next: ?*Fiber,
	137	awaiting_group: Group,
	138	},
	139	cancel_status: CancelStatus,
	140	cancel_protection: CancelProtection,
	141
	142	const CancelStatus = packed struct(u32) {
	143	requested: bool,
	144	awaiting: Awaiting,
	145
	146	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
	147
	148	const Awaiting = enum(u31) {
	149	nothing = std.math.maxInt(u31),
	150	group = std.math.maxInt(u31) - 1,
	151	select = std.math.maxInt(u31) - 2,
	152	/// An io_uring fd.
	153	_,
	154
	155	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
	156	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
	157	}
	158
	159	fn fromIoUringFd(fd: fd_t) Awaiting {
	160	const awaiting: Awaiting = @enumFromInt(fd);
	161	switch (awaiting) {
	162	.nothing, .group, .select => unreachable,
	163	_ => return awaiting,
	164	}
	165	}
	166
	167	fn toIoUringFd(awaiting: Awaiting) fd_t {
	168	switch (awaiting) {
	169	.nothing, .group => unreachable,
	170	_ => return @intFromEnum(awaiting),
	171	}
	172	}
	173	};
	174
	175	fn changeAwaiting(
	176	cancel_status: *CancelStatus,
	177	old_awaiting: Awaiting,
	178	new_awaiting: Awaiting,
	179	) bool {
	180	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
	181	.requested = false,
	182	.awaiting = new_awaiting.subWrap(old_awaiting),
	183	}, .monotonic);
	184	assert(old_cancel_status.awaiting == old_awaiting);
	185	return old_cancel_status.requested;
	186	}
	187	};
	188
	189	const CancelProtection = packed struct {
	190	user: Io.CancelProtection,
	191	acknowledged: bool,
	192
	193	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
	194
	195	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
	196	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
	197	}
	198
	199	fn acknowledge(cancel_protection: *CancelProtection) void {
	200	assert(!cancel_protection.acknowledged);
	201	cancel_protection.acknowledged = true;
	202	}
	203
	204	fn recancel(cancel_protection: *CancelProtection) void {
	205	assert(cancel_protection.acknowledged);
	206	cancel_protection.acknowledged = false;
	207	}
	208
	209	test check {
	210	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
	211	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	212	.user = .unblocked,
	213	.acknowledged = true,
	214	}));
	215	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	216	.user = .blocked,
	217	.acknowledged = false,
	218	}));
	219	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	220	.user = .blocked,
	221	.acknowledged = true,
	222	}));
	223	}
	224	};
60	225
61	226	const finished: ?*Fiber = @ptrFromInt(@alignOf(Thread));
62	227
63	228	const max_result_align: Alignment = .@"16";
64		const max_result_size = max_result_align.forward(64);
	229	const max_result_size = max_result_align.forward(512);
65	230	/// This includes any stack realignments that need to happen, and also the
66	231	/// initial frame return address slot and argument frame, depending on target.
67		const min_stack_size = 4 * 1024 * 1024;
	232	const min_stack_size = 60 * 1024 * 1024;
68	233	const max_context_align: Alignment = .@"16";
69	234	const max_context_size = max_context_align.forward(1024);
70	235	const max_closure_size: usize = @sizeOf(AsyncClosure);
...	...	@@ -76,9 +241,19 @@ const Fiber = struct {
76	241	) + max_closure_size + max_context_size,
77	242	std.heap.page_size_max,
78	243	);
	244	comptime {
	245	assert(max_result_align.compare(.gte, .of(Completion)));
	246	assert(max_result_size >= @sizeOf(Completion));
	247	}
	248
	249	fn create(ev: Evented) error{OutOfMemory}!Fiber {
	250	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
	251	}
79	252
80		fn allocate(el: EventLoop) error{OutOfMemory}!Fiber {
81		return @ptrCast(try el.gpa.alignedAlloc(u8, .of(Fiber), allocation_size));
	253	fn destroy(fiber: *Fiber, gpa: std.mem.Allocator) void {
	254	log.debug("destroying {*}", .{fiber});
	255	assert(fiber.status.queue_next == null);
	256	gpa.free(fiber.allocatedSlice());
82	257	}
83	258
84	259	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
...	...	@@ -98,98 +273,513 @@ const Fiber = struct {
98	273	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
99	274	}
100	275
101		fn enterCancelRegion(fiber: Fiber, thread: Thread) error{Canceled}!void {
102		if (@cmpxchgStrong(
103		?*Thread,
104		&fiber.cancel_thread,
105		null,
106		thread,
	276	const Queue = struct { head: Fiber, tail: Fiber };
	277
	278	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
	279	/// alignment) so that those two bits can be used in a `packed struct`.
	280	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
	281	null = 0,
	282	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
	283	_,
	284
	285	const Split = packed struct(usize) { low: u2, high: PackedPtr };
	286	fn pack(ptr: ?*Fiber) PackedPtr {
	287	const split: Split = @bitCast(@intFromPtr(ptr));
	288	assert(split.low == 0);
	289	return split.high;
	290	}
	291	fn unpack(ptr: PackedPtr) ?*Fiber {
	292	const split: Split = .{ .low = 0, .high = ptr };
	293	return @ptrFromInt(@as(usize, @bitCast(split)));
	294	}
	295	};
	296
	297	fn requestCancel(fiber: Fiber, ev: Evented) void {
	298	const cancel_status = @atomicRmw(
	299	Fiber.CancelStatus,
	300	&fiber.cancel_status,
	301	.Or,
	302	.{ .requested = true, .awaiting = @enumFromInt(0) },
107	303	.acq_rel,
108		.acquire,
109		)) \|cancel_thread\| {
110		assert(cancel_thread == Thread.canceling);
	304	);
	305	assert(!cancel_status.requested);
	306	switch (cancel_status.awaiting) {
	307	.nothing => {},
	308	.group => {
	309	// The awaiter received a cancelation request while awaiting a group,
	310	// so propagate the cancelation to the group.
	311	if (fiber.status.awaiting_group.cancel(ev, null)) {
	312	fiber.status = .{ .queue_next = null };
	313	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
	314	}
	315	},
	316	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
	317	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
	318	},
	319	_ => \|cancel_io_uring_fd\| {
	320	const thread: *Thread = .current();
	321	thread.enqueue().* = if (thread.io_uring.fd == @intFromEnum(cancel_io_uring_fd)) .{
	322	.opcode = .ASYNC_CANCEL,
	323	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	324	.ioprio = 0,
	325	.fd = 0,
	326	.off = 0,
	327	.addr = @intFromPtr(fiber),
	328	.len = 0,
	329	.rw_flags = 0,
	330	.user_data = @intFromEnum(Completion.UserData.wakeup),
	331	.buf_index = 0,
	332	.personality = 0,
	333	.splice_fd_in = 0,
	334	.addr3 = 0,
	335	.resv = 0,
	336	} else .{
	337	.opcode = .MSG_RING,
	338	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	339	.ioprio = 0,
	340	.fd = @intFromEnum(cancel_io_uring_fd),
	341	.off = @intFromPtr(fiber) \| 0b01,
	342	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
	343	.len = 0,
	344	.rw_flags = 0,
	345	.user_data = @intFromEnum(Completion.UserData.cleanup),
	346	.buf_index = 0,
	347	.personality = 0,
	348	.splice_fd_in = 0,
	349	.addr3 = 0,
	350	.resv = 0,
	351	};
	352	},
	353	}
	354	}
	355	};
	356
	357	const CancelRegion = struct {
	358	fiber: *Fiber,
	359	status: Fiber.CancelStatus,
	360	fn init() CancelRegion {
	361	const fiber = Thread.current().currentFiber();
	362	return .{
	363	.fiber = fiber,
	364	.status = .{
	365	.requested = fiber.cancel_protection.check() == .unblocked,
	366	.awaiting = .nothing,
	367	},
	368	};
	369	}
	370	fn initBlocked() CancelRegion {
	371	return .{
	372	.fiber = Thread.current().currentFiber(),
	373	.status = .{ .requested = false, .awaiting = .nothing },
	374	};
	375	}
	376	fn deinit(cancel_region: *CancelRegion) void {
	377	if (cancel_region.status.requested) _ = cancel_region.fiber.cancel_status.changeAwaiting(
	378	cancel_region.status.awaiting,
	379	.nothing,
	380	);
	381	cancel_region.* = undefined;
	382	}
	383	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
	384	if (!cancel_region.status.requested) return;
	385	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
	386	if (cancel_region.fiber.cancel_status.changeAwaiting(
	387	cancel_region.status.awaiting,
	388	status.awaiting,
	389	)) {
	390	cancel_region.fiber.cancel_protection.acknowledge();
	391	cancel_region.status = .unrequested;
111	392	return error.Canceled;
112	393	}
	394	cancel_region.status = status;
	395	}
	396	fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
	397	const thread: *Thread = .current();
	398	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
	399	return thread;
113	400	}
	401	fn completion(cancel_region: *const CancelRegion) Completion {
	402	return cancel_region.fiber.resultPointer(Completion).*;
	403	}
	404	fn errno(cancel_region: *const CancelRegion) linux.E {
	405	return cancel_region.completion().errno();
	406	}
	407	};
114	408
115		fn exitCancelRegion(fiber: Fiber, thread: Thread) void {
116		if (@cmpxchgStrong(
117		?*Thread,
118		&fiber.cancel_thread,
119		thread,
120		null,
121		.acq_rel,
122		.acquire,
123		)) \|cancel_thread\| assert(cancel_thread == Thread.canceling);
	409	const CachedFd = struct {
	410	once: Once,
	411
	412	const Once = enum(fd_t) {
	413	uninitialized = -1,
	414	initializing = -2,
	415	/// fd
	416	_,
	417
	418	fn fromFd(fd: fd_t) Once {
	419	return @enumFromInt(@as(u31, @intCast(fd)));
	420	}
	421
	422	fn toFd(once: Once) fd_t {
	423	return @as(u31, @intCast(@intFromEnum(once)));
	424	}
	425	};
	426
	427	const init: CachedFd = .{ .once = .uninitialized };
	428
	429	fn close(cached_fd: *CachedFd) void {
	430	switch (cached_fd.once) {
	431	.uninitialized => {},
	432	.initializing => unreachable,
	433	_ => \|fd\| {
	434	assert(@intFromEnum(fd) >= 0);
	435	std.posix.close(@intFromEnum(fd));
	436	cached_fd.* = .init;
	437	},
	438	}
124	439	}
125	440
126		const Queue = struct { head: Fiber, tail: Fiber };
	441	fn open(
	442	cached_fd: *CachedFd,
	443	ev: *Evented,
	444	cancel_region: *CancelRegion,
	445	path: [*:0]const u8,
	446	flags: linux.O,
	447	) File.OpenError!fd_t {
	448	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
	449	while (true) {
	450	switch (once) {
	451	.uninitialized => {},
	452	.initializing => try futexWait(
	453	ev,
	454	@ptrCast(&cached_fd.once),
	455	@bitCast(@intFromEnum(once)),
	456	.none,
	457	),
	458	_ => \|fd\| {
	459	@branchHint(.likely);
	460	return fd.toFd();
	461	},
	462	}
	463	once = @cmpxchgWeak(
	464	Once,
	465	&cached_fd.once,
	466	.uninitialized,
	467	.initializing,
	468	.monotonic,
	469	.monotonic,
	470	) orelse {
	471	errdefer {
	472	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
	473	futexWake(ev, @ptrCast(&cached_fd.once), 1);
	474	}
	475	const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
	476	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
	477	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
	478	return fd;
	479	};
	480	}
	481	}
127	482	};
128	483
129		fn recycle(el: EventLoop, fiber: Fiber) void {
130		std.log.debug("recyling {*}", .{fiber});
131		assert(fiber.queue_next == null);
132		el.gpa.free(fiber.allocatedSlice());
	484	pub fn allocator(ev: *Evented) std.mem.Allocator {
	485	return if (ev.backing_allocator_needs_mutex) .{
	486	.ptr = ev,
	487	.vtable = &.{
	488	.alloc = alloc,
	489	.resize = resize,
	490	.remap = remap,
	491	.free = free,
	492	},
	493	} else ev.backing_allocator;
	494	}
	495
	496	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
	497	const ev: *Evented = @ptrCast(@alignCast(userdata));
	498	const ev_io = ev.io();
	499	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	500	defer ev.backing_allocator_mutex.unlock(ev_io);
	501	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
	502	}
	503
	504	fn resize(
	505	userdata: *anyopaque,
	506	memory: []u8,
	507	alignment: std.mem.Alignment,
	508	new_len: usize,
	509	ret_addr: usize,
	510	) bool {
	511	const ev: *Evented = @ptrCast(@alignCast(userdata));
	512	const ev_io = ev.io();
	513	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	514	defer ev.backing_allocator_mutex.unlock(ev_io);
	515	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
	516	}
	517
	518	fn remap(
	519	userdata: *anyopaque,
	520	memory: []u8,
	521	alignment: Alignment,
	522	new_len: usize,
	523	ret_addr: usize,
	524	) ?[*]u8 {
	525	const ev: *Evented = @ptrCast(@alignCast(userdata));
	526	const ev_io = ev.io();
	527	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	528	defer ev.backing_allocator_mutex.unlock(ev_io);
	529	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
133	530	}
134	531
135		pub fn io(el: *EventLoop) Io {
	532	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
	533	const ev: *Evented = @ptrCast(@alignCast(userdata));
	534	const ev_io = ev.io();
	535	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	536	defer ev.backing_allocator_mutex.unlock(ev_io);
	537	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
	538	}
	539
	540	pub fn io(ev: *Evented) Io {
136	541	return .{
137		.userdata = el,
	542	.userdata = ev,
138	543	.vtable = &.{
139	544	.async = async,
140	545	.concurrent = concurrent,
141	546	.await = await,
142		.select = select,
143	547	.cancel = cancel,
144		.cancelRequested = cancelRequested,
145	548
146		.mutexLock = mutexLock,
147		.mutexUnlock = mutexUnlock,
	549	.groupAsync = groupAsync,
	550	.groupConcurrent = groupConcurrent,
	551	.groupAwait = groupAwait,
	552	.groupCancel = groupCancel,
	553
	554	.recancel = recancel,
	555	.swapCancelProtection = swapCancelProtection,
	556	.checkCancel = checkCancel,
	557
	558	.select = select,
	559
	560	.futexWait = futexWait,
	561	.futexWaitUncancelable = futexWaitUncancelable,
	562	.futexWake = futexWake,
	563
	564	.operate = operate,
	565	.batchAwaitAsync = batchAwaitAsync,
	566	.batchAwaitConcurrent = batchAwaitConcurrent,
	567	.batchCancel = batchCancel,
148	568
149		.conditionWait = conditionWait,
150		.conditionWake = conditionWake,
	569	.dirCreateDir = dirCreateDir,
	570	.dirCreateDirPath = dirCreateDirPath,
	571	.dirCreateDirPathOpen = dirCreateDirPathOpen,
	572	.dirOpenDir = dirOpenDir,
	573	.dirStat = dirStat,
	574	.dirStatFile = dirStatFile,
	575	.dirAccess = dirAccess,
	576	.dirCreateFile = dirCreateFile,
	577	.dirCreateFileAtomic = dirCreateFileAtomic,
	578	.dirOpenFile = dirOpenFile,
	579	.dirClose = dirClose,
	580	.dirRead = dirRead,
	581	.dirRealPath = dirRealPath,
	582	.dirRealPathFile = dirRealPathFile,
	583	.dirDeleteFile = dirDeleteFile,
	584	.dirDeleteDir = dirDeleteDir,
	585	.dirRename = dirRename,
	586	.dirRenamePreserve = dirRenamePreserve,
	587	.dirSymLink = dirSymLink,
	588	.dirReadLink = dirReadLink,
	589	.dirSetOwner = dirSetOwner,
	590	.dirSetFileOwner = dirSetFileOwner,
	591	.dirSetPermissions = dirSetPermissions,
	592	.dirSetFilePermissions = dirSetFilePermissions,
	593	.dirSetTimestamps = dirSetTimestamps,
	594	.dirHardLink = dirHardLink,
151	595
152		.createFile = createFile,
153		.fileOpen = fileOpen,
	596	.fileStat = fileStat,
	597	.fileLength = fileLength,
154	598	.fileClose = fileClose,
155		.pread = pread,
156		.pwrite = pwrite,
	599	.fileWritePositional = fileWritePositional,
	600	.fileWriteFileStreaming = fileWriteFileStreaming,
	601	.fileWriteFilePositional = fileWriteFilePositional,
	602	.fileReadPositional = fileReadPositional,
	603	.fileSeekBy = fileSeekBy,
	604	.fileSeekTo = fileSeekTo,
	605	.fileSync = fileSync,
	606	.fileIsTty = fileIsTty,
	607	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
	608	.fileSupportsAnsiEscapeCodes = fileIsTty,
	609	.fileSetLength = fileSetLength,
	610	.fileSetOwner = fileSetOwner,
	611	.fileSetPermissions = fileSetPermissions,
	612	.fileSetTimestamps = fileSetTimestamps,
	613	.fileLock = fileLock,
	614	.fileTryLock = fileTryLock,
	615	.fileUnlock = fileUnlock,
	616	.fileDowngradeLock = fileDowngradeLock,
	617	.fileRealPath = fileRealPath,
	618	.fileHardLink = fileHardLink,
	619
	620	.fileMemoryMapCreate = fileMemoryMapCreate,
	621	.fileMemoryMapDestroy = fileMemoryMapDestroy,
	622	.fileMemoryMapSetLength = fileMemoryMapSetLength,
	623	.fileMemoryMapRead = fileMemoryMapRead,
	624	.fileMemoryMapWrite = fileMemoryMapWrite,
	625
	626	.processExecutableOpen = processExecutableOpen,
	627	.processExecutablePath = processExecutablePath,
	628	.lockStderr = lockStderr,
	629	.tryLockStderr = tryLockStderr,
	630	.unlockStderr = unlockStderr,
	631	.processCurrentPath = processCurrentPath,
	632	.processSetCurrentDir = processSetCurrentDir,
	633	.processReplace = processReplace,
	634	.processReplacePath = processReplacePath,
	635	.processSpawn = processSpawn,
	636	.processSpawnPath = processSpawnPath,
	637	.childWait = childWait,
	638	.childKill = childKill,
	639
	640	.progressParentFile = progressParentFile,
157	641
158	642	.now = now,
	643	.clockResolution = clockResolution,
159	644	.sleep = sleep,
	645
	646	.random = random,
	647	.randomSecure = randomSecure,
	648
	649	.netListenIp = netListenIpUnavailable,
	650	.netAccept = netAcceptUnavailable,
	651	.netBindIp = netBindIp,
	652	.netConnectIp = netConnectIpUnavailable,
	653	.netListenUnix = netListenUnixUnavailable,
	654	.netConnectUnix = netConnectUnixUnavailable,
	655	.netSocketCreatePair = netSocketCreatePairUnavailable,
	656	.netSend = netSendUnavailable,
	657	.netReceive = netReceive,
	658	.netRead = netReadUnavailable,
	659	.netWrite = netWriteUnavailable,
	660	.netWriteFile = netWriteFileUnavailable,
	661	.netClose = netClose,
	662	.netShutdown = netShutdown,
	663	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
	664	.netInterfaceName = netInterfaceNameUnavailable,
	665	.netLookup = netLookupUnavailable,
160	666	},
161	667	};
162	668	}
163	669
164		pub fn init(el: *EventLoop, gpa: Allocator) !void {
	670	fn fileMemoryMapSetLength(
	671	userdata: ?*anyopaque,
	672	mm: *File.MemoryMap,
	673	new_len: usize,
	674	) File.MemoryMap.SetLengthError!void {
	675	const ev: *Evented = @ptrCast(@alignCast(userdata));
	676	_ = ev;
	677	const page_size = std.heap.pageSize();
	678	const alignment: Alignment = .fromByteUnits(page_size);
	679	const page_align = std.heap.page_size_min;
	680	const old_memory = mm.memory;
	681
	682	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
	683	mm.memory.len = new_len;
	684	return;
	685	}
	686	var cancel_region: CancelRegion = .init();
	687	defer cancel_region.deinit();
	688	const flags: linux.MREMAP = .{ .MAYMOVE = true };
	689	const addr_hint: ?[*]const u8 = null;
	690	const new_memory = while (true) {
	691	try cancel_region.await(.nothing);
	692	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
	693	switch (linux.errno(rc)) {
	694	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
	695	.INTR => continue,
	696	.AGAIN => return error.LockedMemoryLimitExceeded,
	697	.NOMEM => return error.OutOfMemory,
	698	.INVAL => \|err\| return errnoBug(err),
	699	.FAULT => \|err\| return errnoBug(err),
	700	else => \|err\| return unexpectedErrno(err),
	701	}
	702	};
	703	mm.memory = new_memory;
	704	}
	705
	706	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
	707	const ev: *Evented = @ptrCast(@alignCast(userdata));
	708	_ = ev;
	709	_ = mm;
	710	}
	711
	712	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
	713	const ev: *Evented = @ptrCast(@alignCast(userdata));
	714	_ = ev;
	715	_ = mm;
	716	}
	717
	718	pub const InitOptions = struct {
	719	backing_allocator_needs_mutex: bool = true,
	720
	721	/// Affects the following operations:
	722	/// * `processExecutablePath` on OpenBSD and Haiku.
	723	argv0: Argv0 = .empty,
	724	/// Affects the following operations:
	725	/// * `fileIsTty`
	726	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
	727	environ: process.Environ,
	728	};
	729
	730	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
165	731	const threads_size = @max(std.Thread.getCpuCount() catch 1, 1) * @sizeOf(Thread);
166		const idle_stack_end_offset = std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
167		const allocated_slice = try gpa.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
168		errdefer gpa.free(allocated_slice);
169		el.* = .{
170		.gpa = gpa,
171		.mutex = .{},
	732	const idle_stack_end_offset =
	733	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
	734	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
	735	errdefer backing_allocator.free(allocated_slice);
	736	ev.* = .{
	737	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
	738	.backing_allocator_mutex = .init,
	739	.backing_allocator = backing_allocator,
172	740	.main_fiber_buffer = undefined,
173	741	.threads = .{
174	742	.allocated = @ptrCast(allocated_slice[0..threads_size]),
175	743	.reserved = 1,
176	744	.active = 1,
177	745	},
	746
	747	.stderr_mutex = .init,
	748	.stderr_writer = .{
	749	.io = ev.io(),
	750	.interface = Io.File.Writer.initInterface(&.{}),
	751	.file = .stderr(),
	752	.mode = .streaming,
	753	},
	754	.stderr_mode = .no_color,
	755	.stderr_writer_initialized = false,
	756
	757	.environ_mutex = .init,
	758	.environ = .{ .process_environ = options.environ },
	759
	760	.null_fd = .init,
	761	.random_fd = .init,
	762
	763	.csprng_mutex = .init,
	764	.csprng = .uninitialized,
178	765	};
179		const main_fiber: *Fiber = @ptrCast(&el.main_fiber_buffer);
	766	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
180	767	main_fiber.* = .{
181	768	.required_align = {},
182	769	.context = undefined,
183		.awaiter = null,
184		.queue_next = null,
185		.cancel_thread = null,
186		.awaiting_completions = .initEmpty(),
	770	.await_count = 0,
	771	.link = .{ .awaiter = null },
	772	.status = .{ .queue_next = null },
	773	.cancel_status = .unrequested,
	774	.cancel_protection = .unblocked,
187	775	};
188		const main_thread = &el.threads.allocated[0];
	776	const main_thread = &ev.threads.allocated[0];
189	777	Thread.self = main_thread;
190		const idle_stack_end: [*]align(16) usize = @ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
191		(idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)};
	778	const idle_stack_end: [*]align(16) usize =
	779	@ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
	780	(idle_stack_end - 1)[0..1].* = .{@intFromPtr(ev)};
192	781	main_thread.* = .{
	782	.required_align = {},
193	783	.thread = undefined,
194	784	.idle_context = switch (builtin.cpu.arch) {
195	785	.aarch64 => .{
...	...	@@ -206,42 +796,56 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void {
206	796	},
207	797	.current_context = &main_fiber.context,
208	798	.ready_queue = null,
209		.io_uring = try IoUring.init(io_uring_entries, 0),
	799	.io_uring = try .init(
	800	io_uring_entries,
	801	linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
	802	),
210	803	.idle_search_index = 1,
211	804	.steal_ready_search_index = 1,
	805	.csprng = .uninitialized,
212	806	};
213	807	errdefer main_thread.io_uring.deinit();
214		std.log.debug("created main idle {*}", .{&main_thread.idle_context});
215		std.log.debug("created main {*}", .{main_fiber});
	808	log.debug("created main idle {*}", .{&main_thread.idle_context});
	809	log.debug("created main {*}", .{main_fiber});
216	810	}
217	811
218		pub fn deinit(el: *EventLoop) void {
219		const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);
220		for (el.threads.allocated[0..active_threads]) \|*thread\| {
	812	pub fn deinit(ev: *Evented) void {
	813	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
	814	for (ev.threads.allocated[0..active_threads]) \|*thread\| {
221	815	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
222	816	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
223	817	}
224		el.yield(null, .exit);
225		const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(el.threads.allocated.ptr));
226		const idle_stack_end_offset = std.mem.alignForward(usize, el.threads.allocated.len * @sizeOf(Thread) + idle_stack_size, std.heap.page_size_max);
227		for (el.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
228		el.gpa.free(allocated_ptr[0..idle_stack_end_offset]);
229		el.* = undefined;
	818	ev.yield(null, .exit);
	819	ev.threads.allocated[0].io_uring.deinit();
	820	ev.null_fd.close();
	821	ev.random_fd.close();
	822	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
	823	const idle_stack_end_offset = std.mem.alignForward(
	824	usize,
	825	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
	826	std.heap.page_size_max,
	827	);
	828	for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
	829	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
	830	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
	831	ev.* = undefined;
230	832	}
231	833
232		fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
	834	fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
233	835	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
234		@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release);
235		ready_fiber.queue_next = null;
	836	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
	837	ready_fiber.status.queue_next = null;
236	838	return ready_fiber;
237	839	}
238		const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);
	840	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
239	841	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
240	842	defer thread.steal_ready_search_index += 1;
241	843	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
242		const steal_ready_search_thread = &el.threads.allocated[0..active_threads][thread.steal_ready_search_index];
	844	const steal_ready_search_thread =
	845	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
243	846	if (steal_ready_search_thread == thread) continue;
244		const ready_fiber = @atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue;
	847	const ready_fiber =
	848	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue;
245	849	if (ready_fiber == Fiber.finished) continue;
246	850	if (@cmpxchgWeak(
247	851	?*Fiber,
...	...	@@ -251,8 +855,8 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
251	855	.acquire,
252	856	.monotonic,
253	857	)) \|_\| continue;
254		@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release);
255		ready_fiber.queue_next = null;
	858	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
	859	ready_fiber.status.queue_next = null;
256	860	return ready_fiber;
257	861	}
258	862	// couldn't find anything to do, so we are now open for business
...	...	@@ -260,9 +864,9 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
260	864	return null;
261	865	}
262	866
263		fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
	867	fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
264	868	const thread: *Thread = .current();
265		const ready_context = if (maybe_ready_fiber orelse el.findReadyFiber(thread)) \|ready_fiber\|
	869	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
266	870	&ready_fiber.context
267	871	else
268	872	&thread.idle_context;
...	...	@@ -273,25 +877,25 @@ fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage
273	877	},
274	878	.pending_task = pending_task,
275	879	};
276		std.log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });
277		contextSwitch(&message).handle(el);
	880	log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });
	881	contextSwitch(&message).handle(ev);
278	882	}
279	883
280		fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
	884	fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
281	885	{
282	886	var fiber = ready_queue.head;
283	887	while (true) {
284		std.log.debug("scheduling {*}", .{fiber});
285		fiber = fiber.queue_next orelse break;
	888	log.debug("scheduling {*}", .{fiber});
	889	fiber = fiber.status.queue_next orelse break;
286	890	}
287	891	assert(fiber == ready_queue.tail);
288	892	}
289	893	// shared fields of previous `Thread` must be initialized before later ones are marked as active
290		const new_thread_index = @atomicLoad(u32, &el.threads.active, .acquire);
	894	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
291	895	for (0..@min(max_idle_search, new_thread_index)) \|_\| {
292	896	defer thread.idle_search_index += 1;
293	897	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
294		const idle_search_thread = &el.threads.allocated[0..new_thread_index][thread.idle_search_index];
	898	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
295	899	if (idle_search_thread == thread) continue;
296	900	if (@cmpxchgWeak(
297	901	?*Fiber,
...	...	@@ -301,13 +905,13 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
301	905	.release,
302	906	.monotonic,
303	907	)) \|_\| continue;
304		getSqe(&thread.io_uring).* = .{
	908	thread.enqueue().* = .{
305	909	.opcode = .MSG_RING,
306		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
	910	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
307	911	.ioprio = 0,
308	912	.fd = idle_search_thread.io_uring.fd,
309	913	.off = @intFromEnum(Completion.UserData.wakeup),
310		.addr = 0,
	914	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
311	915	.len = 0,
312	916	.rw_flags = 0,
313	917	.user_data = @intFromEnum(Completion.UserData.wakeup),
...	...	@@ -317,145 +921,221 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
317	921	.addr3 = 0,
318	922	.resv = 0,
319	923	};
320		return;
	924	return true;
321	925	}
322	926	spawn_thread: {
323	927	// previous failed reservations must have completed before retrying
324		if (new_thread_index == el.threads.allocated.len or @cmpxchgWeak(
	928	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
325	929	u32,
326		&el.threads.reserved,
	930	&ev.threads.reserved,
327	931	new_thread_index,
328	932	new_thread_index + 1,
329	933	.acquire,
330	934	.monotonic,
331	935	) != null) break :spawn_thread;
332		const new_thread = &el.threads.allocated[new_thread_index];
	936	const new_thread = &ev.threads.allocated[new_thread_index];
333	937	const next_thread_index = new_thread_index + 1;
	938	var params = std.mem.zeroInit(linux.io_uring_params, .{
	939	.flags = linux.IORING_SETUP_ATTACH_WQ \|
	940	linux.IORING_SETUP_R_DISABLED \|
	941	linux.IORING_SETUP_COOP_TASKRUN \|
	942	linux.IORING_SETUP_SINGLE_ISSUER,
	943	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
	944	});
334	945	new_thread.* = .{
	946	.required_align = {},
335	947	.thread = undefined,
336	948	.idle_context = undefined,
337	949	.current_context = &new_thread.idle_context,
338	950	.ready_queue = ready_queue.head,
339		.io_uring = IoUring.init(io_uring_entries, 0) catch \|err\| {
340		@atomicStore(u32, &el.threads.reserved, new_thread_index, .release);
	951	.io_uring = IoUring.init_params(io_uring_entries, &params) catch \|err\| {
	952	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
341	953	// no more access to `thread` after giving up reservation
342		std.log.warn("unable to create worker thread due to io_uring init failure: {s}", .{@errorName(err)});
	954	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
	955	@errorName(err),
	956	});
343	957	break :spawn_thread;
344	958	},
345	959	.idle_search_index = 0,
346	960	.steal_ready_search_index = 0,
	961	.csprng = .uninitialized,
347	962	};
348	963	new_thread.thread = std.Thread.spawn(.{
349	964	.stack_size = idle_stack_size,
350		.allocator = el.gpa,
351		}, threadEntry, .{ el, new_thread_index }) catch \|err\| {
	965	.allocator = ev.allocator(),
	966	}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
352	967	new_thread.io_uring.deinit();
353		@atomicStore(u32, &el.threads.reserved, new_thread_index, .release);
	968	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
354	969	// no more access to `thread` after giving up reservation
355		std.log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
	970	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
356	971	break :spawn_thread;
357	972	};
358	973	// shared fields of `Thread` must be initialized before being marked active
359		@atomicStore(u32, &el.threads.active, next_thread_index, .release);
360		return;
	974	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
	975	return false;
361	976	}
362	977	// nobody wanted it, so just queue it on ourselves
363	978	while (@cmpxchgWeak(
364	979	?*Fiber,
365	980	&thread.ready_queue,
366		ready_queue.tail.queue_next,
	981	ready_queue.tail.status.queue_next,
367	982	ready_queue.head,
368	983	.acq_rel,
369	984	.acquire,
370		)) \|old_head\| ready_queue.tail.queue_next = old_head;
	985	)) \|old_head\| ready_queue.tail.status.queue_next = old_head;
	986	return false;
371	987	}
372	988
373		fn mainIdle(el: EventLoop, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
374		message.handle(el);
375		el.idle(&el.threads.allocated[0]);
376		el.yield(@ptrCast(&el.main_fiber_buffer), .nothing);
	989	fn mainIdle(
	990	ev: *Evented,
	991	message: *const SwitchMessage,
	992	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
	993	message.handle(ev);
	994	ev.idle(&ev.threads.allocated[0]);
	995	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
377	996	unreachable; // switched to dead fiber
378	997	}
379	998
380		fn threadEntry(el: *EventLoop, index: u32) void {
381		const thread: *Thread = &el.threads.allocated[index];
	999	fn threadEntry(ev: *Evented, index: u32) void {
	1000	const thread: *Thread = &ev.threads.allocated[index];
382	1001	Thread.self = thread;
383		std.log.debug("created thread idle {*}", .{&thread.idle_context});
384		el.idle(thread);
	1002	defer thread.io_uring.deinit();
	1003	log.debug("created thread idle {*}", .{&thread.idle_context});
	1004	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
	1005	.SUCCESS => ev.idle(thread),
	1006	else => \|err\| @panic(@tagName(err)),
	1007	}
385	1008	}
386	1009
387	1010	const Completion = struct {
	1011	result: i32,
	1012	flags: u32,
	1013
388	1014	const UserData = enum(usize) {
389	1015	unused,
390	1016	wakeup,
	1017	futex_wake,
391	1018	cleanup,
392	1019	exit,
393		/// *Fiber
	1020	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
	1021	/// If bits 0 and 1 are 0, a `*Fiber`.
394	1022	_,
395	1023	};
396		result: i32,
397		flags: u32,
	1024
	1025	fn errno(completion: Completion) linux.E {
	1026	return linux.errno(@bitCast(@as(isize, completion.result)));
	1027	}
398	1028	};
399	1029
400		fn idle(el: EventLoop, thread: Thread) void {
	1030	fn idle(ev: Evented, thread: Thread) void {
401	1031	var maybe_ready_fiber: ?*Fiber = null;
402	1032	while (true) {
403		while (maybe_ready_fiber orelse el.findReadyFiber(thread)) \|ready_fiber\| {
404		el.yield(ready_fiber, .nothing);
	1033	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
	1034	ev.yield(ready_fiber, .nothing);
405	1035	maybe_ready_fiber = null;
406	1036	}
407	1037	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
408		error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}),
	1038	error.SignalInterrupt => {},
409	1039	else => \|e\| @panic(@errorName(e)),
410	1040	};
411		var cqes_buffer: [io_uring_entries]std.os.linux.io_uring_cqe = undefined;
	1041	var cqes_buffer: [io_uring_entries]linux.io_uring_cqe = undefined;
412	1042	var maybe_ready_queue: ?Fiber.Queue = null;
413	1043	for (cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
414		error.SignalInterrupt => cqes_len: {
415		std.log.warn("copy_cqes failed with SignalInterrupt", .{});
416		break :cqes_len 0;
417		},
	1044	error.SignalInterrupt => 0,
418	1045	else => \|e\| @panic(@errorName(e)),
419		}]) \|cqe\| switch (@as(Completion.UserData, @enumFromInt(cqe.user_data))) {
	1046	}]) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
	1047	Completion.UserData,
	1048	@enumFromInt(cqe.user_data),
	1049	)) {
420	1050	.unused => unreachable, // bad submission queued?
421	1051	.wakeup => {},
	1052	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1053	.SUCCESS => recoverableOsBugDetected(), // success is skipped
	1054	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
	1055	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.UserData.futex_wake` is not cancelable
	1056	.FAULT => {}, // pointer became invalid while doing the wake
	1057	else => recoverableOsBugDetected(), // deadlock due to operating system bug
	1058	},
422	1059	.cleanup => @panic("failed to notify other threads that we are exiting"),
423	1060	.exit => {
424	1061	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
425	1062	return;
426	1063	},
427		_ => switch (errno(cqe.res)) {
428		.INTR => getSqe(&thread.io_uring).* = .{
429		.opcode = .ASYNC_CANCEL,
430		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
431		.ioprio = 0,
432		.fd = 0,
433		.off = 0,
434		.addr = cqe.user_data,
435		.len = 0,
436		.rw_flags = 0,
437		.user_data = @intFromEnum(Completion.UserData.wakeup),
438		.buf_index = 0,
439		.personality = 0,
440		.splice_fd_in = 0,
441		.addr3 = 0,
442		.resv = 0,
443		},
444		else => {
445		const fiber: *Fiber = @ptrFromInt(cqe.user_data);
446		assert(fiber.queue_next == null);
447		fiber.resultPointer(Completion).* = .{
	1064	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
	1065	0b00 => {
	1066	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1067	ready_fiber.resultPointer(Completion).* = .{
448	1068	.result = cqe.res,
449	1069	.flags = cqe.flags,
450	1070	};
451		if (maybe_ready_fiber == null) maybe_ready_fiber = fiber else if (maybe_ready_queue) \|*ready_queue\| {
452		ready_queue.tail.queue_next = fiber;
453		ready_queue.tail = fiber;
454		} else maybe_ready_queue = .{ .head = fiber, .tail = fiber };
	1071	break :ready_fiber ready_fiber;
	1072	},
	1073	0b01 => {
	1074	thread.enqueue().* = .{
	1075	.opcode = .ASYNC_CANCEL,
	1076	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	1077	.ioprio = 0,
	1078	.fd = 0,
	1079	.off = 0,
	1080	.addr = cqe.user_data & ~@as(usize, 0b11),
	1081	.len = 0,
	1082	.rw_flags = 0,
	1083	.user_data = @intFromEnum(Completion.UserData.wakeup),
	1084	.buf_index = 0,
	1085	.personality = 0,
	1086	.splice_fd_in = 0,
	1087	.addr3 = 0,
	1088	.resv = 0,
	1089	};
	1090	break :ready_fiber null;
	1091	},
	1092	0b10 => {
	1093	const context: *Io.Operation.Storage.Pending.Context =
	1094	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1095	const batch: *Io.Batch = @ptrFromInt(context[0]);
	1096	var next: usize = 0b00;
	1097	context[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
	1098	while (true) {
	1099	next = @cmpxchgWeak(
	1100	usize,
	1101	@as(*usize, @ptrCast(&batch.context)),
	1102	next,
	1103	cqe.user_data,
	1104	.release,
	1105	.acquire,
	1106	) orelse break;
	1107	context[0] = next;
	1108	}
	1109	break :ready_fiber switch (@as(u2, @truncate(next))) {
	1110	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
	1111	0b10, 0b11 => null,
	1112	};
	1113	},
	1114	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1115	.SUCCESS => unreachable, // no event count specified
	1116	.TIME => {
	1117	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1118	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
	1119	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
	1120	else => unreachable, // timeout completed multiple times
	1121	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
	1122	0b10 => null,
	1123	};
	1124	},
	1125	.CANCELED => null, // user data may have been invalidated
	1126	else => \|err\| unexpectedErrno(err) catch null,
455	1127	},
	1128	})) \|ready_fiber\| {
	1129	assert(ready_fiber.status.queue_next == null);
	1130	if (maybe_ready_fiber == null) {
	1131	maybe_ready_fiber = ready_fiber;
	1132	} else if (maybe_ready_queue) \|*ready_queue\| {
	1133	ready_queue.tail.status.queue_next = ready_fiber;
	1134	ready_queue.tail = ready_fiber;
	1135	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
456	1136	},
457	1137	};
458		if (maybe_ready_queue) \|ready_queue\| el.schedule(thread, ready_queue);
	1138	if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
459	1139	}
460	1140	}
461	1141
...	...	@@ -469,113 +1149,68 @@ const SwitchMessage = struct {
469	1149	const PendingTask = union(enum) {
470	1150	nothing,
471	1151	reschedule,
472		recycle: *Fiber,
473		register_awaiter: ?Fiber,
474		register_select: []const *Io.AnyFuture,
475		mutex_lock: struct {
476		prev_state: Io.Mutex.State,
477		mutex: *Io.Mutex,
478		},
479		condition_wait: struct {
480		cond: *Io.Condition,
481		mutex: *Io.Mutex,
482		},
	1152	await: u31,
	1153	group_await: Group,
	1154	group_cancel: Group,
	1155	batch_await: *Io.Batch,
	1156	destroy,
483	1157	exit,
484	1158	};
485	1159
486		fn handle(message: const SwitchMessage, el: EventLoop) void {
	1160	fn handle(message: const SwitchMessage, ev: Evented) void {
487	1161	const thread: *Thread = .current();
488	1162	thread.current_context = message.contexts.ready;
489	1163	switch (message.pending_task) {
490	1164	.nothing => {},
491	1165	.reschedule => if (message.contexts.prev != &thread.idle_context) {
492		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
493		assert(prev_fiber.queue_next == null);
494		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
495		},
496		.recycle => \|fiber\| {
497		el.recycle(fiber);
	1166	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1167	assert(fiber.status.queue_next == null);
	1168	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
498	1169	},
499		.register_awaiter => \|awaiter\| {
500		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
501		assert(prev_fiber.queue_next == null);
502		if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished)
503		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
	1170	.await => \|count\| {
	1171	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1172	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
	1173	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
504	1174	},
505		.register_select => \|futures\| {
506		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
507		assert(prev_fiber.queue_next == null);
508		for (futures) \|any_future\| {
509		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
510		if (@atomicRmw(?*Fiber, &future_fiber.awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) {
511		const closure: *AsyncClosure = .fromFiber(future_fiber);
512		if (!@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .seq_cst)) {
513		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
514		}
515		}
516		}
	1175	.group_await => \|group\| {
	1176	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1177	if (group.await(ev, fiber))
	1178	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
517	1179	},
518		.mutex_lock => \|mutex_lock\| {
519		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
520		assert(prev_fiber.queue_next == null);
521		var prev_state = mutex_lock.prev_state;
522		while (switch (prev_state) {
523		else => next_state: {
524		prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state));
525		break :next_state @cmpxchgWeak(
526		Io.Mutex.State,
527		&mutex_lock.mutex.state,
528		prev_state,
529		@enumFromInt(@intFromPtr(prev_fiber)),
530		.release,
531		.acquire,
532		);
533		},
534		.unlocked => @cmpxchgWeak(
535		Io.Mutex.State,
536		&mutex_lock.mutex.state,
537		.unlocked,
538		.locked_once,
539		.acquire,
540		.acquire,
541		) orelse {
542		prev_fiber.queue_next = null;
543		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
544		return;
545		},
546		}) \|next_state\| prev_state = next_state;
	1180	.group_cancel => \|group\| {
	1181	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1182	if (group.cancel(ev, fiber))
	1183	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
547	1184	},
548		.condition_wait => \|condition_wait\| {
549		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
550		assert(prev_fiber.queue_next == null);
551		const cond_impl = prev_fiber.resultPointer(ConditionImpl);
552		cond_impl.* = .{
553		.tail = prev_fiber,
554		.event = .queued,
555		};
	1185	.batch_await => \|batch\| {
	1186	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
556	1187	if (@cmpxchgStrong(
557		?*Fiber,
558		@as(?Fiber, @ptrCast(&condition_wait.cond.state)),
	1188	?*anyopaque,
	1189	&batch.context,
559	1190	null,
560		prev_fiber,
	1191	fiber,
561	1192	.release,
562		.acquire,
563		)) \|waiting_fiber\| {
564		const waiting_cond_impl = waiting_fiber.?.resultPointer(ConditionImpl);
565		assert(waiting_cond_impl.tail.queue_next == null);
566		waiting_cond_impl.tail.queue_next = prev_fiber;
567		waiting_cond_impl.tail = prev_fiber;
	1193	.monotonic,
	1194	)) \|head\| {
	1195	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
	1196	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
568	1197	}
569		condition_wait.mutex.unlock(el.io());
570	1198	},
571		.exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) \|*each_thread\| {
572		getSqe(&thread.io_uring).* = .{
	1199	.destroy => {
	1200	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1201	fiber.destroy(ev.backing_allocator);
	1202	ev.backing_allocator_mutex.unlock(ev.io());
	1203	},
	1204	.exit => for (
	1205	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
	1206	) \|*each_thread\| {
	1207	thread.enqueue().* = .{
573	1208	.opcode = .MSG_RING,
574		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
	1209	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
575	1210	.ioprio = 0,
576	1211	.fd = each_thread.io_uring.fd,
577	1212	.off = @intFromEnum(Completion.UserData.exit),
578		.addr = 0,
	1213	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
579	1214	.len = 0,
580	1215	.rw_flags = 0,
581	1216	.user_data = @intFromEnum(Completion.UserData.cleanup),
...	...	@@ -784,65 +1419,73 @@ inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
784	1419
785	1420	fn mainIdleEntry() callconv(.naked) void {
786	1421	switch (builtin.cpu.arch) {
787		.x86_64 => asm volatile (
788		\\ movq (%%rsp), %%rdi
789		\\ jmp %[mainIdle:P]
790		:
791		: [mainIdle] "X" (&mainIdle),
792		),
793	1422	.aarch64 => asm volatile (
794	1423	\\ ldr x0, [sp, #-8]
795	1424	\\ b %[mainIdle]
796	1425	:
797	1426	: [mainIdle] "X" (&mainIdle),
798	1427	),
799		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
800		}
801		}
802
803		fn fiberEntry() callconv(.naked) void {
804		switch (builtin.cpu.arch) {
805	1428	.x86_64 => asm volatile (
806		\\ leaq 8(%%rsp), %%rdi
807		\\ jmp %[AsyncClosure_call:P]
	1429	\\ movq (%%rsp), %%rdi
	1430	\\ jmp %[mainIdle:P]
808	1431	:
809		: [AsyncClosure_call] "X" (&AsyncClosure.call),
	1432	: [mainIdle] "X" (&mainIdle),
810	1433	),
811	1434	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
812	1435	}
813	1436	}
814	1437
815	1438	const AsyncClosure = struct {
816		event_loop: *EventLoop,
	1439	ev: *Evented,
817	1440	fiber: *Fiber,
818	1441	start: const fn (context: const anyopaque, result: *anyopaque) void,
819	1442	result_align: Alignment,
820		already_awaited: bool,
	1443
	1444	fn fromFiber(fiber: Fiber) AsyncClosure {
	1445	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
	1446	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1447	) - @sizeOf(AsyncClosure));
	1448	}
821	1449
822	1450	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
823	1451	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
824	1452	}
825	1453
826		fn call(closure: AsyncClosure, message: const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
827		message.handle(closure.event_loop);
	1454	fn entry() callconv(.naked) void {
	1455	switch (builtin.cpu.arch) {
	1456	.aarch64 => asm volatile (
	1457	\\ mov x0, sp
	1458	\\ b %[call]
	1459	:
	1460	: [call] "X" (&call),
	1461	),
	1462	.x86_64 => asm volatile (
	1463	\\ leaq 8(%%rsp), %%rdi
	1464	\\ jmp %[call:P]
	1465	:
	1466	: [call] "X" (&call),
	1467	),
	1468	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1469	}
	1470	}
	1471
	1472	fn call(
	1473	closure: *AsyncClosure,
	1474	message: *const SwitchMessage,
	1475	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
	1476	message.handle(closure.ev);
828	1477	const fiber = closure.fiber;
829		std.log.debug("{*} performing async", .{fiber});
	1478	log.debug("{*} performing async", .{fiber});
830	1479	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
831		const awaiter = @atomicRmw(?*Fiber, &fiber.awaiter, .Xchg, Fiber.finished, .acq_rel);
832		const ready_awaiter = r: {
833		const a = awaiter orelse break :r null;
834		if (@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .acq_rel)) break :r null;
835		break :r a;
836		};
837		closure.event_loop.yield(ready_awaiter, .nothing);
	1480	closure.ev.yield(
	1481	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
	1482	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
	1483	else
	1484	null,
	1485	.nothing,
	1486	);
838	1487	unreachable; // switched to dead fiber
839	1488	}
840
841		fn fromFiber(fiber: Fiber) AsyncClosure {
842		return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
843		@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
844		) - @sizeOf(AsyncClosure));
845		}
846	1489	};
847	1490
848	1491	fn async(
...	...	@@ -853,7 +1496,8 @@ fn async(
853	1496	context_alignment: Alignment,
854	1497	start: const fn (context: const anyopaque, result: *anyopaque) void,
855	1498	) ?*std.Io.AnyFuture {
856		return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch {
	1499	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1500	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
857	1501	start(context.ptr, result.ptr);
858	1502	return null;
859	1503	};
...	...	@@ -872,626 +1516,4588 @@ fn concurrent(
872	1516	assert(result_len <= Fiber.max_result_size); // TODO
873	1517	assert(context.len <= Fiber.max_context_size); // TODO
874	1518
875		const event_loop: *EventLoop = @ptrCast(@alignCast(userdata));
876		const fiber = try Fiber.allocate(event_loop);
877		std.log.debug("allocated {*}", .{fiber});
	1519	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1520	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1521	error.OutOfMemory => return error.ConcurrencyUnavailable,
	1522	};
	1523	log.debug("allocated {*}", .{fiber});
878	1524
879	1525	const closure: *AsyncClosure = .fromFiber(fiber);
880	1526	fiber.* = .{
881	1527	.required_align = {},
882	1528	.context = switch (builtin.cpu.arch) {
883		.x86_64 => .{
884		.rsp = @intFromPtr(closure) - @sizeOf(usize),
885		.rbp = 0,
886		.rip = @intFromPtr(&fiberEntry),
887		},
888	1529	.aarch64 => .{
889	1530	.sp = @intFromPtr(closure),
890	1531	.fp = 0,
891		.pc = @intFromPtr(&fiberEntry),
	1532	.pc = @intFromPtr(&AsyncClosure.entry),
	1533	},
	1534	.x86_64 => .{
	1535	.rsp = @intFromPtr(closure) - @sizeOf(usize),
	1536	.rbp = 0,
	1537	.rip = @intFromPtr(&AsyncClosure.entry),
892	1538	},
893	1539	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
894	1540	},
895		.awaiter = null,
896		.queue_next = null,
897		.cancel_thread = null,
898		.awaiting_completions = .initEmpty(),
	1541	.await_count = 0,
	1542	.link = .{ .awaiter = null },
	1543	.status = .{ .queue_next = null },
	1544	.cancel_status = .unrequested,
	1545	.cancel_protection = .unblocked,
899	1546	};
900	1547	closure.* = .{
901		.event_loop = event_loop,
	1548	.ev = ev,
902	1549	.fiber = fiber,
903	1550	.start = start,
904	1551	.result_align = result_alignment,
905		.already_awaited = false,
906	1552	};
907	1553	@memcpy(closure.contextPointer(), context);
908	1554
909		event_loop.schedule(.current(), .{ .head = fiber, .tail = fiber });
	1555	const thread: *Thread = .current();
	1556	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
910	1557	return @ptrCast(fiber);
911	1558	}
912	1559
913	1560	fn await(
914	1561	userdata: ?*anyopaque,
915		any_future: *std.Io.AnyFuture,
	1562	future: *std.Io.AnyFuture,
916	1563	result: []u8,
917	1564	result_alignment: Alignment,
918	1565	) void {
919		const event_loop: *EventLoop = @ptrCast(@alignCast(userdata));
920		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
921		if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished)
922		event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter });
	1566	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1567	const fiber = Thread.current().currentFiber();
	1568	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1569	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
	1570	assert(awaiter == Fiber.finished);
	1571	} else while (true) {
	1572	ev.yield(null, .{ .await = 1 });
	1573	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
	1574	if (awaiter == Fiber.finished) break;
	1575	assert(awaiter == fiber); // spurious wakeup
	1576	}
923	1577	@memcpy(result, future_fiber.resultBytes(result_alignment));
924		event_loop.recycle(future_fiber);
	1578	future_fiber.destroy(ev.allocator());
925	1579	}
926	1580
927		fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) usize {
928		const el: *EventLoop = @ptrCast(@alignCast(userdata));
	1581	fn cancel(
	1582	userdata: ?*anyopaque,
	1583	future: *std.Io.AnyFuture,
	1584	result: []u8,
	1585	result_alignment: Alignment,
	1586	) void {
	1587	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1588	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1589	future_fiber.requestCancel(ev);
	1590	await(ev, future, result, result_alignment);
	1591	}
	1592
	1593	const Group = struct {
	1594	ptr: *Io.Group,
	1595
	1596	const List = packed struct(usize) {
	1597	cancel_requested: bool,
	1598	awaiter_delayed: bool,
	1599	fibers: Fiber.PackedPtr,
	1600	};
	1601	fn listPtr(group: Group) *List {
	1602	return @ptrCast(&group.ptr.token);
	1603	}
	1604
	1605	const Mutex = packed struct(u32) {
	1606	locked: bool,
	1607	contended: bool,
	1608	shared2: u30,
	1609	};
	1610	fn mutexPtr(group: Group) *Mutex {
	1611	return switch (comptime builtin.cpu.arch.endian()) {
	1612	.little => @ptrCast(&group.ptr.state),
	1613	.big => @ptrCast(@alignCast(
	1614	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
	1615	)),
	1616	};
	1617	}
	1618
	1619	const Awaiter = packed struct(usize) {
	1620	locked: bool,
	1621	contended: bool,
	1622	awaiter: Fiber.PackedPtr,
	1623	};
	1624	fn awaiterPtr(group: Group) *Awaiter {
	1625	return @ptrCast(&group.ptr.state);
	1626	}
	1627
	1628	fn lock(group: Group, ev: *Evented) void {
	1629	const mutex = group.mutexPtr();
	1630	{
	1631	const old_state = @atomicRmw(
	1632	Mutex,
	1633	mutex,
	1634	.Or,
	1635	.{ .locked = true, .contended = false, .shared2 = 0 },
	1636	.acquire,
	1637	);
	1638	if (!old_state.locked) {
	1639	@branchHint(.likely);
	1640	return;
	1641	}
	1642	if (old_state.contended) {
	1643	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1644	}
	1645	}
	1646	while (true) {
	1647	var old_state = @atomicRmw(
	1648	Mutex,
	1649	mutex,
	1650	.Or,
	1651	.{ .locked = true, .contended = true, .shared2 = 0 },
	1652	.acquire,
	1653	);
	1654	if (!old_state.locked) {
	1655	@branchHint(.likely);
	1656	return;
	1657	}
	1658	old_state.contended = true;
	1659	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1660	}
	1661	}
	1662
	1663	fn unlock(group: Group, ev: *Evented) void {
	1664	const mutex = group.mutexPtr();
	1665	const old_state = @atomicRmw(
	1666	Mutex,
	1667	mutex,
	1668	.And,
	1669	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
	1670	.release,
	1671	);
	1672	assert(old_state.locked);
	1673	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
	1674	}
929	1675
930		// Optimization to avoid the yield below.
931		for (futures, 0..) \|any_future, i\| {
932		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
933		if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) == Fiber.finished)
934		return i;
	1676	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
	1677	group.lock(ev);
	1678	defer group.unlock(ev);
	1679	const list_ptr = group.listPtr();
	1680	const list = @atomicLoad(List, list_ptr, .monotonic);
	1681	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
	1682	const old_head = list.fibers.unpack();
	1683	if (old_head) \|head\| head.link.group.prev = fiber;
	1684	fiber.link.group.next = old_head;
	1685	@atomicStore(List, list_ptr, .{
	1686	.cancel_requested = list.cancel_requested,
	1687	.awaiter_delayed = list.awaiter_delayed,
	1688	.fibers = .pack(fiber),
	1689	}, .monotonic);
935	1690	}
936	1691
937		el.yield(null, .{ .register_select = futures });
	1692	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
	1693	group.lock(ev);
	1694	defer group.unlock(ev);
	1695	const list_ptr = group.listPtr();
	1696	const list = @atomicLoad(List, list_ptr, .monotonic);
	1697	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
	1698	if (fiber.link.group.prev) \|prev\| {
	1699	prev.link.group.next = fiber.link.group.next;
	1700	} else if (fiber.link.group.next) \|new_head\| {
	1701	@atomicStore(List, list_ptr, .{
	1702	.cancel_requested = list.cancel_requested,
	1703	.awaiter_delayed = list.awaiter_delayed,
	1704	.fibers = .pack(new_head),
	1705	}, .monotonic);
	1706	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
	1707	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
	1708	@atomicStore(List, list_ptr, .{
	1709	.cancel_requested = false,
	1710	.awaiter_delayed = false,
	1711	.fibers = .null,
	1712	}, .release);
	1713	assert(awaiter.status.awaiting_group.ptr == group.ptr);
	1714	awaiter.status = .{ .queue_next = null };
	1715	return awaiter;
	1716	}
	1717	// Race with `Fiber.requestCancel`
	1718	@atomicStore(List, list_ptr, .{
	1719	.cancel_requested = false,
	1720	.awaiter_delayed = true,
	1721	.fibers = .null,
	1722	}, .monotonic);
	1723	} else @atomicStore(List, list_ptr, .{
	1724	.cancel_requested = false,
	1725	.awaiter_delayed = false,
	1726	.fibers = .null,
	1727	}, .release);
	1728	return null;
	1729	}
938	1730
939		std.log.debug("back from select yield", .{});
	1731	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
	1732	group.lock(ev);
	1733	defer group.unlock(ev);
	1734	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
	1735	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
	1736	// The awaiter already had an unacknowledged cancelation request before
	1737	// attempting to await a group, so propagate the cancelation to the group.
	1738	assert(!group.cancelLocked(ev, null));
	1739	}
	1740	return false;
	1741	}
	1742	return true;
	1743	}
940	1744
941		const my_thread: *Thread = .current();
942		const my_fiber = my_thread.currentFiber();
943		var result: ?usize = null;
	1745	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1746	group.lock(ev);
	1747	defer group.unlock(ev);
	1748	return group.cancelLocked(ev, maybe_awaiter);
	1749	}
944	1750
945		for (futures, 0..) \|any_future, i\| {
946		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
947		if (@cmpxchgStrong(?*Fiber, &future_fiber.awaiter, my_fiber, null, .seq_cst, .seq_cst)) \|awaiter\| {
948		if (awaiter == Fiber.finished) {
949		if (result == null) result = i;
950		} else if (awaiter) \|a\| {
951		const closure: *AsyncClosure = .fromFiber(a);
952		closure.already_awaited = false;
	1751	/// Assumes the mutex is held.
	1752	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1753	const list_ptr = group.listPtr();
	1754	const list = @atomicRmw(
	1755	List,
	1756	list_ptr,
	1757	.Add,
	1758	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
	1759	.monotonic,
	1760	);
	1761	assert(!list.cancel_requested);
	1762	if (list.fibers.unpack()) \|head\| {
	1763	var maybe_fiber: ?*Fiber = head;
	1764	while (maybe_fiber) \|fiber\| {
	1765	fiber.requestCancel(ev);
	1766	maybe_fiber = fiber.link.group.next;
953	1767	}
954		} else {
955		const closure: *AsyncClosure = .fromFiber(my_fiber);
956		closure.already_awaited = false;
	1768	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
	1769	return false;
957	1770	}
	1771	@atomicStore(
	1772	List,
	1773	list_ptr,
	1774	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
	1775	.release,
	1776	);
	1777	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
958	1778	}
959	1779
960		return result.?;
961		}
	1780	/// Assumes the mutex is held.
	1781	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
	1782	assert(awaiter.status.queue_next == null);
	1783	awaiter.status = .{ .awaiting_group = group };
	1784	assert(@atomicRmw(
	1785	Awaiter,
	1786	group.awaiterPtr(),
	1787	.Add,
	1788	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
	1789	.monotonic,
	1790	).awaiter == .null);
	1791	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
	1792	}
962	1793
963		fn cancel(
	1794	const AsyncClosure = struct {
	1795	ev: *Evented,
	1796	group: Group,
	1797	fiber: *Fiber,
	1798	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1799
	1800	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
	1801	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
	1802	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1803	) - @sizeOf(Group.AsyncClosure));
	1804	}
	1805
	1806	fn contextPointer(
	1807	closure: *Group.AsyncClosure,
	1808	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
	1809	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
	1810	}
	1811
	1812	fn entry() callconv(.naked) void {
	1813	switch (builtin.cpu.arch) {
	1814	.aarch64 => asm volatile (
	1815	\\ mov x0, sp
	1816	\\ b %[call]
	1817	:
	1818	: [call] "X" (&call),
	1819	),
	1820	.x86_64 => asm volatile (
	1821	\\ leaq 8(%%rsp), %%rdi
	1822	\\ jmp %[call:P]
	1823	:
	1824	: [call] "X" (&call),
	1825	),
	1826	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1827	}
	1828	}
	1829
	1830	fn call(
	1831	closure: *Group.AsyncClosure,
	1832	message: *const SwitchMessage,
	1833	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
	1834	message.handle(closure.ev);
	1835	assert(closure.fiber.status.queue_next == null);
	1836	log.debug("{*} performing group async", .{closure.fiber});
	1837	const result = closure.start(closure.contextPointer());
	1838	const ev = closure.ev;
	1839	const group = closure.group;
	1840	const fiber = closure.fiber;
	1841	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
	1842	if (result) {
	1843	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1844	} else \|err\| switch (err) {
	1845	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
	1846	}
	1847	const awaiter = group.removeFiber(ev, fiber);
	1848	ev.backing_allocator_mutex.lockUncancelable(ev.io());
	1849	ev.yield(awaiter, .destroy);
	1850	unreachable; // switched to dead fiber
	1851	}
	1852	};
	1853	};
	1854
	1855	fn groupAsync(
964	1856	userdata: ?*anyopaque,
965		any_future: *std.Io.AnyFuture,
966		result: []u8,
967		result_alignment: Alignment,
	1857	type_erased: *Io.Group,
	1858	context: []const u8,
	1859	context_alignment: Alignment,
	1860	start: const fn (context: const anyopaque) Io.Cancelable!void,
968	1861	) void {
969		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
970		if (@atomicRmw(
971		?*Thread,
972		&future_fiber.cancel_thread,
973		.Xchg,
974		Thread.canceling,
975		.acq_rel,
976		)) \|cancel_thread\| if (cancel_thread != Thread.canceling) {
977		getSqe(&Thread.current().io_uring).* = .{
978		.opcode = .MSG_RING,
979		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
980		.ioprio = 0,
981		.fd = cancel_thread.io_uring.fd,
982		.off = @intFromPtr(future_fiber),
983		.addr = 0,
984		.len = @bitCast(-@as(i32, @intFromEnum(std.os.linux.E.INTR))),
985		.rw_flags = 0,
986		.user_data = @intFromEnum(Completion.UserData.cleanup),
987		.buf_index = 0,
988		.personality = 0,
989		.splice_fd_in = 0,
990		.addr3 = 0,
991		.resv = 0,
992		};
	1862	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1863	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
	1864	const fiber = Thread.current().currentFiber();
	1865	const pre_acknowledged = fiber.cancel_protection.acknowledged;
	1866	const result = start(context.ptr);
	1867	const post_acknowledged = fiber.cancel_protection.acknowledged;
	1868	if (result) {
	1869	if (pre_acknowledged) {
	1870	assert(post_acknowledged); // group task called `recancel` but was not canceled
	1871	} else {
	1872	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1873	}
	1874	} else \|err\| switch (err) {
	1875	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
	1876	error.Canceled => {
	1877	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
	1878	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
	1879	recancel(userdata);
	1880	},
	1881	}
993	1882	};
994		await(userdata, any_future, result, result_alignment);
995		}
996
997		fn cancelRequested(userdata: ?*anyopaque) bool {
998		_ = userdata;
999		return @atomicLoad(?*Thread, &Thread.current().currentFiber().cancel_thread, .acquire) == Thread.canceling;
1000	1883	}
1001	1884
1002		fn createFile(
	1885	fn groupConcurrent(
1003	1886	userdata: ?*anyopaque,
1004		dir: Io.Dir,
1005		sub_path: []const u8,
1006		flags: Io.File.CreateFlags,
1007		) Io.File.OpenError!Io.File {
1008		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1009		const thread: *Thread = .current();
1010		const iou = &thread.io_uring;
1011		const fiber = thread.currentFiber();
1012		try fiber.enterCancelRegion(thread);
1013
1014		const posix = std.posix;
1015		const sub_path_c = try posix.toPosixPath(sub_path);
	1887	type_erased: *Io.Group,
	1888	context: []const u8,
	1889	context_alignment: Alignment,
	1890	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1891	) Io.ConcurrentError!void {
	1892	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
	1893	assert(context.len <= Fiber.max_context_size); // TODO
1016	1894
1017		var os_flags: posix.O = .{
1018		.ACCMODE = if (flags.read) .RDWR else .WRONLY,
1019		.CREAT = true,
1020		.TRUNC = flags.truncate,
1021		.EXCL = flags.exclusive,
	1895	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1896	const group: Group = .{ .ptr = type_erased };
	1897	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1898	error.OutOfMemory => return error.ConcurrencyUnavailable,
1022	1899	};
1023		if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true;
1024		if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true;
	1900	log.debug("allocated {*}", .{fiber});
1025	1901
1026		// Use the O locking flags if the os supports them to acquire the lock
1027		// atomically. Note that the NONBLOCK flag is removed after the openat()
1028		// call is successful.
1029		const has_flock_open_flags = @hasField(posix.O, "EXLOCK");
1030		if (has_flock_open_flags) switch (flags.lock) {
1031		.none => {},
1032		.shared => {
1033		os_flags.SHLOCK = true;
1034		os_flags.NONBLOCK = flags.lock_nonblocking;
1035		},
1036		.exclusive => {
1037		os_flags.EXLOCK = true;
1038		os_flags.NONBLOCK = flags.lock_nonblocking;
	1902	const closure: *Group.AsyncClosure = .fromFiber(fiber);
	1903	fiber.* = .{
	1904	.required_align = {},
	1905	.context = switch (builtin.cpu.arch) {
	1906	.aarch64 => .{
	1907	.sp = @intFromPtr(closure),
	1908	.fp = 0,
	1909	.pc = @intFromPtr(&Group.AsyncClosure.entry),
	1910	},
	1911	.x86_64 => .{
	1912	.rsp = @intFromPtr(closure) - @sizeOf(usize),
	1913	.rbp = 0,
	1914	.rip = @intFromPtr(&Group.AsyncClosure.entry),
	1915	},
	1916	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1039	1917	},
	1918	.await_count = 0,
	1919	.link = .{ .group = .{ .prev = null, .next = null } },
	1920	.status = .{ .queue_next = null },
	1921	.cancel_status = .unrequested,
	1922	.cancel_protection = .unblocked,
1040	1923	};
1041		const have_flock = @TypeOf(posix.system.flock) != void;
1042
1043		if (have_flock and !has_flock_open_flags and flags.lock != .none) {
1044		@panic("TODO");
1045		}
1046
1047		if (has_flock_open_flags and flags.lock_nonblocking) {
1048		@panic("TODO");
1049		}
1050
1051		getSqe(iou).* = .{
1052		.opcode = .OPENAT,
1053		.flags = 0,
1054		.ioprio = 0,
1055		.fd = dir.handle,
1056		.off = 0,
1057		.addr = @intFromPtr(&sub_path_c),
1058		.len = @intCast(flags.mode),
1059		.rw_flags = @bitCast(os_flags),
1060		.user_data = @intFromPtr(fiber),
1061		.buf_index = 0,
1062		.personality = 0,
1063		.splice_fd_in = 0,
1064		.addr3 = 0,
1065		.resv = 0,
	1924	closure.* = .{
	1925	.ev = ev,
	1926	.group = group,
	1927	.fiber = fiber,
	1928	.start = start,
1066	1929	};
	1930	@memcpy(closure.contextPointer(), context);
	1931	group.addFiber(ev, fiber);
	1932	const thread: *Thread = .current();
	1933	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
	1934	}
1067	1935
1068		el.yield(null, .nothing);
1069		fiber.exitCancelRegion(thread);
1070
1071		const completion = fiber.resultPointer(Completion);
1072		switch (errno(completion.result)) {
1073		.SUCCESS => return .{ .handle = completion.result },
1074		.INTR => unreachable,
1075		.CANCELED => return error.Canceled,
	1936	fn groupAwait(
	1937	userdata: ?*anyopaque,
	1938	type_erased: *Io.Group,
	1939	initial_token: *anyopaque,
	1940	) Io.Cancelable!void {
	1941	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1942	_ = initial_token;
	1943	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
	1944	}
1076	1945
1077		.FAULT => unreachable,
1078		.INVAL => return error.BadPathName,
1079		.BADF => unreachable,
1080		.ACCES => return error.AccessDenied,
1081		.FBIG => return error.FileTooBig,
1082		.OVERFLOW => return error.FileTooBig,
1083		.ISDIR => return error.IsDir,
1084		.LOOP => return error.SymLinkLoop,
1085		.MFILE => return error.ProcessFdQuotaExceeded,
1086		.NAMETOOLONG => return error.NameTooLong,
1087		.NFILE => return error.SystemFdQuotaExceeded,
1088		.NODEV => return error.NoDevice,
1089		.NOENT => return error.FileNotFound,
1090		.NOMEM => return error.SystemResources,
1091		.NOSPC => return error.NoSpaceLeft,
1092		.NOTDIR => return error.NotDir,
1093		.PERM => return error.PermissionDenied,
1094		.EXIST => return error.PathAlreadyExists,
1095		.BUSY => return error.DeviceBusy,
1096		.OPNOTSUPP => return error.FileLocksUnsupported,
1097		.AGAIN => return error.WouldBlock,
1098		.TXTBSY => return error.FileBusy,
1099		.NXIO => return error.NoDevice,
1100		else => \|err\| return posix.unexpectedErrno(err),
1101		}
	1946	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
	1947	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1948	_ = initial_token;
	1949	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
1102	1950	}
1103	1951
1104		fn fileOpen(
1105		userdata: ?*anyopaque,
1106		dir: Io.Dir,
1107		sub_path: []const u8,
1108		flags: Io.File.OpenFlags,
1109		) Io.File.OpenError!Io.File {
1110		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1111		const thread: *Thread = .current();
1112		const iou = &thread.io_uring;
1113		const fiber = thread.currentFiber();
1114		try fiber.enterCancelRegion(thread);
	1952	fn recancel(userdata: ?*anyopaque) void {
	1953	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1954	_ = ev;
	1955	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
	1956	assert(cancel_protection.acknowledged);
	1957	cancel_protection.acknowledged = false;
	1958	}
1115	1959
1116		const posix = std.posix;
1117		const sub_path_c = try posix.toPosixPath(sub_path);
	1960	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
	1961	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1962	_ = ev;
	1963	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
	1964	defer cancel_protection.user = new;
	1965	return cancel_protection.user;
	1966	}
1118	1967
1119		var os_flags: posix.O = .{
1120		.ACCMODE = switch (flags.mode) {
1121		.read_only => .RDONLY,
1122		.write_only => .WRONLY,
1123		.read_write => .RDWR,
	1968	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
	1969	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1970	_ = ev;
	1971	const fiber = Thread.current().currentFiber();
	1972	switch (fiber.cancel_protection.check()) {
	1973	.blocked => {},
	1974	.unblocked => if (@atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic).requested) {
	1975	fiber.cancel_protection.acknowledge();
	1976	return error.Canceled;
1124	1977	},
1125		};
1126
1127		if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true;
1128		if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true;
1129		if (@hasField(posix.O, "NOCTTY")) os_flags.NOCTTY = !flags.allow_ctty;
1130
1131		// Use the O locking flags if the os supports them to acquire the lock
1132		// atomically.
1133		const has_flock_open_flags = @hasField(posix.O, "EXLOCK");
1134		if (has_flock_open_flags) {
1135		// Note that the NONBLOCK flag is removed after the openat() call
1136		// is successful.
1137		switch (flags.lock) {
1138		.none => {},
1139		.shared => {
1140		os_flags.SHLOCK = true;
1141		os_flags.NONBLOCK = flags.lock_nonblocking;
1142		},
1143		.exclusive => {
1144		os_flags.EXLOCK = true;
1145		os_flags.NONBLOCK = flags.lock_nonblocking;
1146		},
1147		}
1148	1978	}
1149		const have_flock = @TypeOf(posix.system.flock) != void;
	1979	}
1150	1980
1151		if (have_flock and !has_flock_open_flags and flags.lock != .none) {
1152		@panic("TODO");
	1981	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
	1982	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1983	var cancel_region: CancelRegion = .init();
	1984	defer cancel_region.deinit();
	1985	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
	1986	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1987	if (@atomicRmw(
	1988	?*Fiber,
	1989	&future_fiber.link.awaiter,
	1990	.Xchg,
	1991	cancel_region.fiber,
	1992	.acq_rel,
	1993	)) \|awaiter\| {
	1994	assert(awaiter == Fiber.finished);
	1995	break .{ @intCast(future_index), future_index };
	1996	}
	1997	} else result: {
	1998	const await_count: u31 = @intCast(futures.len);
	1999	cancel_region.await(.select) catch \|err\| switch (err) {
	2000	error.Canceled => \|e\| break :result .{ await_count + 1, e },
	2001	};
	2002	ev.yield(null, .{ .await = 1 });
	2003	cancel_region.await(.nothing) catch \|err\| switch (err) {
	2004	error.Canceled => \|e\| break :result .{ await_count, e },
	2005	};
	2006	break :result .{ await_count - 1, futures.len };
	2007	};
	2008	for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
	2009	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	2010	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
	2011	if (awaiter == Fiber.finished) {
	2012	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
	2013	result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
	2014	} else {
	2015	assert(awaiter == cancel_region.fiber);
	2016	await_count -= 1;
	2017	}
1153	2018	}
1154
1155		if (has_flock_open_flags and flags.lock_nonblocking) {
1156		@panic("TODO");
	2019	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
	2020	// but avoiding a context switch in the common case.
	2021	switch (std.math.order(
	2022	@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
	2023	await_count,
	2024	)) {
	2025	.lt => ev.yield(null, .{ .await = 0 }),
	2026	.eq => {},
	2027	.gt => unreachable,
1157	2028	}
	2029	return result;
	2030	}
1158	2031
1159		getSqe(iou).* = .{
1160		.opcode = .OPENAT,
1161		.flags = 0,
	2032	fn futexWait(
	2033	userdata: ?*anyopaque,
	2034	ptr: *const u32,
	2035	expected: u32,
	2036	timeout: Io.Timeout,
	2037	) Io.Cancelable!void {
	2038	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2039	if (builtin.single_threaded) unreachable; // Deadlock.
	2040	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
	2041	.none => .{
	2042	null,
	2043	.awake,
	2044	linux.IORING_TIMEOUT_ABS,
	2045	},
	2046	.duration => \|duration\| {
	2047	const ns = duration.raw.toNanoseconds();
	2048	break :timespec .{
	2049	.{
	2050	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2051	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2052	},
	2053	duration.clock,
	2054	0,
	2055	};
	2056	},
	2057	.deadline => \|deadline\| {
	2058	const ns = deadline.raw.toNanoseconds();
	2059	break :timespec .{
	2060	.{
	2061	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2062	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2063	},
	2064	deadline.clock,
	2065	linux.IORING_TIMEOUT_ABS,
	2066	};
	2067	},
	2068	};
	2069	var cancel_region: CancelRegion = .init();
	2070	defer cancel_region.deinit();
	2071	const thread = try cancel_region.awaitIoUring();
	2072	thread.enqueue().* = .{
	2073	.opcode = .FUTEX_WAIT,
	2074	.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
1162	2075	.ioprio = 0,
1163		.fd = dir.handle,
1164		.off = 0,
1165		.addr = @intFromPtr(&sub_path_c),
	2076	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2077	.off = expected,
	2078	.addr = @intFromPtr(ptr),
1166	2079	.len = 0,
1167		.rw_flags = @bitCast(os_flags),
1168		.user_data = @intFromPtr(fiber),
	2080	.rw_flags = 0,
	2081	.user_data = @intFromPtr(cancel_region.fiber),
1169	2082	.buf_index = 0,
1170	2083	.personality = 0,
1171	2084	.splice_fd_in = 0,
1172		.addr3 = 0,
	2085	.addr3 = std.math.maxInt(u32),
1173	2086	.resv = 0,
1174	2087	};
1175
1176		el.yield(null, .nothing);
1177		fiber.exitCancelRegion(thread);
1178
1179		const completion = fiber.resultPointer(Completion);
1180		switch (errno(completion.result)) {
1181		.SUCCESS => return .{ .handle = completion.result },
1182		.INTR => unreachable,
1183		.CANCELED => return error.Canceled,
1184
1185		.FAULT => unreachable,
1186		.INVAL => return error.BadPathName,
1187		.BADF => unreachable,
1188		.ACCES => return error.AccessDenied,
1189		.FBIG => return error.FileTooBig,
1190		.OVERFLOW => return error.FileTooBig,
1191		.ISDIR => return error.IsDir,
1192		.LOOP => return error.SymLinkLoop,
1193		.MFILE => return error.ProcessFdQuotaExceeded,
1194		.NAMETOOLONG => return error.NameTooLong,
1195		.NFILE => return error.SystemFdQuotaExceeded,
1196		.NODEV => return error.NoDevice,
1197		.NOENT => return error.FileNotFound,
1198		.NOMEM => return error.SystemResources,
1199		.NOSPC => return error.NoSpaceLeft,
1200		.NOTDIR => return error.NotDir,
1201		.PERM => return error.PermissionDenied,
1202		.EXIST => return error.PathAlreadyExists,
1203		.BUSY => return error.DeviceBusy,
1204		.OPNOTSUPP => return error.FileLocksUnsupported,
1205		.AGAIN => return error.WouldBlock,
1206		.TXTBSY => return error.FileBusy,
1207		.NXIO => return error.NoDevice,
1208		else => \|err\| return posix.unexpectedErrno(err),
1209		}
1210		}
1211
1212		fn fileClose(userdata: ?*anyopaque, file: Io.File) void {
1213		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1214		const thread: *Thread = .current();
1215		const iou = &thread.io_uring;
1216		const fiber = thread.currentFiber();
1217
1218		getSqe(iou).* = .{
1219		.opcode = .CLOSE,
1220		.flags = 0,
	2088	if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
	2089	.opcode = .LINK_TIMEOUT,
	2090	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1221	2091	.ioprio = 0,
1222		.fd = file.handle,
	2092	.fd = 0,
1223	2093	.off = 0,
1224		.addr = 0,
1225		.len = 0,
1226		.rw_flags = 0,
1227		.user_data = @intFromPtr(fiber),
	2094	.addr = @intFromPtr(timespec_ptr),
	2095	.len = 1,
	2096	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	2097	.real => linux.IORING_TIMEOUT_REALTIME,
	2098	else => 0,
	2099	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	2100	}),
	2101	.user_data = @intFromEnum(Completion.UserData.wakeup),
1228	2102	.buf_index = 0,
1229	2103	.personality = 0,
1230	2104	.splice_fd_in = 0,
1231	2105	.addr3 = 0,
1232	2106	.resv = 0,
1233	2107	};
1234
1235		el.yield(null, .nothing);
1236
1237		const completion = fiber.resultPointer(Completion);
1238		switch (errno(completion.result)) {
1239		.SUCCESS => return,
1240		.INTR => unreachable,
1241		.CANCELED => return,
1242
1243		.BADF => unreachable, // Always a race condition.
1244		else => return,
	2108	ev.yield(null, .nothing);
	2109	switch (cancel_region.errno()) {
	2110	.SUCCESS => {}, // notified by `wake()`
	2111	.INTR, .CANCELED => {}, // caller's responsibility to retry
	2112	.AGAIN => {}, // ptr.* != expect
	2113	.INVAL => {}, // possibly timeout overflow
	2114	.TIMEDOUT => unreachable,
	2115	.FAULT => recoverableOsBugDetected(), // ptr was invalid
	2116	else => recoverableOsBugDetected(),
1245	2117	}
1246	2118	}
1247	2119
1248		fn pread(userdata: ?*anyopaque, file: Io.File, buffer: []u8, offset: std.posix.off_t) Io.File.PReadError!usize {
1249		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1250		const thread: *Thread = .current();
1251		const iou = &thread.io_uring;
1252		const fiber = thread.currentFiber();
1253		try fiber.enterCancelRegion(thread);
1254
1255		getSqe(iou).* = .{
1256		.opcode = .READ,
	2120	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
	2121	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2122	if (builtin.single_threaded) unreachable; // Deadlock.
	2123	var cancel_region: CancelRegion = .initBlocked();
	2124	defer cancel_region.deinit();
	2125	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	2126	error.Canceled => unreachable, // blocked
	2127	};
	2128	thread.enqueue().* = .{
	2129	.opcode = .FUTEX_WAIT,
1257	2130	.flags = 0,
1258	2131	.ioprio = 0,
1259		.fd = file.handle,
1260		.off = @bitCast(offset),
1261		.addr = @intFromPtr(buffer.ptr),
1262		.len = @min(buffer.len, 0x7ffff000),
	2132	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2133	.off = expected,
	2134	.addr = @intFromPtr(ptr),
	2135	.len = 0,
1263	2136	.rw_flags = 0,
1264		.user_data = @intFromPtr(fiber),
	2137	.user_data = @intFromPtr(cancel_region.fiber),
1265	2138	.buf_index = 0,
1266	2139	.personality = 0,
1267	2140	.splice_fd_in = 0,
1268		.addr3 = 0,
	2141	.addr3 = std.math.maxInt(u32),
1269	2142	.resv = 0,
1270	2143	};
1271
1272		el.yield(null, .nothing);
1273		fiber.exitCancelRegion(thread);
1274
1275		const completion = fiber.resultPointer(Completion);
1276		switch (errno(completion.result)) {
1277		.SUCCESS => return @as(u32, @bitCast(completion.result)),
1278		.INTR => unreachable,
1279		.CANCELED => return error.Canceled,
1280
1281		.INVAL => unreachable,
1282		.FAULT => unreachable,
1283		.NOENT => return error.ProcessNotFound,
1284		.AGAIN => return error.WouldBlock,
1285		.BADF => return error.NotOpenForReading, // Can be a race condition.
1286		.IO => return error.InputOutput,
1287		.ISDIR => return error.IsDir,
1288		.NOBUFS => return error.SystemResources,
1289		.NOMEM => return error.SystemResources,
1290		.NOTCONN => return error.SocketUnconnected,
1291		.CONNRESET => return error.ConnectionResetByPeer,
1292		.TIMEDOUT => return error.Timeout,
1293		.NXIO => return error.Unseekable,
1294		.SPIPE => return error.Unseekable,
1295		.OVERFLOW => return error.Unseekable,
1296		else => \|err\| return std.posix.unexpectedErrno(err),
	2144	ev.yield(null, .nothing);
	2145	switch (cancel_region.errno()) {
	2146	.SUCCESS => {}, // notified by `wake()`
	2147	.INTR, .CANCELED => {}, // caller's responsibility to retry
	2148	.AGAIN => {}, // ptr.* != expect
	2149	.INVAL => {}, // possibly timeout overflow
	2150	.FAULT => recoverableOsBugDetected(), // ptr was invalid
	2151	else => recoverableOsBugDetected(),
1297	2152	}
1298	2153	}
1299	2154
1300		fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: std.posix.off_t) Io.File.PWriteError!usize {
1301		const el: *EventLoop = @ptrCast(@alignCast(userdata));
	2155	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
	2156	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2157	_ = ev;
	2158	if (builtin.single_threaded) unreachable; // Nothing to wake up.
1302	2159	const thread: *Thread = .current();
1303		const iou = &thread.io_uring;
1304		const fiber = thread.currentFiber();
1305		try fiber.enterCancelRegion(thread);
1306
1307		getSqe(iou).* = .{
1308		.opcode = .WRITE,
1309		.flags = 0,
	2160	thread.enqueue().* = .{
	2161	.opcode = .FUTEX_WAKE,
	2162	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1310	2163	.ioprio = 0,
1311		.fd = file.handle,
1312		.off = @bitCast(offset),
1313		.addr = @intFromPtr(buffer.ptr),
1314		.len = @min(buffer.len, 0x7ffff000),
	2164	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2165	.off = max_waiters,
	2166	.addr = @intFromPtr(ptr),
	2167	.len = 0,
1315	2168	.rw_flags = 0,
1316		.user_data = @intFromPtr(fiber),
	2169	.user_data = @intFromEnum(Completion.UserData.futex_wake),
1317	2170	.buf_index = 0,
1318	2171	.personality = 0,
1319	2172	.splice_fd_in = 0,
1320		.addr3 = 0,
	2173	.addr3 = std.math.maxInt(u32),
1321	2174	.resv = 0,
1322	2175	};
	2176	thread.submit();
	2177	}
1323	2178
1324		el.yield(null, .nothing);
1325		fiber.exitCancelRegion(thread);
1326
1327		const completion = fiber.resultPointer(Completion);
1328		switch (errno(completion.result)) {
1329		.SUCCESS => return @as(u32, @bitCast(completion.result)),
1330		.INTR => unreachable,
1331		.CANCELED => return error.Canceled,
1332
1333		.INVAL => return error.InvalidArgument,
1334		.FAULT => unreachable,
1335		.NOENT => return error.ProcessNotFound,
1336		.AGAIN => return error.WouldBlock,
1337		.BADF => return error.NotOpenForWriting, // can be a race condition.
1338		.DESTADDRREQ => unreachable, // `connect` was never called.
1339		.DQUOT => return error.DiskQuota,
1340		.FBIG => return error.FileTooBig,
1341		.IO => return error.InputOutput,
1342		.NOSPC => return error.NoSpaceLeft,
1343		.ACCES => return error.AccessDenied,
1344		.PERM => return error.PermissionDenied,
1345		.PIPE => return error.BrokenPipe,
1346		.NXIO => return error.Unseekable,
1347		.SPIPE => return error.Unseekable,
1348		.OVERFLOW => return error.Unseekable,
1349		.BUSY => return error.DeviceBusy,
1350		.CONNRESET => return error.ConnectionResetByPeer,
1351		.MSGSIZE => return error.MessageOversize,
1352		else => \|err\| return std.posix.unexpectedErrno(err),
	2179	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
	2180	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2181	switch (operation) {
	2182	.file_read_streaming => \|o\| return .{
	2183	.file_read_streaming = ev.fileReadStreaming(o.file, o.data) catch \|err\| switch (err) {
	2184	error.Canceled => \|e\| return e,
	2185	else => \|e\| e,
	2186	},
	2187	},
	2188	.file_write_streaming => \|o\| return .{
	2189	.file_write_streaming = ev.fileWriteStreaming(o.file, o.header, o.data, o.splat) catch \|err\| switch (err) {
	2190	error.Canceled => \|e\| return e,
	2191	else => \|e\| e,
	2192	},
	2193	},
	2194	.device_io_control => \|*o\| return .{ .device_io_control = try deviceIoControl(o) },
1353	2195	}
1354	2196	}
1355	2197
1356		fn now(userdata: ?*anyopaque, clockid: std.posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp {
1357		_ = userdata;
1358		const timespec = try std.posix.clock_gettime(clockid);
1359		return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);
1360		}
	2198	fn fileReadStreaming(ev: *Evented, file: File, data: []const []u8) File.Reader.Error!usize {
	2199	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
	2200	var i: usize = 0;
	2201	for (data) \|buf\| {
	2202	if (iovecs_buffer.len - i == 0) break;
	2203	if (buf.len != 0) {
	2204	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
	2205	i += 1;
	2206	}
	2207	}
	2208	const dest = iovecs_buffer[0..i];
	2209	assert(dest[0].len > 0);
1361	2210
1362		fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void {
1363		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1364		const thread: *Thread = .current();
1365		const iou = &thread.io_uring;
1366		const fiber = thread.currentFiber();
1367		try fiber.enterCancelRegion(thread);
	2211	var cancel_region: CancelRegion = .init();
	2212	defer cancel_region.deinit();
	2213	return ev.preadv(&cancel_region, file.handle, dest, null);
	2214	}
1368	2215
1369		const deadline_nanoseconds: i96 = switch (deadline) {
1370		.duration => \|duration\| duration.nanoseconds,
1371		.timestamp => \|timestamp\| @intFromEnum(timestamp),
	2216	fn fileWriteStreaming(
	2217	ev: *Evented,
	2218	file: File,
	2219	header: []const u8,
	2220	data: []const []const u8,
	2221	splat: usize,
	2222	) File.Writer.Error!usize {
	2223	var iovecs: [max_iovecs_len]iovec_const = undefined;
	2224	var iovlen: iovlen_t = 0;
	2225	addBuf(&iovecs, &iovlen, header);
	2226	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
	2227	const pattern = data[data.len - 1];
	2228	if (iovecs.len - iovlen != 0) switch (splat) {
	2229	0 => {},
	2230	1 => addBuf(&iovecs, &iovlen, pattern),
	2231	else => switch (pattern.len) {
	2232	0 => {},
	2233	1 => {
	2234	var backup_buffer: [splat_buffer_size]u8 = undefined;
	2235	const splat_buffer = &backup_buffer;
	2236	const memset_len = @min(splat_buffer.len, splat);
	2237	const buf = splat_buffer[0..memset_len];
	2238	@memset(buf, pattern[0]);
	2239	addBuf(&iovecs, &iovlen, buf);
	2240	var remaining_splat = splat - buf.len;
	2241	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
	2242	assert(buf.len == splat_buffer.len);
	2243	addBuf(&iovecs, &iovlen, splat_buffer);
	2244	remaining_splat -= splat_buffer.len;
	2245	}
	2246	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	2247	},
	2248	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	2249	addBuf(&iovecs, &iovlen, pattern);
	2250	},
	2251	},
1372	2252	};
1373		const timespec: std.os.linux.kernel_timespec = .{
1374		.sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)),
1375		.nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)),
	2253
	2254	var cancel_region: CancelRegion = .init();
	2255	defer cancel_region.deinit();
	2256	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], null);
	2257	}
	2258
	2259	fn deviceIoControl(o: *const Io.Operation.DeviceIoControl) Io.Cancelable!i32 {
	2260	var cancel_region: CancelRegion = .init();
	2261	defer cancel_region.deinit();
	2262	while (true) {
	2263	try cancel_region.await(.nothing);
	2264	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
	2265	switch (linux.errno(rc)) {
	2266	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
	2267	.INTR => continue,
	2268	else => \|err\| return -@as(i32, @intFromEnum(err)),
	2269	}
	2270	}
	2271	}
	2272
	2273	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
	2274	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2275	var cancel_region: CancelRegion = .init();
	2276	defer cancel_region.deinit();
	2277	batchDrainSubmitted(batch, &cancel_region, false) catch \|err\| switch (err) {
	2278	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
	2279	else => \|e\| return e,
1376	2280	};
1377		getSqe(iou).* = .{
1378		.opcode = .TIMEOUT,
1379		.flags = 0,
1380		.ioprio = 0,
1381		.fd = 0,
	2281	while (true) {
	2282	batchDrainReady(batch) catch \|err\| switch (err) {
	2283	error.Timeout => unreachable, // no timeout
	2284	};
	2285	if (batch.completed.head != .none) return;
	2286	ev.yield(null, .{ .batch_await = batch });
	2287	}
	2288	}
	2289
	2290	fn batchAwaitConcurrent(
	2291	userdata: ?*anyopaque,
	2292	batch: *Io.Batch,
	2293	timeout: Io.Timeout,
	2294	) Io.Batch.AwaitConcurrentError!void {
	2295	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2296	var cancel_region: CancelRegion = .init();
	2297	defer cancel_region.deinit();
	2298	try batchDrainSubmitted(batch, &cancel_region, true);
	2299	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
	2300	batchDrainReady(batch) catch \|err\| switch (err) {
	2301	error.Timeout => unreachable, // no timeout
	2302	};
	2303	if (batch.completed.head != .none) return;
	2304	switch (timeout) {
	2305	.none => ev.yield(null, .{ .batch_await = batch }),
	2306	.duration => \|duration\| {
	2307	const ns = duration.raw.toNanoseconds();
	2308	break .{
	2309	.{
	2310	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2311	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2312	},
	2313	duration.clock,
	2314	0,
	2315	};
	2316	},
	2317	.deadline => \|deadline\| {
	2318	const ns = deadline.raw.toNanoseconds();
	2319	break .{
	2320	.{
	2321	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2322	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2323	},
	2324	deadline.clock,
	2325	linux.IORING_TIMEOUT_ABS,
	2326	};
	2327	},
	2328	}
	2329	};
	2330	{
	2331	const thread = try cancel_region.awaitIoUring();
	2332	thread.enqueue().* = .{
	2333	.opcode = .TIMEOUT,
	2334	.flags = 0,
	2335	.ioprio = 0,
	2336	.fd = 0,
	2337	.off = 0,
	2338	.addr = @intFromPtr(&timespec),
	2339	.len = 1,
	2340	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	2341	.real => linux.IORING_TIMEOUT_REALTIME,
	2342	else => 0,
	2343	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	2344	}),
	2345	.user_data = @intFromPtr(&batch.context) \| 0b11,
	2346	.buf_index = 0,
	2347	.personality = 0,
	2348	.splice_fd_in = 0,
	2349	.addr3 = 0,
	2350	.resv = 0,
	2351	};
	2352	}
	2353	while (batch.completed.head == .none) {
	2354	ev.yield(null, .{ .batch_await = batch });
	2355	batchDrainReady(batch) catch \|err\| switch (err) {
	2356	error.Timeout => \|e\| return if (batch.completed.head == .none) e,
	2357	};
	2358	if (batch.completed.head == .none) continue;
	2359	}
	2360	const thread = try cancel_region.awaitIoUring();
	2361	thread.enqueue().* = .{
	2362	.opcode = .TIMEOUT_REMOVE,
	2363	.flags = 0,
	2364	.ioprio = 0,
	2365	.fd = 0,
1382	2366	.off = 0,
1383		.addr = @intFromPtr(&timespec),
1384		.len = 1,
1385		.rw_flags = @as(u32, switch (deadline) {
1386		.duration => 0,
1387		.timestamp => std.os.linux.IORING_TIMEOUT_ABS,
1388		}) \| @as(u32, switch (clockid) {
1389		.REALTIME => std.os.linux.IORING_TIMEOUT_REALTIME,
1390		.MONOTONIC => 0,
1391		.BOOTTIME => std.os.linux.IORING_TIMEOUT_BOOTTIME,
1392		else => return error.UnsupportedClock,
1393		}),
1394		.user_data = @intFromPtr(fiber),
	2367	.addr = @intFromPtr(&batch.context) \| 0b11,
	2368	.len = 0,
	2369	.rw_flags = 0,
	2370	.user_data = @intFromPtr(cancel_region.fiber),
1395	2371	.buf_index = 0,
1396	2372	.personality = 0,
1397	2373	.splice_fd_in = 0,
1398	2374	.addr3 = 0,
1399	2375	.resv = 0,
1400	2376	};
	2377	ev.yield(null, .nothing);
	2378	switch (cancel_region.errno()) {
	2379	.SUCCESS => return,
	2380	.BUSY, .NOENT => {},
	2381	else => \|err\| unexpectedErrno(err) catch {},
	2382	}
	2383	while (true) {
	2384	batchDrainReady(batch) catch \|err\| switch (err) {
	2385	error.Timeout => return,
	2386	};
	2387	ev.yield(null, .{ .batch_await = batch });
	2388	}
	2389	}
1401	2390
1402		el.yield(null, .nothing);
1403		fiber.exitCancelRegion(thread);
1404
1405		const completion = fiber.resultPointer(Completion);
1406		switch (errno(completion.result)) {
1407		.SUCCESS, .TIME => return,
1408		.INTR => unreachable,
1409		.CANCELED => return error.Canceled,
	2391	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
	2392	fn batchDrainSubmitted(
	2393	batch: *Io.Batch,
	2394	cancel_region: *CancelRegion,
	2395	concurrency: bool,
	2396	) (Io.ConcurrentError \|\| Io.Cancelable)!void {
	2397	var index = batch.submitted.head;
	2398	if (index == .none) return;
	2399	errdefer batch.submitted.head = index;
	2400	const thread = try cancel_region.awaitIoUring();
	2401	while (index != .none) {
	2402	const storage = &batch.storage[index.toIndex()];
	2403	const next_index = storage.submission.node.next;
	2404	if (@as(?Io.Operation.Result, operation: switch (storage.submission.operation) {
	2405	.file_read_streaming => \|o\| {
	2406	const buffer = for (o.data) \|buffer\| {
	2407	if (buffer.len != 0) break buffer;
	2408	} else break :operation .{ .file_read_streaming = 0 };
	2409	const fd = o.file.handle;
	2410	storage.* = .{ .pending = .{
	2411	.node = .{ .prev = batch.pending.tail, .next = .none },
	2412	.tag = .file_read_streaming,
	2413	.context = undefined,
	2414	} };
	2415	thread.enqueue().* = .{
	2416	.opcode = .READ,
	2417	.flags = 0,
	2418	.ioprio = 0,
	2419	.fd = fd,
	2420	.off = std.math.maxInt(u64),
	2421	.addr = @intFromPtr(buffer.ptr),
	2422	.len = @min(buffer.len, 0xfffff000),
	2423	.rw_flags = 0,
	2424	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
	2425	.buf_index = 0,
	2426	.personality = 0,
	2427	.splice_fd_in = 0,
	2428	.addr3 = 0,
	2429	.resv = 0,
	2430	};
	2431	break :operation null;
	2432	},
	2433	.file_write_streaming => \|o\| {
	2434	const buffer = buffer: {
	2435	if (o.header.len != 0) break :buffer o.header;
	2436	for (o.data[0 .. o.data.len - 1]) \|buffer\| {
	2437	if (buffer.len != 0) break :buffer buffer;
	2438	}
	2439	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
	2440	break :operation .{ .file_write_streaming = 0 };
	2441	};
	2442	const fd = o.file.handle;
	2443	storage.* = .{ .pending = .{
	2444	.node = .{ .prev = batch.pending.tail, .next = .none },
	2445	.tag = .file_write_streaming,
	2446	.context = undefined,
	2447	} };
	2448	thread.enqueue().* = .{
	2449	.opcode = .WRITE,
	2450	.flags = 0,
	2451	.ioprio = 0,
	2452	.fd = fd,
	2453	.off = std.math.maxInt(u64),
	2454	.addr = @intFromPtr(buffer.ptr),
	2455	.len = @min(buffer.len, 0xfffff000),
	2456	.rw_flags = 0,
	2457	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
	2458	.buf_index = 0,
	2459	.personality = 0,
	2460	.splice_fd_in = 0,
	2461	.addr3 = 0,
	2462	.resv = 0,
	2463	};
	2464	break :operation null;
	2465	},
	2466	.device_io_control => \|o\| if (concurrency)
	2467	return error.ConcurrencyUnavailable
	2468	else
	2469	.{ .device_io_control = try deviceIoControl(&o) },
	2470	})) \|result\| {
	2471	switch (batch.completed.tail) {
	2472	.none => batch.completed.head = index,
	2473	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
	2474	}
	2475	batch.completed.tail = index;
	2476	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2477	} else {
	2478	switch (batch.pending.tail) {
	2479	.none => batch.pending.head = index,
	2480	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
	2481	}
	2482	batch.pending.tail = index;
	2483	storage.pending.context[0] = @intFromPtr(batch);
	2484	}
	2485	index = next_index;
	2486	}
	2487	batch.submitted = .{ .head = .none, .tail = .none };
	2488	}
1410	2489
1411		else => \|err\| return std.posix.unexpectedErrno(err),
	2490	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
	2491	while (@atomicRmw(?*anyopaque, &batch.context, .Xchg, null, .acquire)) \|head\| {
	2492	var next: usize = @intFromPtr(head);
	2493	var timeout = false;
	2494	while (cond: switch (@as(u2, @truncate(next))) {
	2495	0b00 => if (timeout) return error.Timeout else false,
	2496	0b01 => {
	2497	assert(!timeout);
	2498	return error.Timeout;
	2499	},
	2500	0b10 => true,
	2501	0b11 => {
	2502	assert(!timeout);
	2503	timeout = true;
	2504	break :cond true;
	2505	},
	2506	}) {
	2507	var context: *Io.Operation.Storage.Pending.Context = @ptrFromInt(next & ~@as(usize, 0b11));
	2508	next = context[0];
	2509	const completion: Completion = .{
	2510	.result = @bitCast(@as(u32, @intCast(context[1]))),
	2511	.flags = @intCast(context[2]),
	2512	};
	2513	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", context);
	2514	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
	2515	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
	2516	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
	2517	switch (pending.node.prev) {
	2518	.none => batch.pending.head = pending.node.next,
	2519	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
	2520	pending.node.next,
	2521	}
	2522	switch (pending.node.next) {
	2523	.none => batch.pending.tail = pending.node.prev,
	2524	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
	2525	pending.node.prev,
	2526	}
	2527	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
	2528	.file_read_streaming => .{
	2529	.file_read_streaming = switch (completion.errno()) {
	2530	.SUCCESS => @as(u32, @bitCast(completion.result)),
	2531	.INTR => 0,
	2532	.CANCELED => break :result null,
	2533	.INVAL => \|err\| errnoBug(err),
	2534	.FAULT => \|err\| errnoBug(err),
	2535	.AGAIN => error.WouldBlock,
	2536	.BADF => \|err\| errnoBug(err), // File descriptor used after closed
	2537	.IO => error.InputOutput,
	2538	.ISDIR => error.IsDir,
	2539	.NOBUFS => error.SystemResources,
	2540	.NOMEM => error.SystemResources,
	2541	.NOTCONN => error.SocketUnconnected,
	2542	.CONNRESET => error.ConnectionResetByPeer,
	2543	else => \|err\| unexpectedErrno(err),
	2544	},
	2545	},
	2546	.file_write_streaming => .{
	2547	.file_write_streaming = switch (completion.errno()) {
	2548	.SUCCESS => @as(u32, @bitCast(completion.result)),
	2549	.INTR => 0,
	2550	.CANCELED => break :result null,
	2551	.INVAL => \|err\| errnoBug(err),
	2552	.FAULT => \|err\| errnoBug(err),
	2553	.AGAIN => error.WouldBlock,
	2554	.BADF => error.NotOpenForWriting, // Can be a race condition.
	2555	.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
	2556	.DQUOT => error.DiskQuota,
	2557	.FBIG => error.FileTooBig,
	2558	.IO => error.InputOutput,
	2559	.NOSPC => error.NoSpaceLeft,
	2560	.PERM => error.PermissionDenied,
	2561	.PIPE => error.BrokenPipe,
	2562	.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
	2563	.BUSY => error.DeviceBusy,
	2564	else => \|err\| unexpectedErrno(err),
	2565	},
	2566	},
	2567	.device_io_control => unreachable,
	2568	})) \|result\| {
	2569	switch (batch.completed.tail) {
	2570	.none => batch.completed.head = index,
	2571	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
	2572	index,
	2573	}
	2574	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2575	batch.completed.tail = index;
	2576	} else {
	2577	switch (batch.unused.tail) {
	2578	.none => batch.unused.head = index,
	2579	else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
	2580	}
	2581	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
	2582	batch.unused.tail = index;
	2583	}
	2584	}
1412	2585	}
1413	2586	}
1414	2587
1415		fn mutexLock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) error{Canceled}!void {
1416		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1417		el.yield(null, .{ .mutex_lock = .{ .prev_state = prev_state, .mutex = mutex } });
	2588	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
	2589	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2590	_ = ev;
	2591	batchDrainReady(batch) catch \|err\| switch (err) {
	2592	error.Timeout => unreachable, // no timeout
	2593	};
	2594	var index = batch.pending.head;
	2595	if (index == .none) return;
	2596	var cancel_region: CancelRegion = .initBlocked();
	2597	defer cancel_region.deinit();
	2598	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	2599	error.Canceled => unreachable, // blocked
	2600	};
	2601	while (index != .none) {
	2602	const pending = &batch.storage[index.toIndex()].pending;
	2603	thread.enqueue().* = .{
	2604	.opcode = .ASYNC_CANCEL,
	2605	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	2606	.ioprio = 0,
	2607	.fd = 0,
	2608	.off = 0,
	2609	.addr = @intFromPtr(&pending.context) \| 0b10,
	2610	.len = 0,
	2611	.rw_flags = 0,
	2612	.user_data = @intFromEnum(Completion.UserData.wakeup),
	2613	.buf_index = 0,
	2614	.personality = 0,
	2615	.splice_fd_in = 0,
	2616	.addr3 = 0,
	2617	.resv = 0,
	2618	};
	2619	index = pending.node.next;
	2620	}
	2621	while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
	2622	error.Timeout => unreachable, // no timeout
	2623	};
1418	2624	}
1419		fn mutexUnlock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) void {
1420		var maybe_waiting_fiber: ?*Fiber = @ptrFromInt(@intFromEnum(prev_state));
1421		while (if (maybe_waiting_fiber) \|waiting_fiber\| @cmpxchgWeak(
1422		Io.Mutex.State,
1423		&mutex.state,
1424		@enumFromInt(@intFromPtr(waiting_fiber)),
1425		@enumFromInt(@intFromPtr(waiting_fiber.queue_next)),
1426		.release,
1427		.acquire,
1428		) else @cmpxchgWeak(
1429		Io.Mutex.State,
1430		&mutex.state,
1431		.locked_once,
1432		.unlocked,
1433		.release,
1434		.acquire,
1435		) orelse return) \|next_state\| maybe_waiting_fiber = @ptrFromInt(@intFromEnum(next_state));
1436		maybe_waiting_fiber.?.queue_next = null;
1437		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1438		el.yield(maybe_waiting_fiber.?, .reschedule);
	2625
	2626	fn dirCreateDir(
	2627	userdata: ?*anyopaque,
	2628	dir: Dir,
	2629	sub_path: []const u8,
	2630	permissions: Dir.Permissions,
	2631	) Dir.CreateDirError!void {
	2632	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2633
	2634	var path_buffer: [PATH_MAX]u8 = undefined;
	2635	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2636
	2637	var cancel_region: CancelRegion = .init();
	2638	defer cancel_region.deinit();
	2639	while (true) {
	2640	const thread = try cancel_region.awaitIoUring();
	2641	thread.enqueue().* = .{
	2642	.opcode = .MKDIRAT,
	2643	.flags = 0,
	2644	.ioprio = 0,
	2645	.fd = dir.handle,
	2646	.off = 0,
	2647	.addr = @intFromPtr(sub_path_posix.ptr),
	2648	.len = permissions.toMode(),
	2649	.rw_flags = 0,
	2650	.user_data = @intFromPtr(cancel_region.fiber),
	2651	.buf_index = 0,
	2652	.personality = 0,
	2653	.splice_fd_in = 0,
	2654	.addr3 = 0,
	2655	.resv = 0,
	2656	};
	2657	ev.yield(null, .nothing);
	2658	switch (cancel_region.errno()) {
	2659	.SUCCESS => return,
	2660	.INTR, .CANCELED => continue,
	2661	.ACCES => return error.AccessDenied,
	2662	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2663	.PERM => return error.PermissionDenied,
	2664	.DQUOT => return error.DiskQuota,
	2665	.EXIST => return error.PathAlreadyExists,
	2666	.FAULT => \|err\| return errnoBug(err),
	2667	.LOOP => return error.SymLinkLoop,
	2668	.MLINK => return error.LinkQuotaExceeded,
	2669	.NAMETOOLONG => return error.NameTooLong,
	2670	.NOENT => return error.FileNotFound,
	2671	.NOMEM => return error.SystemResources,
	2672	.NOSPC => return error.NoSpaceLeft,
	2673	.NOTDIR => return error.NotDir,
	2674	.ROFS => return error.ReadOnlyFileSystem,
	2675	// dragonfly: when dir_fd is unlinked from filesystem
	2676	.NOTCONN => return error.FileNotFound,
	2677	.ILSEQ => return error.BadPathName,
	2678	else => \|err\| return unexpectedErrno(err),
	2679	}
	2680	}
1439	2681	}
1440	2682
1441		const ConditionImpl = struct {
1442		tail: *Fiber,
1443		event: union(enum) {
1444		queued,
1445		wake: Io.Condition.Wake,
1446		},
1447		};
	2683	fn dirCreateDirPath(
	2684	userdata: ?*anyopaque,
	2685	dir: Dir,
	2686	sub_path: []const u8,
	2687	permissions: Dir.Permissions,
	2688	) Dir.CreateDirPathError!Dir.CreatePathStatus {
	2689	const ev: *Evented = @ptrCast(@alignCast(userdata));
1448	2690
1449		fn conditionWait(userdata: ?anyopaque, cond: Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void {
1450		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1451		el.yield(null, .{ .condition_wait = .{ .cond = cond, .mutex = mutex } });
1452		const thread = Thread.current();
1453		const fiber = thread.currentFiber();
1454		const cond_impl = fiber.resultPointer(ConditionImpl);
1455		try mutex.lock(el.io());
1456		switch (cond_impl.event) {
1457		.queued => {},
1458		.wake => \|wake\| if (fiber.queue_next) \|next_fiber\| switch (wake) {
1459		.one => if (@cmpxchgStrong(
1460		?*Fiber,
1461		@as(?Fiber, @ptrCast(&cond.state)),
1462		null,
1463		next_fiber,
1464		.release,
1465		.acquire,
1466		)) \|old_fiber\| {
1467		const old_cond_impl = old_fiber.?.resultPointer(ConditionImpl);
1468		assert(old_cond_impl.tail.queue_next == null);
1469		old_cond_impl.tail.queue_next = next_fiber;
1470		old_cond_impl.tail = cond_impl.tail;
	2691	var it = Dir.path.componentIterator(sub_path);
	2692	var status: Dir.CreatePathStatus = .existed;
	2693	var component = it.last() orelse return error.BadPathName;
	2694	while (true) {
	2695	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
	2696	status = .created;
	2697	} else \|err\| switch (err) {
	2698	error.PathAlreadyExists => {
	2699	// stat the file and return an error if it's not a directory
	2700	// this is important because otherwise a dangling symlink
	2701	// could cause an infinite loop
	2702	const fstat = try dirStatFile(ev, dir, component.path, .{});
	2703	if (fstat.kind != .directory) return error.NotDir;
	2704	},
	2705	error.FileNotFound => \|e\| {
	2706	component = it.previous() orelse return e;
	2707	continue;
1471	2708	},
1472		.all => el.schedule(thread, .{ .head = next_fiber, .tail = cond_impl.tail }),
	2709	else => \|e\| return e,
	2710	}
	2711	component = it.next() orelse return status;
	2712	}
	2713	}
	2714
	2715	fn dirCreateDirPathOpen(
	2716	userdata: ?*anyopaque,
	2717	dir: Dir,
	2718	sub_path: []const u8,
	2719	permissions: Dir.Permissions,
	2720	options: Dir.OpenOptions,
	2721	) Dir.CreateDirPathOpenError!Dir {
	2722	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2723	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
	2724	error.FileNotFound => {
	2725	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
	2726	return dirOpenDir(ev, dir, sub_path, options);
	2727	},
	2728	else => \|e\| return e,
	2729	};
	2730	}
	2731
	2732	fn dirOpenDir(
	2733	userdata: ?*anyopaque,
	2734	dir: Dir,
	2735	sub_path: []const u8,
	2736	options: Dir.OpenOptions,
	2737	) Dir.OpenError!Dir {
	2738	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2739
	2740	var path_buffer: [PATH_MAX]u8 = undefined;
	2741	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2742
	2743	var cancel_region: CancelRegion = .init();
	2744	defer cancel_region.deinit();
	2745	return .{
	2746	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	2747	.ACCMODE = .RDONLY,
	2748	.DIRECTORY = true,
	2749	.NOFOLLOW = !options.follow_symlinks,
	2750	.CLOEXEC = true,
	2751	.PATH = !options.iterate,
	2752	}, 0) catch \|err\| switch (err) {
	2753	error.IsDir => return errnoBug(.ISDIR),
	2754	error.WouldBlock => return errnoBug(.AGAIN),
	2755	error.FileTooBig => return errnoBug(.FBIG),
	2756	error.NoSpaceLeft => return errnoBug(.NOSPC),
	2757	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
	2758	error.FileBusy => return errnoBug(.TXTBSY),
	2759	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
	2760	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2761	error.AntivirusInterference => unreachable, // Windows-only
	2762	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	2763	else => \|e\| return e,
1473	2764	},
	2765	};
	2766	}
	2767
	2768	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
	2769	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2770	var cancel_region: CancelRegion = .init();
	2771	defer cancel_region.deinit();
	2772	return ev.stat(&cancel_region, dir.handle);
	2773	}
	2774
	2775	fn dirStatFile(
	2776	userdata: ?*anyopaque,
	2777	dir: Dir,
	2778	sub_path: []const u8,
	2779	options: Dir.StatFileOptions,
	2780	) Dir.StatFileError!File.Stat {
	2781	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2782	var path_buffer: [PATH_MAX]u8 = undefined;
	2783	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2784	var cancel_region: CancelRegion = .init();
	2785	defer cancel_region.deinit();
	2786	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
	2787	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
	2788	}
	2789
	2790	fn dirAccess(
	2791	userdata: ?*anyopaque,
	2792	dir: Dir,
	2793	sub_path: []const u8,
	2794	options: Dir.AccessOptions,
	2795	) Dir.AccessError!void {
	2796	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2797	_ = ev;
	2798
	2799	var path_buffer: [PATH_MAX]u8 = undefined;
	2800	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2801
	2802	const mode: u32 =
	2803	@as(u32, if (options.read) linux.R_OK else 0) \|
	2804	@as(u32, if (options.write) linux.W_OK else 0) \|
	2805	@as(u32, if (options.execute) linux.X_OK else 0);
	2806	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
	2807
	2808	var cancel_region: CancelRegion = .init();
	2809	defer cancel_region.deinit();
	2810	while (true) {
	2811	try cancel_region.await(.nothing);
	2812	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
	2813	.SUCCESS => return,
	2814	.INTR => continue,
	2815	.ACCES => return error.AccessDenied,
	2816	.PERM => return error.PermissionDenied,
	2817	.ROFS => return error.ReadOnlyFileSystem,
	2818	.LOOP => return error.SymLinkLoop,
	2819	.TXTBSY => return error.FileBusy,
	2820	.NOTDIR => return error.FileNotFound,
	2821	.NOENT => return error.FileNotFound,
	2822	.NAMETOOLONG => return error.NameTooLong,
	2823	.INVAL => \|err\| return errnoBug(err),
	2824	.FAULT => \|err\| return errnoBug(err),
	2825	.IO => return error.InputOutput,
	2826	.NOMEM => return error.SystemResources,
	2827	.ILSEQ => return error.BadPathName,
	2828	else => \|err\| return unexpectedErrno(err),
	2829	}
	2830	}
	2831	}
	2832
	2833	fn dirCreateFile(
	2834	userdata: ?*anyopaque,
	2835	dir: Dir,
	2836	sub_path: []const u8,
	2837	flags: File.CreateFlags,
	2838	) File.OpenError!File {
	2839	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2840
	2841	var path_buffer: [PATH_MAX]u8 = undefined;
	2842	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2843
	2844	var cancel_region: CancelRegion = .init();
	2845	defer cancel_region.deinit();
	2846	const fd = try ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	2847	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
	2848	.CREAT = true,
	2849	.TRUNC = flags.truncate,
	2850	.EXCL = flags.exclusive,
	2851	.CLOEXEC = true,
	2852	}, flags.permissions.toMode());
	2853	errdefer ev.close(fd);
	2854
	2855	switch (flags.lock) {
	2856	.none => {},
	2857	.shared, .exclusive => try ev.flock(
	2858	&cancel_region,
	2859	fd,
	2860	flags.lock,
	2861	if (flags.lock_nonblocking) .nonblocking else .blocking,
	2862	),
1474	2863	}
1475		fiber.queue_next = null;
	2864
	2865	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
1476	2866	}
1477	2867
1478		fn conditionWake(userdata: ?anyopaque, cond: Io.Condition, wake: Io.Condition.Wake) void {
1479		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1480		const waiting_fiber = @atomicRmw(?Fiber, @as(?*Fiber, @ptrCast(&cond.state)), .Xchg, null, .acquire) orelse return;
1481		waiting_fiber.resultPointer(ConditionImpl).event = .{ .wake = wake };
1482		el.yield(waiting_fiber, .reschedule);
	2868	fn dirCreateFileAtomic(
	2869	userdata: ?*anyopaque,
	2870	dir: Dir,
	2871	dest_path: []const u8,
	2872	options: Dir.CreateFileAtomicOptions,
	2873	) Dir.CreateFileAtomicError!File.Atomic {
	2874	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2875	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
	2876	// useless when we have to make up a bogus path name to do the rename()
	2877	// anyway.
	2878	if (!options.replace) tmpfile: {
	2879	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
	2880	.ACCMODE = .RDWR,
	2881	.TMPFILE = true,
	2882	.DIRECTORY = true,
	2883	.CLOEXEC = true,
	2884	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
	2885	.ACCMODE = .RDWR,
	2886	.TMPFILE0 = true,
	2887	.TMPFILE1 = true,
	2888	.DIRECTORY = true,
	2889	.CLOEXEC = true,
	2890	} else break :tmpfile;
	2891
	2892	const dest_dirname = Dir.path.dirname(dest_path);
	2893	if (dest_dirname) \|dirname\| {
	2894	// This has a nice side effect of preemptively triggering EISDIR or
	2895	// ENOENT, avoiding the ambiguity below.
	2896	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
	2897	// None of these make sense in this context.
	2898	error.IsDir,
	2899	error.Streaming,
	2900	error.DiskQuota,
	2901	error.PathAlreadyExists,
	2902	error.LinkQuotaExceeded,
	2903	error.PipeBusy,
	2904	error.FileTooBig,
	2905	error.DeviceBusy,
	2906	error.FileLocksUnsupported,
	2907	error.FileBusy,
	2908	=> return error.Unexpected,
	2909
	2910	else => \|e\| return e,
	2911	};
	2912	}
	2913
	2914	var path_buffer: [PATH_MAX]u8 = undefined;
	2915	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
	2916
	2917	var cancel_region: CancelRegion = .init();
	2918	defer cancel_region.deinit();
	2919	return .{
	2920	.file = .{
	2921	.handle = ev.openat(
	2922	&cancel_region,
	2923	dir.handle,
	2924	sub_path_posix,
	2925	flags,
	2926	options.permissions.toMode(),
	2927	) catch \|err\| switch (err) {
	2928	error.IsDir, error.FileNotFound => {
	2929	// Ambiguous error code. It might mean the file system
	2930	// does not support O_TMPFILE. Therefore, we must fall
	2931	// back to not using O_TMPFILE.
	2932	break :tmpfile;
	2933	},
	2934	error.FileTooBig => return errnoBug(.FBIG),
	2935	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
	2936	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
	2937	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2938	error.AntivirusInterference => unreachable, // Windows-only
	2939	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	2940	else => \|e\| return e,
	2941	},
	2942	.flags = .{ .nonblocking = false },
	2943	},
	2944	.file_basename_hex = 0,
	2945	.dest_sub_path = dest_path,
	2946	.file_open = true,
	2947	.file_exists = false,
	2948	.close_dir_on_deinit = false,
	2949	.dir = dir,
	2950	};
	2951	}
	2952
	2953	if (Dir.path.dirname(dest_path)) \|dirname\| {
	2954	const new_dir = if (options.make_path)
	2955	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
	2956	// None of these make sense in this context.
	2957	error.IsDir,
	2958	error.Streaming,
	2959	error.DiskQuota,
	2960	error.PathAlreadyExists,
	2961	error.LinkQuotaExceeded,
	2962	error.PipeBusy,
	2963	error.FileTooBig,
	2964	error.FileLocksUnsupported,
	2965	error.DeviceBusy,
	2966	=> return error.Unexpected,
	2967
	2968	else => \|e\| return e,
	2969	}
	2970	else
	2971	try dirOpenDir(ev, dir, dirname, .{});
	2972
	2973	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
	2974	}
	2975
	2976	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
1483	2977	}
1484	2978
1485		fn errno(signed: i32) std.os.linux.E {
1486		return .init(@bitCast(@as(isize, signed)));
	2979	fn atomicFileInit(
	2980	ev: *Evented,
	2981	dest_basename: []const u8,
	2982	permissions: File.Permissions,
	2983	dir: Dir,
	2984	close_dir_on_deinit: bool,
	2985	) Dir.CreateFileAtomicError!File.Atomic {
	2986	while (true) {
	2987	var random_integer: u64 = undefined;
	2988	random(ev, @ptrCast(&random_integer));
	2989	const tmp_sub_path = std.fmt.hex(random_integer);
	2990	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
	2991	.permissions = permissions,
	2992	.exclusive = true,
	2993	}) catch \|err\| switch (err) {
	2994	error.PathAlreadyExists => continue,
	2995	error.DeviceBusy => continue,
	2996	error.FileBusy => continue,
	2997
	2998	error.IsDir => return error.Unexpected, // No path components.
	2999	error.FileTooBig => return error.Unexpected, // Creating, not opening.
	3000	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
	3001	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	3002
	3003	else => \|e\| return e,
	3004	};
	3005	return .{
	3006	.file = file,
	3007	.file_basename_hex = random_integer,
	3008	.dest_sub_path = dest_basename,
	3009	.file_open = true,
	3010	.file_exists = true,
	3011	.close_dir_on_deinit = close_dir_on_deinit,
	3012	.dir = dir,
	3013	};
	3014	}
1487	3015	}
1488	3016
1489		fn getSqe(iou: IoUring) std.os.linux.io_uring_sqe {
1490		while (true) return iou.get_sqe() catch {
1491		_ = iou.submit_and_wait(0) catch \|err\| switch (err) {
1492		error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}),
1493		else => \|e\| @panic(@errorName(e)),
	3017	fn dirOpenFile(
	3018	userdata: ?*anyopaque,
	3019	dir: Dir,
	3020	sub_path: []const u8,
	3021	flags: File.OpenFlags,
	3022	) File.OpenError!File {
	3023	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3024
	3025	var path_buffer: [PATH_MAX]u8 = undefined;
	3026	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3027
	3028	var cancel_region: CancelRegion = .init();
	3029	defer cancel_region.deinit();
	3030	const fd = try ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	3031	.ACCMODE = switch (flags.mode) {
	3032	.read_only => .RDONLY,
	3033	.write_only => .WRONLY,
	3034	.read_write => .RDWR,
	3035	},
	3036	.NOCTTY = !flags.allow_ctty,
	3037	.NOFOLLOW = !flags.follow_symlinks,
	3038	.CLOEXEC = true,
	3039	.PATH = flags.path_only,
	3040	}, 0);
	3041	errdefer ev.close(fd);
	3042
	3043	if (!flags.allow_directory) {
	3044	const is_dir = is_dir: {
	3045	const s = ev.stat(&cancel_region, fd) catch \|err\| switch (err) {
	3046	// The directory-ness is either unknown or unknowable
	3047	error.Streaming => break :is_dir false,
	3048	else => \|e\| return e,
	3049	};
	3050	break :is_dir s.kind == .directory;
	3051	};
	3052	if (is_dir) return error.IsDir;
	3053	}
	3054
	3055	switch (flags.lock) {
	3056	.none => {},
	3057	.shared, .exclusive => try ev.flock(
	3058	&cancel_region,
	3059	fd,
	3060	flags.lock,
	3061	if (flags.lock_nonblocking) .nonblocking else .blocking,
	3062	),
	3063	}
	3064
	3065	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
	3066	}
	3067
	3068	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
	3069	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3070	for (dirs) \|dir\| ev.close(dir.handle);
	3071	}
	3072
	3073	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
	3074	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3075	var buffer_index: usize = 0;
	3076	while (buffer.len - buffer_index != 0) {
	3077	if (dr.end - dr.index == 0) {
	3078	// Refill the buffer, unless we've already created references to
	3079	// buffered data.
	3080	if (buffer_index != 0) break;
	3081	var cancel_region: CancelRegion = .init();
	3082	defer cancel_region.deinit();
	3083	if (dr.state == .reset) {
	3084	ev.lseek(&cancel_region, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
	3085	error.Unseekable => return error.Unexpected,
	3086	else => \|e\| return e,
	3087	};
	3088	dr.state = .reading;
	3089	}
	3090	const n = while (true) {
	3091	try cancel_region.await(.nothing);
	3092	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
	3093	switch (linux.errno(rc)) {
	3094	.SUCCESS => break rc,
	3095	.INTR => continue,
	3096	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
	3097	.FAULT => \|err\| return errnoBug(err),
	3098	.NOTDIR => \|err\| return errnoBug(err),
	3099	// To be consistent across platforms, iteration
	3100	// ends if the directory being iterated is deleted
	3101	// during iteration. This matches the behavior of
	3102	// non-Linux, non-WASI UNIX platforms.
	3103	.NOENT => {
	3104	dr.state = .finished;
	3105	return 0;
	3106	},
	3107	// This can occur when reading /proc/$PID/net, or
	3108	// if the provided buffer is too small. Neither
	3109	// scenario is intended to be handled by this API.
	3110	.INVAL => return error.Unexpected,
	3111	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
	3112	else => \|err\| return unexpectedErrno(err),
	3113	}
	3114	};
	3115	if (n == 0) {
	3116	dr.state = .finished;
	3117	return 0;
	3118	}
	3119	dr.index = 0;
	3120	dr.end = n;
	3121	}
	3122	// Linux aligns the header by padding after the null byte of the name
	3123	// to align the next entry. This means we can find the end of the name
	3124	// by looking at only the 8 bytes before the next record. However since
	3125	// file names are usually short it's better to keep the machine code
	3126	// simpler.
	3127	//
	3128	// Furthermore, I observed qemu user mode to not align this struct, so
	3129	// this code makes the conservative choice to not assume alignment.
	3130	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
	3131	const next_index = dr.index + linux_entry.reclen;
	3132	dr.index = next_index;
	3133	const name_ptr: [*]u8 = &linux_entry.name;
	3134	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
	3135	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
	3136	const name = name_ptr[0..name_len :0];
	3137
	3138	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
	3139
	3140	const entry_kind: File.Kind = switch (linux_entry.type) {
	3141	linux.DT.BLK => .block_device,
	3142	linux.DT.CHR => .character_device,
	3143	linux.DT.DIR => .directory,
	3144	linux.DT.FIFO => .named_pipe,
	3145	linux.DT.LNK => .sym_link,
	3146	linux.DT.REG => .file,
	3147	linux.DT.SOCK => .unix_domain_socket,
	3148	else => .unknown,
	3149	};
	3150	buffer[buffer_index] = .{
	3151	.name = name,
	3152	.kind = entry_kind,
	3153	.inode = linux_entry.ino,
1494	3154	};
1495		continue;
	3155	buffer_index += 1;
	3156	}
	3157	return buffer_index;
	3158	}
	3159
	3160	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
	3161	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3162	var cancel_region: CancelRegion = .init();
	3163	defer cancel_region.deinit();
	3164	return ev.realPath(&cancel_region, dir.handle, out_buffer);
	3165	}
	3166
	3167	fn dirRealPathFile(
	3168	userdata: ?*anyopaque,
	3169	dir: Dir,
	3170	sub_path: []const u8,
	3171	out_buffer: []u8,
	3172	) Dir.RealPathFileError!usize {
	3173	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3174
	3175	var path_buffer: [PATH_MAX]u8 = undefined;
	3176	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3177
	3178	var cancel_region: CancelRegion = .init();
	3179	defer cancel_region.deinit();
	3180	const fd = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	3181	.CLOEXEC = true,
	3182	.PATH = true,
	3183	}, 0) catch \|err\| switch (err) {
	3184	error.WouldBlock => return errnoBug(.AGAIN),
	3185	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	3186	else => \|e\| return e,
1496	3187	};
	3188	defer ev.close(fd);
	3189	return ev.realPath(&cancel_region, fd, out_buffer);
	3190	}
	3191
	3192	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
	3193	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3194
	3195	var path_buffer: [PATH_MAX]u8 = undefined;
	3196	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3197
	3198	var cancel_region: CancelRegion = .init();
	3199	defer cancel_region.deinit();
	3200	while (true) {
	3201	const thread = try cancel_region.awaitIoUring();
	3202	thread.enqueue().* = .{
	3203	.opcode = .UNLINKAT,
	3204	.flags = 0,
	3205	.ioprio = 0,
	3206	.fd = dir.handle,
	3207	.off = 0,
	3208	.addr = @intFromPtr(sub_path_posix.ptr),
	3209	.len = 0,
	3210	.rw_flags = 0,
	3211	.user_data = @intFromPtr(cancel_region.fiber),
	3212	.buf_index = 0,
	3213	.personality = 0,
	3214	.splice_fd_in = 0,
	3215	.addr3 = 0,
	3216	.resv = 0,
	3217	};
	3218	ev.yield(null, .nothing);
	3219	switch (cancel_region.errno()) {
	3220	.SUCCESS => return,
	3221	.INTR, .CANCELED => continue,
	3222	.PERM => return error.PermissionDenied,
	3223	.ACCES => return error.AccessDenied,
	3224	.BUSY => return error.FileBusy,
	3225	.FAULT => \|err\| return errnoBug(err),
	3226	.IO => return error.FileSystem,
	3227	.ISDIR => return error.IsDir,
	3228	.LOOP => return error.SymLinkLoop,
	3229	.NAMETOOLONG => return error.NameTooLong,
	3230	.NOENT => return error.FileNotFound,
	3231	.NOTDIR => return error.NotDir,
	3232	.NOMEM => return error.SystemResources,
	3233	.ROFS => return error.ReadOnlyFileSystem,
	3234	.EXIST => \|err\| return errnoBug(err),
	3235	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
	3236	.ILSEQ => return error.BadPathName,
	3237	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	3238	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3239	else => \|err\| return unexpectedErrno(err),
	3240	}
	3241	}
	3242	}
	3243
	3244	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
	3245	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3246
	3247	var path_buffer: [PATH_MAX]u8 = undefined;
	3248	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3249
	3250	var cancel_region: CancelRegion = .init();
	3251	defer cancel_region.deinit();
	3252	while (true) {
	3253	const thread = try cancel_region.awaitIoUring();
	3254	thread.enqueue().* = .{
	3255	.opcode = .UNLINKAT,
	3256	.flags = 0,
	3257	.ioprio = 0,
	3258	.fd = dir.handle,
	3259	.off = 0,
	3260	.addr = @intFromPtr(sub_path_posix.ptr),
	3261	.len = 0,
	3262	.rw_flags = linux.AT.REMOVEDIR,
	3263	.user_data = @intFromPtr(cancel_region.fiber),
	3264	.buf_index = 0,
	3265	.personality = 0,
	3266	.splice_fd_in = 0,
	3267	.addr3 = 0,
	3268	.resv = 0,
	3269	};
	3270	ev.yield(null, .nothing);
	3271	switch (cancel_region.errno()) {
	3272	.SUCCESS => return,
	3273	.INTR, .CANCELED => continue,
	3274	.ACCES => return error.AccessDenied,
	3275	.PERM => return error.PermissionDenied,
	3276	.BUSY => return error.FileBusy,
	3277	.FAULT => \|err\| return errnoBug(err),
	3278	.IO => return error.FileSystem,
	3279	.ISDIR => \|err\| return errnoBug(err),
	3280	.LOOP => return error.SymLinkLoop,
	3281	.NAMETOOLONG => return error.NameTooLong,
	3282	.NOENT => return error.FileNotFound,
	3283	.NOTDIR => return error.NotDir,
	3284	.NOMEM => return error.SystemResources,
	3285	.ROFS => return error.ReadOnlyFileSystem,
	3286	.EXIST => \|err\| return errnoBug(err),
	3287	.NOTEMPTY => return error.DirNotEmpty,
	3288	.ILSEQ => return error.BadPathName,
	3289	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	3290	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3291	else => \|err\| return unexpectedErrno(err),
	3292	}
	3293	}
	3294	}
	3295
	3296	fn dirRename(
	3297	userdata: ?*anyopaque,
	3298	old_dir: Dir,
	3299	old_sub_path: []const u8,
	3300	new_dir: Dir,
	3301	new_sub_path: []const u8,
	3302	) Dir.RenameError!void {
	3303	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3304
	3305	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3306	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3307
	3308	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3309	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3310
	3311	var cancel_region: CancelRegion = .init();
	3312	defer cancel_region.deinit();
	3313	return ev.renameat(
	3314	&cancel_region,
	3315	old_dir.handle,
	3316	old_sub_path_posix,
	3317	new_dir.handle,
	3318	new_sub_path_posix,
	3319	.{},
	3320	);
	3321	}
	3322
	3323	fn dirRenamePreserve(
	3324	userdata: ?*anyopaque,
	3325	old_dir: Dir,
	3326	old_sub_path: []const u8,
	3327	new_dir: Dir,
	3328	new_sub_path: []const u8,
	3329	) Dir.RenamePreserveError!void {
	3330	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3331
	3332	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3333	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3334
	3335	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3336	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3337
	3338	var cancel_region: CancelRegion = .init();
	3339	defer cancel_region.deinit();
	3340	return ev.renameat(
	3341	&cancel_region,
	3342	old_dir.handle,
	3343	old_sub_path_posix,
	3344	new_dir.handle,
	3345	new_sub_path_posix,
	3346	.{ .NOREPLACE = true },
	3347	);
	3348	}
	3349
	3350	fn dirSymLink(
	3351	userdata: ?*anyopaque,
	3352	dir: Dir,
	3353	target_path: []const u8,
	3354	sym_link_path: []const u8,
	3355	flags: Dir.SymLinkFlags,
	3356	) Dir.SymLinkError!void {
	3357	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3358	_ = flags;
	3359
	3360	var target_path_buffer: [PATH_MAX]u8 = undefined;
	3361	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
	3362
	3363	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
	3364	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
	3365
	3366	var cancel_region: CancelRegion = .init();
	3367	defer cancel_region.deinit();
	3368	while (true) {
	3369	const thread = try cancel_region.awaitIoUring();
	3370	thread.enqueue().* = .{
	3371	.opcode = .SYMLINKAT,
	3372	.flags = 0,
	3373	.ioprio = 0,
	3374	.fd = dir.handle,
	3375	.off = @intFromPtr(sym_link_path_posix.ptr),
	3376	.addr = @intFromPtr(target_path_posix.ptr),
	3377	.len = 0,
	3378	.rw_flags = 0,
	3379	.user_data = @intFromPtr(cancel_region.fiber),
	3380	.buf_index = 0,
	3381	.personality = 0,
	3382	.splice_fd_in = 0,
	3383	.addr3 = 0,
	3384	.resv = 0,
	3385	};
	3386	ev.yield(null, .nothing);
	3387	switch (cancel_region.errno()) {
	3388	.SUCCESS => return,
	3389	.INTR, .CANCELED => continue,
	3390	.FAULT => \|err\| return errnoBug(err),
	3391	.INVAL => \|err\| return errnoBug(err),
	3392	.ACCES => return error.AccessDenied,
	3393	.PERM => return error.PermissionDenied,
	3394	.DQUOT => return error.DiskQuota,
	3395	.EXIST => return error.PathAlreadyExists,
	3396	.IO => return error.FileSystem,
	3397	.LOOP => return error.SymLinkLoop,
	3398	.NAMETOOLONG => return error.NameTooLong,
	3399	.NOENT => return error.FileNotFound,
	3400	.NOTDIR => return error.NotDir,
	3401	.NOMEM => return error.SystemResources,
	3402	.NOSPC => return error.NoSpaceLeft,
	3403	.ROFS => return error.ReadOnlyFileSystem,
	3404	.ILSEQ => return error.BadPathName,
	3405	else => \|err\| return unexpectedErrno(err),
	3406	}
	3407	}
	3408	}
	3409
	3410	fn dirReadLink(
	3411	userdata: ?*anyopaque,
	3412	dir: Dir,
	3413	sub_path: []const u8,
	3414	buffer: []u8,
	3415	) Dir.ReadLinkError!usize {
	3416	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3417	_ = ev;
	3418
	3419	var sub_path_buffer: [PATH_MAX]u8 = undefined;
	3420	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
	3421
	3422	var cancel_region: CancelRegion = .init();
	3423	defer cancel_region.deinit();
	3424	while (true) {
	3425	try cancel_region.await(.nothing);
	3426	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
	3427	switch (linux.errno(rc)) {
	3428	.SUCCESS => {
	3429	const len: usize = @bitCast(rc);
	3430	return len;
	3431	},
	3432	.INTR => continue,
	3433	.ACCES => return error.AccessDenied,
	3434	.FAULT => \|err\| return errnoBug(err),
	3435	.INVAL => return error.NotLink,
	3436	.IO => return error.FileSystem,
	3437	.LOOP => return error.SymLinkLoop,
	3438	.NAMETOOLONG => return error.NameTooLong,
	3439	.NOENT => return error.FileNotFound,
	3440	.NOMEM => return error.SystemResources,
	3441	.NOTDIR => return error.NotDir,
	3442	.ILSEQ => return error.BadPathName,
	3443	else => \|err\| return unexpectedErrno(err),
	3444	}
	3445	}
	3446	}
	3447
	3448	fn dirSetOwner(
	3449	userdata: ?*anyopaque,
	3450	dir: Dir,
	3451	owner: ?File.Uid,
	3452	group: ?File.Gid,
	3453	) Dir.SetOwnerError!void {
	3454	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3455	var cancel_region: CancelRegion = .init();
	3456	defer cancel_region.deinit();
	3457	try ev.fchownat(
	3458	&cancel_region,
	3459	dir.handle,
	3460	"",
	3461	owner orelse std.math.maxInt(linux.uid_t),
	3462	group orelse std.math.maxInt(linux.gid_t),
	3463	linux.AT.EMPTY_PATH,
	3464	);
	3465	}
	3466
	3467	fn dirSetFileOwner(
	3468	userdata: ?*anyopaque,
	3469	dir: Dir,
	3470	sub_path: []const u8,
	3471	owner: ?File.Uid,
	3472	group: ?File.Gid,
	3473	options: Dir.SetFileOwnerOptions,
	3474	) Dir.SetFileOwnerError!void {
	3475	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3476	var path_buffer: [PATH_MAX]u8 = undefined;
	3477	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3478	var cancel_region: CancelRegion = .init();
	3479	defer cancel_region.deinit();
	3480	try ev.fchownat(
	3481	&cancel_region,
	3482	dir.handle,
	3483	sub_path_posix,
	3484	owner orelse std.math.maxInt(linux.uid_t),
	3485	group orelse std.math.maxInt(linux.gid_t),
	3486	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3487	);
	3488	}
	3489
	3490	fn dirSetPermissions(
	3491	userdata: ?*anyopaque,
	3492	dir: Dir,
	3493	permissions: Dir.Permissions,
	3494	) Dir.SetPermissionsError!void {
	3495	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3496	var cancel_region: CancelRegion = .init();
	3497	defer cancel_region.deinit();
	3498	ev.fchmodat(
	3499	&cancel_region,
	3500	dir.handle,
	3501	"",
	3502	permissions.toMode(),
	3503	linux.AT.EMPTY_PATH,
	3504	) catch \|err\| switch (err) {
	3505	error.NameTooLong => return errnoBug(.NAMETOOLONG),
	3506	error.BadPathName => return errnoBug(.ILSEQ),
	3507	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
	3508	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
	3509	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
	3510	else => \|e\| return e,
	3511	};
	3512	}
	3513
	3514	fn dirSetFilePermissions(
	3515	userdata: ?*anyopaque,
	3516	dir: Dir,
	3517	sub_path: []const u8,
	3518	permissions: Dir.Permissions,
	3519	options: Dir.SetFilePermissionsOptions,
	3520	) Dir.SetFilePermissionsError!void {
	3521	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3522	var path_buffer: [PATH_MAX]u8 = undefined;
	3523	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3524	var cancel_region: CancelRegion = .init();
	3525	defer cancel_region.deinit();
	3526	try ev.fchmodat(
	3527	&cancel_region,
	3528	dir.handle,
	3529	sub_path_posix,
	3530	permissions.toMode(),
	3531	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3532	);
	3533	}
	3534
	3535	fn dirSetTimestamps(
	3536	userdata: ?*anyopaque,
	3537	dir: Dir,
	3538	sub_path: []const u8,
	3539	options: Dir.SetTimestampsOptions,
	3540	) Dir.SetTimestampsError!void {
	3541	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3542	var path_buffer: [PATH_MAX]u8 = undefined;
	3543	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3544	var cancel_region: CancelRegion = .init();
	3545	defer cancel_region.deinit();
	3546	try ev.utimensat(
	3547	&cancel_region,
	3548	dir.handle,
	3549	sub_path_posix,
	3550	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
	3551	setTimestampToPosix(options.access_timestamp),
	3552	setTimestampToPosix(options.modify_timestamp),
	3553	} else null,
	3554	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3555	);
	3556	}
	3557
	3558	fn dirHardLink(
	3559	userdata: ?*anyopaque,
	3560	old_dir: Dir,
	3561	old_sub_path: []const u8,
	3562	new_dir: Dir,
	3563	new_sub_path: []const u8,
	3564	options: Dir.HardLinkOptions,
	3565	) Dir.HardLinkError!void {
	3566	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3567
	3568	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3569	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3570
	3571	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3572	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3573
	3574	var cancel_region: CancelRegion = .init();
	3575	defer cancel_region.deinit();
	3576	return ev.linkat(
	3577	&cancel_region,
	3578	old_dir.handle,
	3579	old_sub_path_posix,
	3580	new_dir.handle,
	3581	new_sub_path_posix,
	3582	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3583	);
	3584	}
	3585
	3586	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
	3587	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3588	var cancel_region: CancelRegion = .init();
	3589	defer cancel_region.deinit();
	3590	return ev.stat(&cancel_region, file.handle);
	3591	}
	3592
	3593	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
	3594	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3595	var cancel_region: CancelRegion = .init();
	3596	defer cancel_region.deinit();
	3597	while (true) {
	3598	var statx_buf = std.mem.zeroes(linux.Statx);
	3599	const thread = try cancel_region.awaitIoUring();
	3600	thread.enqueue().* = .{
	3601	.opcode = .STATX,
	3602	.flags = 0,
	3603	.ioprio = 0,
	3604	.fd = file.handle,
	3605	.off = @intFromPtr(&statx_buf),
	3606	.addr = @intFromPtr(""),
	3607	.len = @bitCast(linux.STATX{ .SIZE = true }),
	3608	.rw_flags = linux.AT.EMPTY_PATH,
	3609	.user_data = @intFromPtr(cancel_region.fiber),
	3610	.buf_index = 0,
	3611	.personality = 0,
	3612	.splice_fd_in = 0,
	3613	.addr3 = 0,
	3614	.resv = 0,
	3615	};
	3616	ev.yield(null, .nothing);
	3617	switch (cancel_region.errno()) {
	3618	.SUCCESS => {
	3619	if (!statx_buf.mask.SIZE) return error.Unexpected;
	3620	return statx_buf.size;
	3621	},
	3622	.INTR, .CANCELED => continue,
	3623	.ACCES => \|err\| return errnoBug(err),
	3624	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3625	.FAULT => \|err\| return errnoBug(err),
	3626	.INVAL => \|err\| return errnoBug(err),
	3627	.LOOP => \|err\| return errnoBug(err),
	3628	.NAMETOOLONG => \|err\| return errnoBug(err),
	3629	.NOENT => \|err\| return errnoBug(err),
	3630	.NOMEM => return error.SystemResources,
	3631	.NOTDIR => \|err\| return errnoBug(err),
	3632	else => \|err\| return unexpectedErrno(err),
	3633	}
	3634	}
	3635	}
	3636
	3637	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
	3638	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3639	for (files) \|file\| ev.close(file.handle);
	3640	}
	3641
	3642	fn fileWritePositional(
	3643	userdata: ?*anyopaque,
	3644	file: File,
	3645	header: []const u8,
	3646	data: []const []const u8,
	3647	splat: usize,
	3648	offset: u64,
	3649	) File.WritePositionalError!usize {
	3650	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3651
	3652	var iovecs: [max_iovecs_len]iovec_const = undefined;
	3653	var iovlen: iovlen_t = 0;
	3654	addBuf(&iovecs, &iovlen, header);
	3655	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
	3656	const pattern = data[data.len - 1];
	3657	if (iovecs.len - iovlen != 0) switch (splat) {
	3658	0 => {},
	3659	1 => addBuf(&iovecs, &iovlen, pattern),
	3660	else => switch (pattern.len) {
	3661	0 => {},
	3662	1 => {
	3663	var backup_buffer: [splat_buffer_size]u8 = undefined;
	3664	const splat_buffer = &backup_buffer;
	3665	const memset_len = @min(splat_buffer.len, splat);
	3666	const buf = splat_buffer[0..memset_len];
	3667	@memset(buf, pattern[0]);
	3668	addBuf(&iovecs, &iovlen, buf);
	3669	var remaining_splat = splat - buf.len;
	3670	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
	3671	assert(buf.len == splat_buffer.len);
	3672	addBuf(&iovecs, &iovlen, splat_buffer);
	3673	remaining_splat -= splat_buffer.len;
	3674	}
	3675	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	3676	},
	3677	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	3678	addBuf(&iovecs, &iovlen, pattern);
	3679	},
	3680	},
	3681	};
	3682
	3683	var cancel_region: CancelRegion = .init();
	3684	defer cancel_region.deinit();
	3685	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
	3686	}
	3687
	3688	/// This is either usize or u32. Since, either is fine, let's use the same
	3689	/// `addBuf` function for both writing to a file and sending network messages.
	3690	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
	3691
	3692	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
	3693	// OS checks ptr addr before length so zero length vectors must be omitted.
	3694	if (bytes.len == 0) return;
	3695	if (v.len - i.* == 0) return;
	3696	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
	3697	i.* += 1;
	3698	}
	3699
	3700	fn fileWriteFileStreaming(
	3701	userdata: ?*anyopaque,
	3702	file: File,
	3703	header: []const u8,
	3704	file_reader: *File.Reader,
	3705	limit: Io.Limit,
	3706	) File.Writer.WriteFileError!usize {
	3707	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3708	_ = ev;
	3709	_ = file;
	3710	_ = header;
	3711	_ = file_reader;
	3712	_ = limit;
	3713	return error.Unimplemented;
	3714	}
	3715
	3716	fn fileWriteFilePositional(
	3717	userdata: ?*anyopaque,
	3718	file: File,
	3719	header: []const u8,
	3720	file_reader: *File.Reader,
	3721	limit: Io.Limit,
	3722	offset: u64,
	3723	) File.WriteFilePositionalError!usize {
	3724	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3725	_ = ev;
	3726	_ = file;
	3727	_ = header;
	3728	_ = file_reader;
	3729	_ = limit;
	3730	_ = offset;
	3731	return error.Unimplemented;
	3732	}
	3733
	3734	fn fileReadPositional(
	3735	userdata: ?*anyopaque,
	3736	file: File,
	3737	data: []const []u8,
	3738	offset: u64,
	3739	) File.ReadPositionalError!usize {
	3740	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3741
	3742	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
	3743	var i: usize = 0;
	3744	for (data) \|buf\| {
	3745	if (iovecs_buffer.len - i == 0) break;
	3746	if (buf.len != 0) {
	3747	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
	3748	i += 1;
	3749	}
	3750	}
	3751	if (i == 0) return 0;
	3752	const dest = iovecs_buffer[0..i];
	3753	assert(dest[0].len > 0);
	3754
	3755	var cancel_region: CancelRegion = .init();
	3756	defer cancel_region.deinit();
	3757	return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
	3758	error.SocketUnconnected => errnoBug(.NOTCONN), // not a socket
	3759	error.ConnectionResetByPeer => errnoBug(.CONNRESET), // not a socket
	3760	else => \|e\| e,
	3761	};
	3762	}
	3763
	3764	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
	3765	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3766	var cancel_region: CancelRegion = .init();
	3767	defer cancel_region.deinit();
	3768	try ev.lseek(&cancel_region, file.handle, @bitCast(offset), linux.SEEK.CUR);
	3769	}
	3770
	3771	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
	3772	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3773	var cancel_region: CancelRegion = .init();
	3774	defer cancel_region.deinit();
	3775	try ev.lseek(&cancel_region, file.handle, offset, linux.SEEK.SET);
	3776	}
	3777
	3778	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
	3779	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3780	var cancel_region: CancelRegion = .init();
	3781	defer cancel_region.deinit();
	3782	while (true) {
	3783	const thread = try cancel_region.awaitIoUring();
	3784	thread.enqueue().* = .{
	3785	.opcode = .FSYNC,
	3786	.flags = 0,
	3787	.ioprio = 0,
	3788	.fd = file.handle,
	3789	.off = 0,
	3790	.addr = 0,
	3791	.len = 0,
	3792	.rw_flags = 0,
	3793	.user_data = @intFromPtr(cancel_region.fiber),
	3794	.buf_index = 0,
	3795	.personality = 0,
	3796	.splice_fd_in = 0,
	3797	.addr3 = 0,
	3798	.resv = 0,
	3799	};
	3800	ev.yield(null, .nothing);
	3801	switch (cancel_region.errno()) {
	3802	.SUCCESS => return,
	3803	.INTR, .CANCELED => continue,
	3804	.BADF => \|err\| return errnoBug(err),
	3805	.INVAL => \|err\| return errnoBug(err),
	3806	.ROFS => \|err\| return errnoBug(err),
	3807	.IO => return error.InputOutput,
	3808	.NOSPC => return error.NoSpaceLeft,
	3809	.DQUOT => return error.DiskQuota,
	3810	else => \|err\| return unexpectedErrno(err),
	3811	}
	3812	}
	3813	}
	3814
	3815	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
	3816	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3817	_ = ev;
	3818	var cancel_region: CancelRegion = .init();
	3819	defer cancel_region.deinit();
	3820	while (true) {
	3821	try cancel_region.await(.nothing);
	3822	var wsz: winsize = undefined;
	3823	const fd: usize = @bitCast(@as(isize, file.handle));
	3824	const rc = linux.syscall3(.ioctl, fd, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
	3825	switch (linux.errno(rc)) {
	3826	.SUCCESS => return true,
	3827	.INTR => continue,
	3828	else => return false,
	3829	}
	3830	}
	3831	}
	3832
	3833	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
	3834	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3835	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
	3836	}
	3837
	3838	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
	3839	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3840	var cancel_region: CancelRegion = .init();
	3841	defer cancel_region.deinit();
	3842	while (true) {
	3843	const thread = try cancel_region.awaitIoUring();
	3844	thread.enqueue().* = .{
	3845	.opcode = .FTRUNCATE,
	3846	.flags = 0,
	3847	.ioprio = 0,
	3848	.fd = file.handle,
	3849	.off = length,
	3850	.addr = 0,
	3851	.len = 0,
	3852	.rw_flags = 0,
	3853	.user_data = @intFromPtr(cancel_region.fiber),
	3854	.buf_index = 0,
	3855	.personality = 0,
	3856	.splice_fd_in = 0,
	3857	.addr3 = 0,
	3858	.resv = 0,
	3859	};
	3860	ev.yield(null, .nothing);
	3861	switch (cancel_region.errno()) {
	3862	.SUCCESS => return,
	3863	.INTR, .CANCELED => continue,
	3864	.FBIG => return error.FileTooBig,
	3865	.IO => return error.InputOutput,
	3866	.PERM => return error.PermissionDenied,
	3867	.TXTBSY => return error.FileBusy,
	3868	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
	3869	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
	3870	else => \|err\| return unexpectedErrno(err),
	3871	}
	3872	}
	3873	}
	3874
	3875	fn fileSetOwner(
	3876	userdata: ?*anyopaque,
	3877	file: File,
	3878	owner: ?File.Uid,
	3879	group: ?File.Gid,
	3880	) File.SetOwnerError!void {
	3881	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3882	var cancel_region: CancelRegion = .init();
	3883	defer cancel_region.deinit();
	3884	try ev.fchownat(
	3885	&cancel_region,
	3886	file.handle,
	3887	"",
	3888	owner orelse std.math.maxInt(linux.uid_t),
	3889	group orelse std.math.maxInt(linux.gid_t),
	3890	linux.AT.EMPTY_PATH,
	3891	);
	3892	}
	3893
	3894	fn fileSetPermissions(
	3895	userdata: ?*anyopaque,
	3896	file: File,
	3897	permissions: File.Permissions,
	3898	) File.SetPermissionsError!void {
	3899	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3900	var cancel_region: CancelRegion = .init();
	3901	defer cancel_region.deinit();
	3902	ev.fchmodat(
	3903	&cancel_region,
	3904	file.handle,
	3905	"",
	3906	permissions.toMode(),
	3907	linux.AT.EMPTY_PATH,
	3908	) catch \|err\| switch (err) {
	3909	error.NameTooLong => return errnoBug(.NAMETOOLONG),
	3910	error.BadPathName => return errnoBug(.ILSEQ),
	3911	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
	3912	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
	3913	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
	3914	else => \|e\| return e,
	3915	};
	3916	}
	3917
	3918	fn fileSetTimestamps(
	3919	userdata: ?*anyopaque,
	3920	file: File,
	3921	options: File.SetTimestampsOptions,
	3922	) File.SetTimestampsError!void {
	3923	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3924	var cancel_region: CancelRegion = .init();
	3925	defer cancel_region.deinit();
	3926	try ev.utimensat(
	3927	&cancel_region,
	3928	file.handle,
	3929	"",
	3930	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
	3931	setTimestampToPosix(options.access_timestamp),
	3932	setTimestampToPosix(options.modify_timestamp),
	3933	} else null,
	3934	linux.AT.EMPTY_PATH,
	3935	);
	3936	}
	3937
	3938	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
	3939	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3940	var cancel_region: CancelRegion = .init();
	3941	defer cancel_region.deinit();
	3942	ev.flock(&cancel_region, file.handle, lock, .blocking) catch \|err\| switch (err) {
	3943	error.WouldBlock => unreachable, // blocking
	3944	else => \|e\| return e,
	3945	};
	3946	}
	3947
	3948	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
	3949	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3950	var cancel_region: CancelRegion = .init();
	3951	defer cancel_region.deinit();
	3952	ev.flock(&cancel_region, file.handle, lock, switch (lock) {
	3953	.none => .blocking,
	3954	.shared, .exclusive => .nonblocking,
	3955	}) catch \|err\| switch (err) {
	3956	error.WouldBlock => return false,
	3957	else => \|e\| return e,
	3958	};
	3959	return true;
	3960	}
	3961
	3962	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
	3963	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3964	var cancel_region: CancelRegion = .initBlocked();
	3965	defer cancel_region.deinit();
	3966	ev.flock(&cancel_region, file.handle, .none, .blocking) catch \|err\| switch (err) {
	3967	error.Canceled => unreachable, // blocked
	3968	error.WouldBlock => unreachable, // blocking
	3969	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
	3970	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
	3971	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
	3972	};
	3973	}
	3974
	3975	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
	3976	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3977	var cancel_region: CancelRegion = .init();
	3978	defer cancel_region.deinit();
	3979	ev.flock(&cancel_region, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
	3980	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
	3981	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
	3982	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
	3983	else => \|e\| return e,
	3984	};
	3985	}
	3986
	3987	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
	3988	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3989	var cancel_region: CancelRegion = .init();
	3990	defer cancel_region.deinit();
	3991	return ev.realPath(&cancel_region, file.handle, out_buffer);
	3992	}
	3993
	3994	fn fileHardLink(
	3995	userdata: ?*anyopaque,
	3996	file: File,
	3997	new_dir: Dir,
	3998	new_sub_path: []const u8,
	3999	options: File.HardLinkOptions,
	4000	) File.HardLinkError!void {
	4001	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4002
	4003	var new_path_buffer: [PATH_MAX]u8 = undefined;
	4004	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	4005
	4006	var cancel_region: CancelRegion = .init();
	4007	defer cancel_region.deinit();
	4008	return ev.linkat(
	4009	&cancel_region,
	4010	file.handle,
	4011	"",
	4012	new_dir.handle,
	4013	new_sub_path_posix,
	4014	linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
	4015	);
	4016	}
	4017
	4018	fn fileMemoryMapCreate(
	4019	userdata: ?*anyopaque,
	4020	file: File,
	4021	options: File.MemoryMap.CreateOptions,
	4022	) File.MemoryMap.CreateError!File.MemoryMap {
	4023	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4024	_ = ev;
	4025	const prot: linux.PROT = .{
	4026	.READ = options.protection.read,
	4027	.WRITE = options.protection.write,
	4028	.EXEC = options.protection.execute,
	4029	};
	4030	const flags: linux.MAP = .{
	4031	.TYPE = .SHARED_VALIDATE,
	4032	.POPULATE = options.populate,
	4033	};
	4034
	4035	const page_align = std.heap.page_size_min;
	4036
	4037	var cancel_region: CancelRegion = .init();
	4038	defer cancel_region.deinit();
	4039	const contents = while (true) {
	4040	try cancel_region.await(.nothing);
	4041	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
	4042	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
	4043	switch (linux.errno(rc)) {
	4044	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
	4045	.INTR => continue,
	4046	.ACCES => return error.AccessDenied,
	4047	.AGAIN => return error.LockedMemoryLimitExceeded,
	4048	.MFILE => return error.ProcessFdQuotaExceeded,
	4049	.NFILE => return error.SystemFdQuotaExceeded,
	4050	.NOMEM => return error.OutOfMemory,
	4051	.PERM => return error.PermissionDenied,
	4052	.OVERFLOW => return error.Unseekable,
	4053	.BADF => \|err\| return errnoBug(err), // Always a race condition.
	4054	.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
	4055	.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
	4056	else => \|err\| return unexpectedErrno(err),
	4057	}
	4058	};
	4059	return .{
	4060	.file = file,
	4061	.offset = options.offset,
	4062	.memory = contents,
	4063	.section = {},
	4064	};
	4065	}
	4066
	4067	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
	4068	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4069	_ = ev;
	4070	const memory = mm.memory;
	4071	if (memory.len == 0) return;
	4072	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
	4073	.SUCCESS => {},
	4074	else => \|err\| if (builtin.mode == .Debug)
	4075	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
	4076	}
	4077	mm.* = undefined;
	4078	}
	4079
	4080	fn processExecutableOpen(
	4081	userdata: ?*anyopaque,
	4082	flags: File.OpenFlags,
	4083	) process.OpenExecutableError!File {
	4084	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4085	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
	4086	}
	4087
	4088	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
	4089	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4090	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
	4091	error.UnsupportedReparsePointType => unreachable, // Windows-only
	4092	error.NetworkNotFound => unreachable, // Windows-only
	4093	error.FileBusy => unreachable, // Windows-only
	4094	else => \|e\| return e,
	4095	};
	4096	}
	4097
	4098	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4099	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4100	const ev_io = ev.io();
	4101	ev.stderr_mutex.lockUncancelable(ev_io);
	4102	errdefer ev.stderr_mutex.unlock(ev_io);
	4103	return ev.initLockedStderr(terminal_mode);
	4104	}
	4105
	4106	fn tryLockStderr(
	4107	userdata: ?*anyopaque,
	4108	terminal_mode: ?Io.Terminal.Mode,
	4109	) Io.Cancelable!?Io.LockedStderr {
	4110	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4111	const ev_io = ev.io();
	4112	if (!ev.stderr_mutex.tryLock()) return null;
	4113	errdefer ev.stderr_mutex.unlock(ev_io);
	4114	return try ev.initLockedStderr(terminal_mode);
	4115	}
	4116
	4117	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4118	if (!ev.stderr_writer_initialized) {
	4119	const ev_io = ev.io();
	4120	try ev.scanEnviron();
	4121	const NO_COLOR = ev.environ.exist.NO_COLOR;
	4122	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
	4123	ev.stderr_mode = terminal_mode orelse
	4124	try .detect(ev_io, ev.stderr_writer.file, NO_COLOR, CLICOLOR_FORCE);
	4125	ev.stderr_writer_initialized = true;
	4126	}
	4127	return .{
	4128	.file_writer = &ev.stderr_writer,
	4129	.terminal_mode = terminal_mode orelse ev.stderr_mode,
	4130	};
	4131	}
	4132
	4133	fn unlockStderr(userdata: ?*anyopaque) void {
	4134	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4135	ev.stderr_writer.interface.flush() catch \|err\| switch (err) {
	4136	error.WriteFailed => switch (ev.stderr_writer.err.?) {
	4137	error.Canceled => recancel(ev),
	4138	else => {},
	4139	},
	4140	};
	4141	ev.stderr_writer.interface.end = 0;
	4142	ev.stderr_writer.interface.buffer = &.{};
	4143	ev.stderr_mutex.unlock(ev.io());
	4144	}
	4145
	4146	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
	4147	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4148	_ = ev;
	4149	var cancel_region: CancelRegion = .init();
	4150	defer cancel_region.deinit();
	4151	while (true) {
	4152	try cancel_region.await(.nothing);
	4153	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
	4154	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
	4155	.INTR => continue,
	4156	.NOENT => return error.CurrentDirUnlinked,
	4157	.RANGE => return error.NameTooLong,
	4158	.FAULT => \|err\| return errnoBug(err),
	4159	.INVAL => \|err\| return errnoBug(err),
	4160	else => \|err\| return unexpectedErrno(err),
	4161	}
	4162	}
	4163	}
	4164
	4165	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
	4166	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4167	_ = ev;
	4168	if (dir.handle == linux.AT.FDCWD) return;
	4169	var cancel_region: CancelRegion = .init();
	4170	defer cancel_region.deinit();
	4171	while (true) {
	4172	try cancel_region.await(.nothing);
	4173	switch (linux.errno(linux.fchdir(dir.handle))) {
	4174	.SUCCESS => return,
	4175	.INTR => continue,
	4176	.ACCES => return error.AccessDenied,
	4177	.NOTDIR => return error.NotDir,
	4178	.IO => return error.FileSystem,
	4179	.BADF => \|err\| return errnoBug(err),
	4180	else => \|err\| return unexpectedErrno(err),
	4181	}
	4182	}
	4183	}
	4184
	4185	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
	4186	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4187	_ = ev;
	4188	var path_buffer: [PATH_MAX]u8 = undefined;
	4189	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
	4190	var cancel_region: CancelRegion = .init();
	4191	defer cancel_region.deinit();
	4192	while (true) {
	4193	try cancel_region.await(.nothing);
	4194	switch (linux.errno(linux.chdir(dir_path_posix))) {
	4195	.SUCCESS => return,
	4196	.INTR => continue,
	4197	.ACCES => return error.AccessDenied,
	4198	.IO => return error.FileSystem,
	4199	.LOOP => return error.SymLinkLoop,
	4200	.NAMETOOLONG => return error.NameTooLong,
	4201	.NOENT => return error.FileNotFound,
	4202	.NOMEM => return error.SystemResources,
	4203	.NOTDIR => return error.NotDir,
	4204	.ILSEQ => return error.BadPathName,
	4205	.FAULT => \|err\| return errnoBug(err),
	4206	else => \|err\| return unexpectedErrno(err),
	4207	}
	4208	}
	4209	}
	4210
	4211	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
	4212	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4213
	4214	try ev.scanEnviron(); // for PATH
	4215	const PATH = ev.environ.string.PATH orelse default_PATH;
	4216
	4217	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4218	defer arena_allocator.deinit();
	4219	const arena = arena_allocator.allocator();
	4220
	4221	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4222	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4223
	4224	const env_block = env_block: {
	4225	const prog_fd: i32 = -1;
	4226	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4227	.zig_progress_fd = prog_fd,
	4228	});
	4229	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4230	.zig_progress_fd = prog_fd,
	4231	});
	4232	};
	4233
	4234	return execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
	4235	}
	4236
	4237	fn processReplacePath(
	4238	userdata: ?*anyopaque,
	4239	dir: Dir,
	4240	options: process.ReplaceOptions,
	4241	) process.ReplaceError {
	4242	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4243	_ = ev;
	4244	_ = dir;
	4245	_ = options;
	4246	@panic("TODO processReplacePath");
	4247	}
	4248
	4249	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
	4250	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4251	const spawned = try ev.spawn(options);
	4252	defer ev.close(spawned.err_fd);
	4253
	4254	// Wait for the child to report any errors in or before `execvpe`.
	4255	var child_err: ForkBailError = undefined;
	4256	var cancel_region: CancelRegion = .initBlocked();
	4257	defer cancel_region.deinit();
	4258	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
	4259	switch (read_err) {
	4260	error.Canceled => unreachable, // blocked
	4261	error.EndOfStream => {
	4262	// Write end closed by CLOEXEC at the time of the `execvpe` call,
	4263	// indicating success.
	4264	},
	4265	else => {
	4266	// Problem reading the error from the error reporting pipe. We
	4267	// don't know if the child is alive or dead. Better to assume it is
	4268	// alive so the resource does not risk being leaked.
	4269	},
	4270	}
	4271	return .{
	4272	.id = spawned.pid,
	4273	.thread_handle = {},
	4274	.stdin = spawned.stdin,
	4275	.stdout = spawned.stdout,
	4276	.stderr = spawned.stderr,
	4277	.request_resource_usage_statistics = options.request_resource_usage_statistics,
	4278	};
	4279	};
	4280	return child_err;
	4281	}
	4282
	4283	fn processSpawnPath(
	4284	userdata: ?*anyopaque,
	4285	dir: Dir,
	4286	options: process.SpawnOptions,
	4287	) process.SpawnError!process.Child {
	4288	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4289	_ = ev;
	4290	_ = dir;
	4291	_ = options;
	4292	@panic("TODO processSpawnPath");
	4293	}
	4294
	4295	const Spawned = struct {
	4296	pid: pid_t,
	4297	err_fd: fd_t,
	4298	stdin: ?File,
	4299	stdout: ?File,
	4300	stderr: ?File,
	4301	};
	4302	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
	4303	// The child process does need to access (one end of) these pipes. However,
	4304	// we must initially set CLOEXEC to avoid a race condition. If another thread
	4305	// is racing to spawn a different child process, we don't want it to inherit
	4306	// these FDs in any scenario; that would mean that, for instance, calls to
	4307	// `poll` from the parent would not report the child's stdout as closing when
	4308	// expected, since the other child may retain a reference to the write end of
	4309	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
	4310	// need to do something in the new child to make sure we preserve the reference
	4311	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
	4312	// turns out, we `dup2` everything anyway, so there's no need!
	4313	const pipe_flags: linux.O = .{ .CLOEXEC = true };
	4314
	4315	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
	4316	errdefer if (options.stdin == .pipe) {
	4317	ev.destroyPipe(stdin_pipe);
	4318	};
	4319
	4320	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
	4321	errdefer if (options.stdout == .pipe) {
	4322	ev.destroyPipe(stdout_pipe);
	4323	};
	4324
	4325	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
	4326	errdefer if (options.stderr == .pipe) {
	4327	ev.destroyPipe(stderr_pipe);
	4328	};
	4329
	4330	const any_ignore =
	4331	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
	4332	const dev_null_fd = if (any_ignore) dev_null_fd: {
	4333	var cancel_region: CancelRegion = .init();
	4334	defer cancel_region.deinit();
	4335	break :dev_null_fd try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
	4336	.ACCMODE = .RDWR,
	4337	});
	4338	} else undefined;
	4339
	4340	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
	4341	// We use CLOEXEC for the same reason as in `pipe_flags`.
	4342	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
	4343	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
	4344	break :pipe pipe;
	4345	} else .{ -1, -1 };
	4346	errdefer ev.destroyPipe(prog_pipe);
	4347
	4348	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4349	defer arena_allocator.deinit();
	4350	const arena = arena_allocator.allocator();
	4351
	4352	// The POSIX standard does not allow malloc() between fork() and execve(),
	4353	// and this allocator may be a libc allocator.
	4354	// I have personally observed the child process deadlocking when it tries
	4355	// to call malloc() due to a heap allocation between fork() and execve(),
	4356	// in musl v1.1.24.
	4357	// Additionally, we want to reduce the number of possible ways things
	4358	// can fail between fork() and execve().
	4359	// Therefore, we do all the allocation for the execve() before the fork().
	4360	// This means we must do the null-termination of argv and env vars here.
	4361	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4362	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4363
	4364	const prog_fileno = 3;
	4365	comptime assert(@max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO) + 1 == prog_fileno);
	4366
	4367	const env_block = env_block: {
	4368	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
	4369	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4370	.zig_progress_fd = prog_fd,
	4371	});
	4372	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4373	.zig_progress_fd = prog_fd,
	4374	});
	4375	};
	4376
	4377	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
	4378	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
	4379	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
	4380	errdefer ev.destroyPipe(err_pipe);
	4381
	4382	try ev.scanEnviron(); // for PATH
	4383	const PATH = ev.environ.string.PATH orelse default_PATH;
	4384
	4385	const pid_result: pid_t = fork: {
	4386	const rc = linux.fork();
	4387	switch (linux.errno(rc)) {
	4388	.SUCCESS => break :fork @intCast(rc),
	4389	.AGAIN => return error.SystemResources,
	4390	.NOMEM => return error.SystemResources,
	4391	.NOSYS => return error.OperationUnsupported,
	4392	else => \|err\| return unexpectedErrno(err),
	4393	}
	4394	};
	4395
	4396	if (pid_result == 0) {
	4397	defer comptime unreachable; // We are the child.
	4398	_ = swapCancelProtection(ev, .blocked);
	4399	const ep1 = err_pipe[1];
	4400
	4401	ev.setUpChildIo(options.stdin, stdin_pipe[0], linux.STDIN_FILENO, dev_null_fd) catch \|err\|
	4402	ev.forkBail(ep1, err);
	4403	ev.setUpChildIo(options.stdout, stdout_pipe[1], linux.STDOUT_FILENO, dev_null_fd) catch \|err\|
	4404	ev.forkBail(ep1, err);
	4405	ev.setUpChildIo(options.stderr, stderr_pipe[1], linux.STDERR_FILENO, dev_null_fd) catch \|err\|
	4406	ev.forkBail(ep1, err);
	4407
	4408	switch (options.cwd) {
	4409	.inherit => {},
	4410	.dir => \|cwd\| processSetCurrentDir(ev, cwd) catch \|err\| ev.forkBail(ep1, err),
	4411	.path => \|cwd\| processSetCurrentPath(ev, cwd) catch \|err\| ev.forkBail(ep1, err),
	4412	}
	4413
	4414	// Must happen after fchdir above, the cwd file descriptor might be
	4415	// equal to prog_fileno and be clobbered by this dup2 call.
	4416	if (prog_pipe[1] != -1) dup2(prog_pipe[1], prog_fileno) catch \|err\| ev.forkBail(ep1, err);
	4417
	4418	if (options.gid) \|gid\| {
	4419	switch (linux.errno(linux.setregid(gid, gid))) {
	4420	.SUCCESS => {},
	4421	.AGAIN => ev.forkBail(ep1, error.ResourceLimitReached),
	4422	.INVAL => ev.forkBail(ep1, error.InvalidUserId),
	4423	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4424	else => ev.forkBail(ep1, error.Unexpected),
	4425	}
	4426	}
	4427
	4428	if (options.uid) \|uid\| {
	4429	switch (linux.errno(linux.setreuid(uid, uid))) {
	4430	.SUCCESS => {},
	4431	.AGAIN => ev.forkBail(ep1, error.ResourceLimitReached),
	4432	.INVAL => ev.forkBail(ep1, error.InvalidUserId),
	4433	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4434	else => ev.forkBail(ep1, error.Unexpected),
	4435	}
	4436	}
	4437
	4438	if (options.pgid) \|pid\| {
	4439	switch (linux.errno(linux.setpgid(0, pid))) {
	4440	.SUCCESS => {},
	4441	.ACCES => ev.forkBail(ep1, error.ProcessAlreadyExec),
	4442	.INVAL => ev.forkBail(ep1, error.InvalidProcessGroupId),
	4443	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4444	else => ev.forkBail(ep1, error.Unexpected),
	4445	}
	4446	}
	4447
	4448	if (options.start_suspended) {
	4449	switch (linux.errno(linux.kill(linux.getpid(), .STOP))) {
	4450	.SUCCESS => {},
	4451	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4452	else => ev.forkBail(ep1, error.Unexpected),
	4453	}
	4454	}
	4455
	4456	const err = execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
	4457	ev.forkBail(ep1, err);
	4458	}
	4459
	4460	const pid: pid_t = @intCast(pid_result); // We are the parent.
	4461	errdefer comptime unreachable; // The child is forked; we must not error from now on
	4462
	4463	ev.close(err_pipe[1]); // make sure only the child holds the write end open
	4464
	4465	if (options.stdin == .pipe) ev.close(stdin_pipe[0]);
	4466	if (options.stdout == .pipe) ev.close(stdout_pipe[1]);
	4467	if (options.stderr == .pipe) ev.close(stderr_pipe[1]);
	4468
	4469	if (prog_pipe[1] != -1) ev.close(prog_pipe[1]);
	4470
	4471	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
	4472
	4473	return .{
	4474	.pid = pid,
	4475	.err_fd = err_pipe[0],
	4476	.stdin = switch (options.stdin) {
	4477	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
	4478	else => null,
	4479	},
	4480	.stdout = switch (options.stdout) {
	4481	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
	4482	else => null,
	4483	},
	4484	.stderr = switch (options.stderr) {
	4485	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
	4486	else => null,
	4487	},
	4488	};
	4489	}
	4490
	4491	pub const PipeError = error{
	4492	SystemFdQuotaExceeded,
	4493	ProcessFdQuotaExceeded,
	4494	} \|\| Io.UnexpectedError;
	4495	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
	4496	var fds: [2]fd_t = undefined;
	4497	switch (linux.errno(linux.pipe2(&fds, flags))) {
	4498	.SUCCESS => return fds,
	4499	.INVAL => \|err\| return errnoBug(err), // Invalid flags
	4500	.NFILE => return error.SystemFdQuotaExceeded,
	4501	.MFILE => return error.ProcessFdQuotaExceeded,
	4502	else => \|err\| return unexpectedErrno(err),
	4503	}
	4504	}
	4505	fn destroyPipe(ev: *Evented, pipe: [2]fd_t) void {
	4506	if (pipe[0] != -1) ev.close(pipe[0]);
	4507	if (pipe[0] != pipe[1]) ev.close(pipe[1]);
	4508	}
	4509
	4510	fn setUpChildIo(
	4511	ev: *Evented,
	4512	stdio: process.SpawnOptions.StdIo,
	4513	pipe_fd: fd_t,
	4514	std_fileno: i32,
	4515	dev_null_fd: fd_t,
	4516	) !void {
	4517	switch (stdio) {
	4518	.pipe => try dup2(pipe_fd, std_fileno),
	4519	.close => ev.close(std_fileno),
	4520	.inherit => {},
	4521	.ignore => try dup2(dev_null_fd, std_fileno),
	4522	.file => \|file\| try dup2(file.handle, std_fileno),
	4523	}
	4524	}
	4525
	4526	pub const DupError = error{
	4527	ProcessFdQuotaExceeded,
	4528	SystemResources,
	4529	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
	4530	pub fn dup2(old_fd: fd_t, new_fd: fd_t) DupError!void {
	4531	var cancel_region: CancelRegion = .init();
	4532	defer cancel_region.deinit();
	4533	while (true) {
	4534	try cancel_region.await(.nothing);
	4535	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
	4536	.SUCCESS => {},
	4537	.BUSY, .INTR => continue,
	4538	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	4539	.BADF => \|err\| return errnoBug(err), // use after free
	4540	.MFILE => return error.ProcessFdQuotaExceeded,
	4541	.NOMEM => return error.SystemResources,
	4542	else => \|err\| return unexpectedErrno(err),
	4543	}
	4544	}
	4545	}
	4546
	4547	/// Errors that can occur between fork() and execv()
	4548	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
	4549	process.SpawnError \|\| process.ReplaceError;
	4550	/// Child of fork calls this to report an error to the fork parent. Then the
	4551	/// child exits.
	4552	fn forkBail(ev: *Evented, fd: fd_t, err: ForkBailError) noreturn {
	4553	var cancel_region: CancelRegion = .initBlocked();
	4554	defer cancel_region.deinit();
	4555	ev.writeAll(&cancel_region, fd, @ptrCast(&err)) catch {};
	4556	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
	4557	exit(1);
	4558	}
	4559
	4560	fn execv(
	4561	arg0_expand: process.ArgExpansion,
	4562	file: [*:0]const u8,
	4563	child_argv: [:null]?[:0]const u8,
	4564	env_block: process.Environ.PosixBlock,
	4565	PATH: []const u8,
	4566	) process.ReplaceError {
	4567	const file_slice = std.mem.sliceTo(file, 0);
	4568	if (std.mem.findScalar(u8, file_slice, '/') != null) return execvPath(file, child_argv, env_block);
	4569
	4570	// Use of PATH_MAX here is valid as the path_buf will be passed
	4571	// directly to the operating system in posixExecvPath.
	4572	var path_buf: [PATH_MAX]u8 = undefined;
	4573	var it = std.mem.tokenizeScalar(u8, PATH, ':');
	4574	var seen_eacces = false;
	4575	var err: process.ReplaceError = error.FileNotFound;
	4576
	4577	// In case of expanding arg0 we must put it back if we return with an error.
	4578	const prev_arg0 = child_argv[0];
	4579	defer switch (arg0_expand) {
	4580	.expand => child_argv[0] = prev_arg0,
	4581	.no_expand => {},
	4582	};
	4583
	4584	while (it.next()) \|search_path\| {
	4585	const path_len = search_path.len + file_slice.len + 1;
	4586	if (path_buf.len < path_len + 1) return error.NameTooLong;
	4587	@memcpy(path_buf[0..search_path.len], search_path);
	4588	path_buf[search_path.len] = '/';
	4589	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
	4590	path_buf[path_len] = 0;
	4591	const full_path = path_buf[0..path_len :0].ptr;
	4592	switch (arg0_expand) {
	4593	.expand => child_argv[0] = full_path,
	4594	.no_expand => {},
	4595	}
	4596	err = execvPath(full_path, child_argv, env_block);
	4597	switch (err) {
	4598	error.AccessDenied => seen_eacces = true,
	4599	error.FileNotFound, error.NotDir => {},
	4600	else => \|e\| return e,
	4601	}
	4602	}
	4603	if (seen_eacces) return error.AccessDenied;
	4604	return err;
	4605	}
	4606	/// This function ignores PATH environment variable.
	4607	pub fn execvPath(
	4608	path: [*:0]const u8,
	4609	child_argv: [:null]const ?[:0]const u8,
	4610	env_block: process.Environ.PosixBlock,
	4611	) process.ReplaceError {
	4612	var cancel_region: CancelRegion = .init();
	4613	defer cancel_region.deinit();
	4614	try cancel_region.await(.nothing);
	4615	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
	4616	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
	4617	.@"2BIG" => return error.SystemResources,
	4618	.MFILE => return error.ProcessFdQuotaExceeded,
	4619	.NAMETOOLONG => return error.NameTooLong,
	4620	.NFILE => return error.SystemFdQuotaExceeded,
	4621	.NOMEM => return error.SystemResources,
	4622	.ACCES => return error.AccessDenied,
	4623	.PERM => return error.PermissionDenied,
	4624	.INVAL => return error.InvalidExe,
	4625	.NOEXEC => return error.InvalidExe,
	4626	.IO => return error.FileSystem,
	4627	.LOOP => return error.FileSystem,
	4628	.ISDIR => return error.IsDir,
	4629	.NOENT => return error.FileNotFound,
	4630	.NOTDIR => return error.NotDir,
	4631	.TXTBSY => return error.FileBusy,
	4632	.LIBBAD => return error.InvalidExe,
	4633	else => \|err\| return unexpectedErrno(err),
	4634	}
	4635	}
	4636
	4637	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
	4638	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4639	defer ev.childCleanup(child);
	4640
	4641	const pid = child.id.?;
	4642	var info: linux.siginfo_t = undefined;
	4643	var cancel_region: CancelRegion = .init();
	4644	defer cancel_region.deinit();
	4645	while (true) {
	4646	const thread = try cancel_region.awaitIoUring();
	4647	thread.enqueue().* = .{
	4648	.opcode = .WAITID,
	4649	.flags = 0,
	4650	.ioprio = 0,
	4651	.fd = pid,
	4652	.off = @intFromPtr(&info),
	4653	.addr = 0,
	4654	.len = @intFromEnum(linux.P.PID),
	4655	.rw_flags = 0,
	4656	.user_data = @intFromPtr(cancel_region.fiber),
	4657	.buf_index = 0,
	4658	.personality = 0,
	4659	.splice_fd_in = linux.W.EXITED \|
	4660	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
	4661	.addr3 = 0,
	4662	.resv = 0,
	4663	};
	4664	ev.yield(null, .nothing);
	4665	switch (cancel_region.errno()) {
	4666	.SUCCESS => {
	4667	if (child.request_resource_usage_statistics) while (true) {
	4668	try cancel_region.await(.nothing);
	4669	var rusage: linux.rusage = undefined;
	4670	switch (linux.errno(linux.waitid(
	4671	.PID,
	4672	pid,
	4673	&info,
	4674	linux.W.EXITED \| linux.W.NOHANG,
	4675	&rusage,
	4676	))) {
	4677	.SUCCESS => {
	4678	child.resource_usage_statistics.rusage = rusage;
	4679	break;
	4680	},
	4681	.INTR, .CANCELED => continue,
	4682	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4683	else => \|err\| return unexpectedErrno(err),
	4684	}
	4685	};
	4686	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
	4687	const code: linux.CLD = @enumFromInt(info.code);
	4688	return switch (code) {
	4689	.EXITED => .{ .exited = @truncate(status) },
	4690	.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
	4691	.TRAPPED, .STOPPED => .{ .stopped = status },
	4692	_, .CONTINUED => .{ .unknown = status },
	4693	};
	4694	},
	4695	.INTR, .CANCELED => continue,
	4696	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4697	else => \|err\| return unexpectedErrno(err),
	4698	}
	4699	}
	4700	}
	4701
	4702	fn childKill(userdata: ?anyopaque, child: process.Child) void {
	4703	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4704	defer ev.childCleanup(child);
	4705
	4706	const pid = child.id.?;
	4707	var cancel_region: CancelRegion = .initBlocked();
	4708	defer cancel_region.deinit();
	4709	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
	4710	.SUCCESS => break,
	4711	.INTR => continue,
	4712	.PERM => return,
	4713	.INVAL => \|err\| return errnoBug(err) catch {},
	4714	.SRCH => \|err\| return errnoBug(err) catch {},
	4715	else => \|err\| return unexpectedErrno(err) catch {},
	4716	};
	4717
	4718	var info: linux.siginfo_t = undefined;
	4719	while (true) {
	4720	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	4721	error.Canceled => unreachable, // blocked
	4722	};
	4723	thread.enqueue().* = .{
	4724	.opcode = .WAITID,
	4725	.flags = 0,
	4726	.ioprio = 0,
	4727	.fd = pid,
	4728	.off = @intFromPtr(&info),
	4729	.addr = 0,
	4730	.len = @intFromEnum(linux.P.PID),
	4731	.rw_flags = 0,
	4732	.user_data = @intFromPtr(cancel_region.fiber),
	4733	.buf_index = 0,
	4734	.personality = 0,
	4735	.splice_fd_in = linux.W.EXITED,
	4736	.addr3 = 0,
	4737	.resv = 0,
	4738	};
	4739	ev.yield(null, .nothing);
	4740	switch (cancel_region.errno()) {
	4741	.SUCCESS => return,
	4742	.INTR, .CANCELED => continue,
	4743	.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
	4744	else => \|err\| return unexpectedErrno(err) catch {},
	4745	}
	4746	}
	4747	}
	4748
	4749	fn childCleanup(ev: Evented, child: process.Child) void {
	4750	if (child.stdin) \|*stdin\| {
	4751	ev.close(stdin.handle);
	4752	child.stdin = null;
	4753	}
	4754	if (child.stdout) \|*stdout\| {
	4755	ev.close(stdout.handle);
	4756	child.stdout = null;
	4757	}
	4758	if (child.stderr) \|*stderr\| {
	4759	ev.close(stderr.handle);
	4760	child.stderr = null;
	4761	}
	4762	child.id = null;
	4763	}
	4764
	4765	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
	4766	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4767	const cancel_protection = swapCancelProtection(ev, .blocked);
	4768	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
	4769	ev.scanEnviron() catch \|err\| switch (err) {
	4770	error.Canceled => unreachable, // blocked
	4771	};
	4772	return ev.environ.zig_progress_file;
	4773	}
	4774
	4775	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
	4776	const ev_io = ev.io();
	4777	try ev.environ_mutex.lock(ev_io);
	4778	defer ev.environ_mutex.unlock(ev_io);
	4779	ev.environ.scan(ev.allocator());
	4780	}
	4781
	4782	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
	4783	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4784	_ = ev;
	4785	const clock_id = clockToPosix(clock);
	4786	var timespec: linux.timespec = undefined;
	4787	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
	4788	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
	4789	.INVAL => return error.ClockUnavailable,
	4790	else => \|err\| return unexpectedErrno(err),
	4791	};
	4792	}
	4793
	4794	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
	4795	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4796	_ = ev;
	4797	var tp: linux.timespec = undefined;
	4798	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
	4799	.SUCCESS => return timestampFromPosix(&tp),
	4800	else => return .zero,
	4801	}
	4802	}
	4803
	4804	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
	4805	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4806
	4807	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
	4808	.none => .{
	4809	.{
	4810	.sec = std.math.maxInt(i64),
	4811	.nsec = std.time.ns_per_s - 1,
	4812	},
	4813	.awake,
	4814	linux.IORING_TIMEOUT_ABS,
	4815	},
	4816	.duration => \|duration\| {
	4817	const ns = duration.raw.toNanoseconds();
	4818	break :timespec .{
	4819	.{
	4820	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	4821	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	4822	},
	4823	duration.clock,
	4824	0,
	4825	};
	4826	},
	4827	.deadline => \|deadline\| {
	4828	const ns = deadline.raw.toNanoseconds();
	4829	break :timespec .{
	4830	.{
	4831	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	4832	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	4833	},
	4834	deadline.clock,
	4835	linux.IORING_TIMEOUT_ABS,
	4836	};
	4837	},
	4838	};
	4839	var cancel_region: CancelRegion = .init();
	4840	defer cancel_region.deinit();
	4841	const thread = try cancel_region.awaitIoUring();
	4842	thread.enqueue().* = .{
	4843	.opcode = .TIMEOUT,
	4844	.flags = 0,
	4845	.ioprio = 0,
	4846	.fd = 0,
	4847	.off = 0,
	4848	.addr = @intFromPtr(&timespec),
	4849	.len = 1,
	4850	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	4851	.real => linux.IORING_TIMEOUT_REALTIME,
	4852	else => 0,
	4853	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	4854	}),
	4855	.user_data = @intFromPtr(cancel_region.fiber),
	4856	.buf_index = 0,
	4857	.personality = 0,
	4858	.splice_fd_in = 0,
	4859	.addr3 = 0,
	4860	.resv = 0,
	4861	};
	4862	ev.yield(null, .nothing);
	4863	switch (cancel_region.errno()) {
	4864	// Handles SUCCESS as well as clock not available and unexpected
	4865	// errors. The user had a chance to check clock resolution before
	4866	// getting here, which would have reported 0, making this a legal
	4867	// amount of time to sleep.
	4868	else => return,
	4869	.INTR, .CANCELED => return error.Canceled,
	4870	}
	4871	}
	4872
	4873	fn random(userdata: ?*anyopaque, buffer: []u8) void {
	4874	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4875	var thread: *Thread = .current();
	4876	if (!thread.csprng.isInitialized()) {
	4877	@branchHint(.unlikely);
	4878	var seed: [Csprng.seed_len]u8 = undefined;
	4879	{
	4880	const ev_io = ev.io();
	4881	ev.csprng_mutex.lockUncancelable(ev_io);
	4882	defer ev.csprng_mutex.unlock(ev_io);
	4883	if (!ev.csprng.isInitialized()) {
	4884	@branchHint(.unlikely);
	4885	var cancel_region: CancelRegion = .initBlocked();
	4886	defer cancel_region.deinit();
	4887	ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
	4888	error.Canceled => unreachable, // blocked
	4889	else => fallbackSeed(ev, &seed),
	4890	};
	4891	ev.csprng.rng = .init(seed);
	4892	thread = .current();
	4893	}
	4894	ev.csprng.rng.fill(&seed);
	4895	}
	4896	if (!thread.csprng.isInitialized()) {
	4897	@branchHint(.likely);
	4898	thread.csprng.rng = .init(seed);
	4899	} else thread.csprng.rng.addEntropy(&seed);
	4900	}
	4901	thread.csprng.rng.fill(buffer);
	4902	}
	4903
	4904	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
	4905	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4906	if (buffer.len == 0) return;
	4907	var cancel_region: CancelRegion = .init();
	4908	defer cancel_region.deinit();
	4909	ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
	4910	error.Canceled => return error.Canceled,
	4911	else => return error.EntropyUnavailable,
	4912	};
	4913	}
	4914
	4915	fn netListenIpUnavailable(
	4916	userdata: ?*anyopaque,
	4917	address: net.IpAddress,
	4918	options: net.IpAddress.ListenOptions,
	4919	) net.IpAddress.ListenError!net.Server {
	4920	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4921	_ = ev;
	4922	_ = address;
	4923	_ = options;
	4924	return error.NetworkDown;
	4925	}
	4926
	4927	fn netAcceptUnavailable(
	4928	userdata: ?*anyopaque,
	4929	listen_handle: net.Socket.Handle,
	4930	) net.Server.AcceptError!net.Stream {
	4931	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4932	_ = ev;
	4933	_ = listen_handle;
	4934	return error.NetworkDown;
	4935	}
	4936
	4937	fn netBindIp(
	4938	userdata: ?*anyopaque,
	4939	address: *const net.IpAddress,
	4940	options: net.IpAddress.BindOptions,
	4941	) net.IpAddress.BindError!net.Socket {
	4942	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4943	const family = posixAddressFamily(address);
	4944	var cancel_region: CancelRegion = .init();
	4945	defer cancel_region.deinit();
	4946	const socket_fd = try ev.socket(&cancel_region, family, options);
	4947	errdefer ev.close(socket_fd);
	4948	var storage: PosixAddress = undefined;
	4949	var addr_len = addressToPosix(address, &storage);
	4950	try ev.bind(&cancel_region, socket_fd, &storage.any, addr_len);
	4951	try ev.getsockname(&cancel_region, socket_fd, &storage.any, &addr_len);
	4952	return .{
	4953	.handle = socket_fd,
	4954	.address = addressFromPosix(&storage),
	4955	};
	4956	}
	4957
	4958	fn netBindIpUnavailable(
	4959	userdata: ?*anyopaque,
	4960	address: *const net.IpAddress,
	4961	options: net.IpAddress.BindOptions,
	4962	) net.IpAddress.BindError!net.Socket {
	4963	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4964	_ = ev;
	4965	_ = address;
	4966	_ = options;
	4967	return error.NetworkDown;
	4968	}
	4969
	4970	fn netConnectIpUnavailable(
	4971	userdata: ?*anyopaque,
	4972	address: *const net.IpAddress,
	4973	options: net.IpAddress.ConnectOptions,
	4974	) net.IpAddress.ConnectError!net.Stream {
	4975	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4976	_ = ev;
	4977	_ = address;
	4978	_ = options;
	4979	return error.NetworkDown;
	4980	}
	4981
	4982	fn netListenUnixUnavailable(
	4983	userdata: ?*anyopaque,
	4984	address: *const net.UnixAddress,
	4985	options: net.UnixAddress.ListenOptions,
	4986	) net.UnixAddress.ListenError!net.Socket.Handle {
	4987	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4988	_ = ev;
	4989	_ = address;
	4990	_ = options;
	4991	return error.AddressFamilyUnsupported;
	4992	}
	4993
	4994	fn netConnectUnixUnavailable(
	4995	userdata: ?*anyopaque,
	4996	address: *const net.UnixAddress,
	4997	) net.UnixAddress.ConnectError!net.Socket.Handle {
	4998	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4999	_ = ev;
	5000	_ = address;
	5001	return error.AddressFamilyUnsupported;
	5002	}
	5003
	5004	fn netSocketCreatePairUnavailable(
	5005	userdata: ?*anyopaque,
	5006	options: net.Socket.CreatePairOptions,
	5007	) net.Socket.CreatePairError![2]net.Socket {
	5008	_ = userdata;
	5009	_ = options;
	5010	return error.OperationUnsupported;
	5011	}
	5012
	5013	fn netSendUnavailable(
	5014	userdata: ?*anyopaque,
	5015	handle: net.Socket.Handle,
	5016	messages: []net.OutgoingMessage,
	5017	flags: net.SendFlags,
	5018	) struct { ?net.Socket.SendError, usize } {
	5019	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5020	_ = ev;
	5021	_ = handle;
	5022	_ = messages;
	5023	_ = flags;
	5024	return .{ error.NetworkDown, 0 };
	5025	}
	5026
	5027	fn netReceive(
	5028	userdata: ?*anyopaque,
	5029	handle: net.Socket.Handle,
	5030	message_buffer: []net.IncomingMessage,
	5031	data_buffer: []u8,
	5032	flags: net.ReceiveFlags,
	5033	timeout: Io.Timeout,
	5034	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
	5035	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5036	const ev_io = ev.io();
	5037
	5038	var message_i: usize = 0;
	5039	var data_i: usize = 0;
	5040
	5041	const deadline: ?struct {
	5042	raw: Io.Timestamp,
	5043	timespec: linux.kernel_timespec,
	5044	clock: Io.Clock,
	5045	} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
	5046	const ns = deadline.raw.toNanoseconds();
	5047	break :deadline .{
	5048	.raw = deadline.raw,
	5049	.timespec = .{
	5050	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	5051	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	5052	},
	5053	.clock = deadline.clock,
	5054	};
	5055	} else null;
	5056
	5057	var cancel_region: CancelRegion = .init();
	5058	defer cancel_region.deinit();
	5059	while (true) {
	5060	if (message_buffer.len - message_i == 0) return .{ null, message_i };
	5061	const message = &message_buffer[message_i];
	5062	const remaining_data_buffer = data_buffer[data_i..];
	5063	var storage: PosixAddress = undefined;
	5064	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
	5065	var msg: linux.msghdr = .{
	5066	.name = &storage.any,
	5067	.namelen = @sizeOf(PosixAddress),
	5068	.iov = (&iov)[0..1],
	5069	.iovlen = 1,
	5070	.control = message.control.ptr,
	5071	.controllen = @intCast(message.control.len),
	5072	.flags = undefined,
	5073	};
	5074
	5075	const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
	5076	thread.enqueue().* = .{
	5077	.opcode = .RECVMSG,
	5078	.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
	5079	.ioprio = 0,
	5080	.fd = handle,
	5081	.off = 0,
	5082	.addr = @intFromPtr(&msg),
	5083	.len = 0,
	5084	.rw_flags = linux.MSG.NOSIGNAL \|
	5085	@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
	5086	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
	5087	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
	5088	.user_data = @intFromPtr(cancel_region.fiber),
	5089	.buf_index = 0,
	5090	.personality = 0,
	5091	.splice_fd_in = 0,
	5092	.addr3 = 0,
	5093	.resv = 0,
	5094	};
	5095	if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
	5096	.opcode = .LINK_TIMEOUT,
	5097	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	5098	.ioprio = 0,
	5099	.fd = 0,
	5100	.off = 0,
	5101	.addr = @intFromPtr(&deadline_ptr.timespec),
	5102	.len = 1,
	5103	.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
	5104	.real => linux.IORING_TIMEOUT_REALTIME,
	5105	else => 0,
	5106	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	5107	}),
	5108	.user_data = @intFromEnum(Completion.UserData.wakeup),
	5109	.buf_index = 0,
	5110	.personality = 0,
	5111	.splice_fd_in = 0,
	5112	.addr3 = 0,
	5113	.resv = 0,
	5114	};
	5115	ev.yield(null, .nothing);
	5116	const completion = cancel_region.completion();
	5117	switch (completion.errno()) {
	5118	.SUCCESS => {
	5119	const data = remaining_data_buffer[0..@intCast(completion.result)];
	5120	data_i += data.len;
	5121	message.* = .{
	5122	.from = addressFromPosix(&storage),
	5123	.data = data,
	5124	.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
	5125	.flags = .{
	5126	.eor = (msg.flags & linux.MSG.EOR) != 0,
	5127	.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
	5128	.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
	5129	.oob = (msg.flags & linux.MSG.OOB) != 0,
	5130	.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
	5131	},
	5132	};
	5133	message_i += 1;
	5134	continue;
	5135	},
	5136	.AGAIN => unreachable,
	5137	.INTR, .CANCELED => {
	5138	if (deadline) \|d\| {
	5139	if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
	5140	}
	5141	continue;
	5142	},
	5143
	5144	.BADF => \|err\| return .{ errnoBug(err), message_i },
	5145	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
	5146	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
	5147	.FAULT => \|err\| return .{ errnoBug(err), message_i },
	5148	.INVAL => \|err\| return .{ errnoBug(err), message_i },
	5149	.NOBUFS => return .{ error.SystemResources, message_i },
	5150	.NOMEM => return .{ error.SystemResources, message_i },
	5151	.NOTCONN => return .{ error.SocketUnconnected, message_i },
	5152	.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
	5153	.MSGSIZE => return .{ error.MessageOversize, message_i },
	5154	.PIPE => return .{ error.SocketUnconnected, message_i },
	5155	.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
	5156	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
	5157	.NETDOWN => return .{ error.NetworkDown, message_i },
	5158	else => \|err\| return .{ unexpectedErrno(err), message_i },
	5159	}
	5160	}
	5161	}
	5162
	5163	fn netReceiveUnavailable(
	5164	userdata: ?*anyopaque,
	5165	handle: net.Socket.Handle,
	5166	message_buffer: []net.IncomingMessage,
	5167	data_buffer: []u8,
	5168	flags: net.ReceiveFlags,
	5169	timeout: Io.Timeout,
	5170	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
	5171	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5172	_ = ev;
	5173	_ = handle;
	5174	_ = message_buffer;
	5175	_ = data_buffer;
	5176	_ = flags;
	5177	_ = timeout;
	5178	return .{ error.NetworkDown, 0 };
	5179	}
	5180
	5181	fn netReadUnavailable(
	5182	userdata: ?*anyopaque,
	5183	fd: net.Socket.Handle,
	5184	data: [][]u8,
	5185	) net.Stream.Reader.Error!usize {
	5186	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5187	_ = ev;
	5188	_ = fd;
	5189	_ = data;
	5190	return error.NetworkDown;
	5191	}
	5192
	5193	fn netWriteUnavailable(
	5194	userdata: ?*anyopaque,
	5195	handle: net.Socket.Handle,
	5196	header: []const u8,
	5197	data: []const []const u8,
	5198	splat: usize,
	5199	) net.Stream.Writer.Error!usize {
	5200	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5201	_ = ev;
	5202	_ = handle;
	5203	_ = header;
	5204	_ = data;
	5205	_ = splat;
	5206	return error.NetworkDown;
	5207	}
	5208
	5209	fn netWriteFileUnavailable(
	5210	userdata: ?*anyopaque,
	5211	socket_handle: net.Socket.Handle,
	5212	header: []const u8,
	5213	file_reader: *File.Reader,
	5214	limit: Io.Limit,
	5215	) net.Stream.Writer.WriteFileError!usize {
	5216	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5217	_ = ev;
	5218	_ = socket_handle;
	5219	_ = header;
	5220	_ = file_reader;
	5221	_ = limit;
	5222	return error.NetworkDown;
	5223	}
	5224
	5225	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
	5226	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5227	for (handles) \|handle\| ev.close(handle);
	5228	}
	5229
	5230	fn netCloseUnavailable(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
	5231	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5232	_ = ev;
	5233	_ = handles;
	5234	unreachable; // How you gonna close something that was impossible to open?
	5235	}
	5236
	5237	fn netShutdown(
	5238	userdata: ?*anyopaque,
	5239	handle: net.Socket.Handle,
	5240	how: net.ShutdownHow,
	5241	) net.ShutdownError!void {
	5242	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5243	var cancel_region: CancelRegion = .init();
	5244	defer cancel_region.deinit();
	5245	while (true) {
	5246	const thread = try cancel_region.awaitIoUring();
	5247	thread.enqueue().* = .{
	5248	.opcode = .SHUTDOWN,
	5249	.flags = 0,
	5250	.ioprio = 0,
	5251	.fd = handle,
	5252	.off = 0,
	5253	.addr = 0,
	5254	.len = switch (how) {
	5255	.recv => linux.SHUT.RD,
	5256	.send => linux.SHUT.WR,
	5257	.both => linux.SHUT.RDWR,
	5258	},
	5259	.rw_flags = 0,
	5260	.user_data = @intFromPtr(cancel_region.fiber),
	5261	.buf_index = 0,
	5262	.personality = 0,
	5263	.splice_fd_in = 0,
	5264	.addr3 = 0,
	5265	.resv = 0,
	5266	};
	5267	ev.yield(null, .nothing);
	5268	switch (cancel_region.errno()) {
	5269	.SUCCESS => return,
	5270	.INTR, .CANCELED => continue,
	5271	.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
	5272	.NOTCONN => return error.SocketUnconnected,
	5273	.NOBUFS => return error.SystemResources,
	5274	else => \|err\| return unexpectedErrno(err),
	5275	}
	5276	}
	5277	}
	5278
	5279	fn netShutdownUnavailable(
	5280	userdata: ?*anyopaque,
	5281	handle: net.Socket.Handle,
	5282	how: net.ShutdownHow,
	5283	) net.ShutdownError!void {
	5284	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5285	_ = ev;
	5286	_ = handle;
	5287	_ = how;
	5288	unreachable; // How you gonna shutdown something that was impossible to open?
	5289	}
	5290
	5291	fn netInterfaceNameResolveUnavailable(
	5292	userdata: ?*anyopaque,
	5293	name: *const net.Interface.Name,
	5294	) net.Interface.Name.ResolveError!net.Interface {
	5295	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5296	_ = ev;
	5297	_ = name;
	5298	return error.InterfaceNotFound;
	5299	}
	5300
	5301	fn netInterfaceNameUnavailable(
	5302	userdata: ?*anyopaque,
	5303	interface: net.Interface,
	5304	) net.Interface.NameError!net.Interface.Name {
	5305	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5306	_ = ev;
	5307	_ = interface;
	5308	return error.Unexpected;
	5309	}
	5310
	5311	fn netLookupUnavailable(
	5312	userdata: ?*anyopaque,
	5313	host_name: net.HostName,
	5314	resolved: *Io.Queue(net.HostName.LookupResult),
	5315	options: net.HostName.LookupOptions,
	5316	) net.HostName.LookupError!void {
	5317	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5318	_ = host_name;
	5319	_ = options;
	5320	resolved.close(ev.io());
	5321	return error.NetworkDown;
	5322	}
	5323
	5324	fn bind(
	5325	ev: *Evented,
	5326	cancel_region: *CancelRegion,
	5327	socket_fd: fd_t,
	5328	addr: *const linux.sockaddr,
	5329	addr_len: linux.socklen_t,
	5330	) !void {
	5331	while (true) {
	5332	const thread = try cancel_region.awaitIoUring();
	5333	thread.enqueue().* = .{
	5334	.opcode = .BIND,
	5335	.flags = 0,
	5336	.ioprio = 0,
	5337	.fd = socket_fd,
	5338	.off = addr_len,
	5339	.addr = @intFromPtr(addr),
	5340	.len = 0,
	5341	.rw_flags = 0,
	5342	.user_data = @intFromPtr(cancel_region.fiber),
	5343	.buf_index = 0,
	5344	.personality = 0,
	5345	.splice_fd_in = 0,
	5346	.addr3 = 0,
	5347	.resv = 0,
	5348	};
	5349	ev.yield(null, .nothing);
	5350	switch (cancel_region.errno()) {
	5351	.SUCCESS => return,
	5352	.INTR, .CANCELED => continue,
	5353	.ADDRINUSE => return error.AddressInUse,
	5354	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5355	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5356	.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
	5357	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
	5358	.ADDRNOTAVAIL => return error.AddressUnavailable,
	5359	.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
	5360	.NOMEM => return error.SystemResources,
	5361	else => \|err\| return unexpectedErrno(err),
	5362	}
	5363	}
	5364	}
	5365
	5366	fn close(ev: *Evented, fd: fd_t) void {
	5367	var cancel_region: CancelRegion = .initBlocked();
	5368	defer cancel_region.deinit();
	5369	while (true) {
	5370	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	5371	error.Canceled => unreachable, // blocked
	5372	};
	5373	thread.enqueue().* = .{
	5374	.opcode = .CLOSE,
	5375	.flags = 0,
	5376	.ioprio = 0,
	5377	.fd = fd,
	5378	.off = 0,
	5379	.addr = 0,
	5380	.len = 0,
	5381	.rw_flags = 0,
	5382	.user_data = @intFromPtr(cancel_region.fiber),
	5383	.buf_index = 0,
	5384	.personality = 0,
	5385	.splice_fd_in = 0,
	5386	.addr3 = 0,
	5387	.resv = 0,
	5388	};
	5389	ev.yield(null, .nothing);
	5390	switch (cancel_region.errno()) {
	5391	.SUCCESS => return,
	5392	.INTR, .CANCELED => continue,
	5393	.BADF => unreachable, // Always a race condition.
	5394	else => break,
	5395	}
	5396	}
	5397	}
	5398
	5399	fn fchmodat(
	5400	ev: *Evented,
	5401	cancel_region: *CancelRegion,
	5402	dir: fd_t,
	5403	path: [*:0]const u8,
	5404	mode: linux.mode_t,
	5405	flags: u32,
	5406	) Dir.SetFilePermissionsError!void {
	5407	_ = ev;
	5408	while (true) {
	5409	try cancel_region.await(.nothing);
	5410	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
	5411	.SUCCESS => return,
	5412	.INTR => continue,
	5413	.BADF => \|err\| return errnoBug(err),
	5414	.FAULT => \|err\| return errnoBug(err),
	5415	.INVAL => \|err\| return errnoBug(err),
	5416	.ACCES => return error.AccessDenied,
	5417	.IO => return error.InputOutput,
	5418	.LOOP => return error.SymLinkLoop,
	5419	.NOENT => return error.FileNotFound,
	5420	.NOMEM => return error.SystemResources,
	5421	.NOTDIR => return error.FileNotFound,
	5422	.OPNOTSUPP => return error.OperationUnsupported,
	5423	.PERM => return error.PermissionDenied,
	5424	.ROFS => return error.ReadOnlyFileSystem,
	5425	else => \|err\| return unexpectedErrno(err),
	5426	}
	5427	}
	5428	}
	5429
	5430	fn fchownat(
	5431	ev: *Evented,
	5432	cancel_region: *CancelRegion,
	5433	dir: fd_t,
	5434	path: [*:0]const u8,
	5435	owner: linux.uid_t,
	5436	group: linux.gid_t,
	5437	flags: u32,
	5438	) File.SetOwnerError!void {
	5439	_ = ev;
	5440	while (true) {
	5441	try cancel_region.await(.nothing);
	5442	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
	5443	.SUCCESS => return,
	5444	.INTR => continue,
	5445	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
	5446	.FAULT => \|err\| return errnoBug(err),
	5447	.INVAL => \|err\| return errnoBug(err),
	5448	.ACCES => return error.AccessDenied,
	5449	.IO => return error.InputOutput,
	5450	.LOOP => return error.SymLinkLoop,
	5451	.NOENT => return error.FileNotFound,
	5452	.NOMEM => return error.SystemResources,
	5453	.NOTDIR => return error.FileNotFound,
	5454	.PERM => return error.PermissionDenied,
	5455	.ROFS => return error.ReadOnlyFileSystem,
	5456	else => \|err\| return unexpectedErrno(err),
	5457	}
	5458	}
	5459	}
	5460
	5461	fn flock(
	5462	ev: *Evented,
	5463	cancel_region: *CancelRegion,
	5464	fd: fd_t,
	5465	op: File.Lock,
	5466	blocking: enum { blocking, nonblocking },
	5467	) (File.LockError \|\| error{WouldBlock})!void {
	5468	while (true) {
	5469	try cancel_region.await(.nothing);
	5470	switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
	5471	.none => LOCK.UN,
	5472	.shared => LOCK.SH,
	5473	.exclusive => LOCK.EX,
	5474	})))) {
	5475	.SUCCESS => return,
	5476	.INTR => continue,
	5477	.BADF => \|err\| return errnoBug(err),
	5478	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5479	.NOLCK => return error.SystemResources,
	5480	.AGAIN => {
	5481	const thread = try cancel_region.awaitIoUring();
	5482	thread.enqueue().* = .{
	5483	.opcode = .NOP,
	5484	.flags = 0,
	5485	.ioprio = 0,
	5486	.fd = 0,
	5487	.off = 0,
	5488	.addr = 0,
	5489	.len = 0,
	5490	.rw_flags = 0,
	5491	.user_data = @intFromPtr(cancel_region.fiber),
	5492	.buf_index = 0,
	5493	.personality = 0,
	5494	.splice_fd_in = 0,
	5495	.addr3 = 0,
	5496	.resv = 0,
	5497	};
	5498	ev.yield(null, .nothing);
	5499	switch (cancel_region.errno()) {
	5500	.SUCCESS, .INTR, .CANCELED => {},
	5501	else => unreachable,
	5502	}
	5503	switch (blocking) {
	5504	.blocking => continue,
	5505	.nonblocking => return error.WouldBlock,
	5506	}
	5507	},
	5508	.OPNOTSUPP => return error.FileLocksUnsupported,
	5509	else => \|err\| return unexpectedErrno(err),
	5510	}
	5511	}
	5512	}
	5513
	5514	fn getsockname(
	5515	ev: *Evented,
	5516	cancel_region: *CancelRegion,
	5517	socket_fd: fd_t,
	5518	addr: *linux.sockaddr,
	5519	addr_len: *linux.socklen_t,
	5520	) !void {
	5521	_ = ev;
	5522	while (true) {
	5523	try cancel_region.await(.nothing);
	5524	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
	5525	.SUCCESS => return,
	5526	.INTR => continue,
	5527	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5528	.FAULT => \|err\| return errnoBug(err),
	5529	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5530	.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
	5531	.NOBUFS => return error.SystemResources,
	5532	else => \|err\| return unexpectedErrno(err),
	5533	}
	5534	}
	5535	}
	5536
	5537	fn linkat(
	5538	ev: *Evented,
	5539	cancel_region: *CancelRegion,
	5540	old_dir: fd_t,
	5541	old_path: [*:0]const u8,
	5542	new_dir: fd_t,
	5543	new_path: [*:0]const u8,
	5544	flags: u32,
	5545	) File.HardLinkError!void {
	5546	while (true) {
	5547	const thread = try cancel_region.awaitIoUring();
	5548	thread.enqueue().* = .{
	5549	.opcode = .LINKAT,
	5550	.flags = 0,
	5551	.ioprio = 0,
	5552	.fd = old_dir,
	5553	.off = @intFromPtr(new_path),
	5554	.addr = @intFromPtr(old_path),
	5555	.len = @bitCast(new_dir),
	5556	.rw_flags = flags,
	5557	.user_data = @intFromPtr(cancel_region.fiber),
	5558	.buf_index = 0,
	5559	.personality = 0,
	5560	.splice_fd_in = 0,
	5561	.addr3 = 0,
	5562	.resv = 0,
	5563	};
	5564	ev.yield(null, .nothing);
	5565	switch (cancel_region.errno()) {
	5566	.SUCCESS => return,
	5567	.INTR, .CANCELED => continue,
	5568	.ACCES => return error.AccessDenied,
	5569	.DQUOT => return error.DiskQuota,
	5570	.EXIST => return error.PathAlreadyExists,
	5571	.IO => return error.HardwareFailure,
	5572	.LOOP => return error.SymLinkLoop,
	5573	.MLINK => return error.LinkQuotaExceeded,
	5574	.NAMETOOLONG => return error.NameTooLong,
	5575	.NOENT => return error.FileNotFound,
	5576	.NOMEM => return error.SystemResources,
	5577	.NOSPC => return error.NoSpaceLeft,
	5578	.NOTDIR => return error.NotDir,
	5579	.PERM => return error.PermissionDenied,
	5580	.ROFS => return error.ReadOnlyFileSystem,
	5581	.XDEV => return error.CrossDevice,
	5582	.ILSEQ => return error.BadPathName,
	5583	.FAULT => \|err\| return errnoBug(err),
	5584	.INVAL => \|err\| return errnoBug(err),
	5585	else => \|err\| return unexpectedErrno(err),
	5586	}
	5587	}
	5588	}
	5589
	5590	fn lseek(
	5591	ev: *Evented,
	5592	cancel_region: *CancelRegion,
	5593	fd: fd_t,
	5594	offset: u64,
	5595	whence: u32,
	5596	) File.SeekError!void {
	5597	_ = ev;
	5598	while (true) {
	5599	try cancel_region.await(.nothing);
	5600	var result: u64 = undefined;
	5601	switch (linux.errno(switch (@sizeOf(usize)) {
	5602	else => comptime unreachable,
	5603	4 => linux.llseek(fd, offset, &result, whence),
	5604	8 => linux.lseek(fd, @bitCast(offset), whence),
	5605	})) {
	5606	.SUCCESS => return,
	5607	.INTR => continue,
	5608	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5609	.INVAL => return error.Unseekable,
	5610	.OVERFLOW => return error.Unseekable,
	5611	.SPIPE => return error.Unseekable,
	5612	.NXIO => return error.Unseekable,
	5613	else => \|err\| return unexpectedErrno(err),
	5614	}
	5615	}
	5616	}
	5617
	5618	fn openat(
	5619	ev: *Evented,
	5620	cancel_region: *CancelRegion,
	5621	dir: fd_t,
	5622	path: [*:0]const u8,
	5623	flags: linux.O,
	5624	mode: linux.mode_t,
	5625	) File.OpenError!fd_t {
	5626	var mut_flags = flags;
	5627	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
	5628	while (true) {
	5629	const thread = try cancel_region.awaitIoUring();
	5630	thread.enqueue().* = .{
	5631	.opcode = .OPENAT,
	5632	.flags = 0,
	5633	.ioprio = 0,
	5634	.fd = dir,
	5635	.off = 0,
	5636	.addr = @intFromPtr(path),
	5637	.len = mode,
	5638	.rw_flags = @bitCast(mut_flags),
	5639	.user_data = @intFromPtr(cancel_region.fiber),
	5640	.buf_index = 0,
	5641	.personality = 0,
	5642	.splice_fd_in = 0,
	5643	.addr3 = 0,
	5644	.resv = 0,
	5645	};
	5646	ev.yield(null, .nothing);
	5647	const completion = cancel_region.completion();
	5648	switch (completion.errno()) {
	5649	.SUCCESS => return completion.result,
	5650	.INTR, .CANCELED => continue,
	5651	.FAULT => \|err\| return errnoBug(err),
	5652	.INVAL => return error.BadPathName,
	5653	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5654	.ACCES => return error.AccessDenied,
	5655	.FBIG => return error.FileTooBig,
	5656	.OVERFLOW => return error.FileTooBig,
	5657	.ISDIR => return error.IsDir,
	5658	.LOOP => return error.SymLinkLoop,
	5659	.MFILE => return error.ProcessFdQuotaExceeded,
	5660	.NAMETOOLONG => return error.NameTooLong,
	5661	.NFILE => return error.SystemFdQuotaExceeded,
	5662	.NODEV => return error.NoDevice,
	5663	.NOENT => return error.FileNotFound,
	5664	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
	5665	.NOMEM => return error.SystemResources,
	5666	.NOSPC => return error.NoSpaceLeft,
	5667	.NOTDIR => return error.NotDir,
	5668	.PERM => return error.PermissionDenied,
	5669	.EXIST => return error.PathAlreadyExists,
	5670	.BUSY => return error.DeviceBusy,
	5671	.OPNOTSUPP => return error.FileLocksUnsupported,
	5672	.AGAIN => return error.WouldBlock,
	5673	.TXTBSY => return error.FileBusy,
	5674	.NXIO => return error.NoDevice,
	5675	.ILSEQ => return error.BadPathName,
	5676	else => \|err\| return unexpectedErrno(err),
	5677	}
	5678	}
	5679	}
	5680
	5681	fn preadv(
	5682	ev: *Evented,
	5683	cancel_region: *CancelRegion,
	5684	fd: fd_t,
	5685	iov: []const iovec,
	5686	offset: ?u64,
	5687	) File.Reader.Error!usize {
	5688	if (iov.len == 0) return 0;
	5689	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
	5690	while (true) {
	5691	const thread = try cancel_region.awaitIoUring();
	5692	thread.enqueue().* = .{
	5693	.opcode = if (gather) .READV else .READ,
	5694	.flags = 0,
	5695	.ioprio = 0,
	5696	.fd = fd,
	5697	.off = offset orelse std.math.maxInt(u64),
	5698	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
	5699	.len = @intCast(if (gather) iov.len else iov[0].len),
	5700	.rw_flags = 0,
	5701	.user_data = @intFromPtr(cancel_region.fiber),
	5702	.buf_index = 0,
	5703	.personality = 0,
	5704	.splice_fd_in = 0,
	5705	.addr3 = 0,
	5706	.resv = 0,
	5707	};
	5708	ev.yield(null, .nothing);
	5709	const completion = cancel_region.completion();
	5710	switch (completion.errno()) {
	5711	.SUCCESS => return @as(u32, @bitCast(completion.result)),
	5712	.INTR, .CANCELED => continue,
	5713	.INVAL => \|err\| return errnoBug(err),
	5714	.FAULT => \|err\| return errnoBug(err),
	5715	.AGAIN => return error.WouldBlock,
	5716	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
	5717	.IO => return error.InputOutput,
	5718	.ISDIR => return error.IsDir,
	5719	.NOBUFS => return error.SystemResources,
	5720	.NOMEM => return error.SystemResources,
	5721	.NOTCONN => return error.SocketUnconnected,
	5722	.CONNRESET => return error.ConnectionResetByPeer,
	5723	else => \|err\| return unexpectedErrno(err),
	5724	}
	5725	}
	5726	}
	5727
	5728	fn pwritev(
	5729	ev: *Evented,
	5730	cancel_region: *CancelRegion,
	5731	fd: fd_t,
	5732	iov: []const iovec_const,
	5733	offset: ?u64,
	5734	) File.Writer.Error!usize {
	5735	if (iov.len == 0) return 0;
	5736	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
	5737	while (true) {
	5738	const thread = try cancel_region.awaitIoUring();
	5739	thread.enqueue().* = .{
	5740	.opcode = if (scatter) .WRITEV else .WRITE,
	5741	.flags = 0,
	5742	.ioprio = 0,
	5743	.fd = fd,
	5744	.off = offset orelse std.math.maxInt(u64),
	5745	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
	5746	.len = @intCast(if (scatter) iov.len else iov[0].len),
	5747	.rw_flags = 0,
	5748	.user_data = @intFromPtr(cancel_region.fiber),
	5749	.buf_index = 0,
	5750	.personality = 0,
	5751	.splice_fd_in = 0,
	5752	.addr3 = 0,
	5753	.resv = 0,
	5754	};
	5755	ev.yield(null, .nothing);
	5756	const completion = cancel_region.completion();
	5757	switch (completion.errno()) {
	5758	.SUCCESS => return @as(u32, @bitCast(completion.result)),
	5759	.INTR, .CANCELED => continue,
	5760	.INVAL => \|err\| return errnoBug(err),
	5761	.FAULT => \|err\| return errnoBug(err),
	5762	.AGAIN => return error.WouldBlock,
	5763	.BADF => return error.NotOpenForWriting, // Can be a race condition.
	5764	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
	5765	.DQUOT => return error.DiskQuota,
	5766	.FBIG => return error.FileTooBig,
	5767	.IO => return error.InputOutput,
	5768	.NOSPC => return error.NoSpaceLeft,
	5769	.PERM => return error.PermissionDenied,
	5770	.PIPE => return error.BrokenPipe,
	5771	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
	5772	.BUSY => return error.DeviceBusy,
	5773	else => \|err\| return unexpectedErrno(err),
	5774	}
	5775	}
	5776	}
	5777
	5778	fn readAll(
	5779	ev: *Evented,
	5780	cancel_region: *CancelRegion,
	5781	fd: fd_t,
	5782	buffer: []u8,
	5783	) (File.Reader.Error \|\| error{EndOfStream})!void {
	5784	var index: usize = 0;
	5785	while (buffer.len - index != 0) {
	5786	const len = try ev.preadv(cancel_region, fd, &.{
	5787	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
	5788	}, null);
	5789	if (len == 0) return error.EndOfStream;
	5790	index += len;
	5791	}
	5792	}
	5793
	5794	fn realPath(
	5795	ev: *Evented,
	5796	cancel_region: *CancelRegion,
	5797	fd: fd_t,
	5798	out_buffer: []u8,
	5799	) File.RealPathError!usize {
	5800	_ = ev;
	5801	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
	5802	const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
	5803	unreachable;
	5804	while (true) {
	5805	try cancel_region.await(.nothing);
	5806	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
	5807	switch (linux.errno(rc)) {
	5808	.SUCCESS => return rc,
	5809	.INTR => continue,
	5810	.ACCES => return error.AccessDenied,
	5811	.FAULT => \|err\| return errnoBug(err),
	5812	.IO => return error.FileSystem,
	5813	.LOOP => return error.SymLinkLoop,
	5814	.NAMETOOLONG => return error.NameTooLong,
	5815	.NOENT => return error.FileNotFound,
	5816	.NOMEM => return error.SystemResources,
	5817	.NOTDIR => return error.NotDir,
	5818	.ILSEQ => \|err\| return errnoBug(err),
	5819	else => \|err\| return unexpectedErrno(err),
	5820	}
	5821	}
	5822	}
	5823
	5824	fn renameat(
	5825	ev: *Evented,
	5826	cancel_region: *CancelRegion,
	5827	old_dir: fd_t,
	5828	old_path: [*:0]const u8,
	5829	new_dir: fd_t,
	5830	new_path: [*:0]const u8,
	5831	flags: linux.RENAME,
	5832	) Dir.RenameError!void {
	5833	while (true) {
	5834	const thread = try cancel_region.awaitIoUring();
	5835	thread.enqueue().* = .{
	5836	.opcode = .RENAMEAT,
	5837	.flags = 0,
	5838	.ioprio = 0,
	5839	.fd = old_dir,
	5840	.off = @intFromPtr(new_path),
	5841	.addr = @intFromPtr(old_path),
	5842	.len = @bitCast(new_dir),
	5843	.rw_flags = @bitCast(flags),
	5844	.user_data = @intFromPtr(cancel_region.fiber),
	5845	.buf_index = 0,
	5846	.personality = 0,
	5847	.splice_fd_in = 0,
	5848	.addr3 = 0,
	5849	.resv = 0,
	5850	};
	5851	ev.yield(null, .nothing);
	5852	switch (cancel_region.errno()) {
	5853	.SUCCESS => return,
	5854	.INTR, .CANCELED => continue,
	5855	.ACCES => return error.AccessDenied,
	5856	.PERM => return error.PermissionDenied,
	5857	.BUSY => return error.FileBusy,
	5858	.DQUOT => return error.DiskQuota,
	5859	.ISDIR => return error.IsDir,
	5860	.IO => return error.HardwareFailure,
	5861	.LOOP => return error.SymLinkLoop,
	5862	.MLINK => return error.LinkQuotaExceeded,
	5863	.NAMETOOLONG => return error.NameTooLong,
	5864	.NOENT => return error.FileNotFound,
	5865	.NOTDIR => return error.NotDir,
	5866	.NOMEM => return error.SystemResources,
	5867	.NOSPC => return error.NoSpaceLeft,
	5868	.EXIST => return error.DirNotEmpty,
	5869	.NOTEMPTY => return error.DirNotEmpty,
	5870	.ROFS => return error.ReadOnlyFileSystem,
	5871	.XDEV => return error.CrossDevice,
	5872	.ILSEQ => return error.BadPathName,
	5873	.FAULT => \|err\| return errnoBug(err),
	5874	.INVAL => \|err\| return errnoBug(err),
	5875	else => \|err\| return unexpectedErrno(err),
	5876	}
	5877	}
	5878	}
	5879
	5880	fn setsockopt(
	5881	ev: *Evented,
	5882	cancel_region: *CancelRegion,
	5883	fd: fd_t,
	5884	level: i32,
	5885	opt_name: u32,
	5886	option: u32,
	5887	) !void {
	5888	const o: []const u8 = @ptrCast(&option);
	5889	while (true) {
	5890	const off: extern struct {
	5891	cmd_op: linux.IO_URING_SOCKET_OP,
	5892	pad: u32,
	5893	} align(@alignOf(u64)) = .{
	5894	.cmd_op = .SETSOCKOPT,
	5895	.pad = 0,
	5896	};
	5897	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
	5898	.level = level,
	5899	.opt_name = opt_name,
	5900	};
	5901	const thread = try cancel_region.awaitIoUring();
	5902	thread.enqueue().* = .{
	5903	.opcode = .URING_CMD,
	5904	.flags = 0,
	5905	.ioprio = 0,
	5906	.fd = fd,
	5907	.off = @as(const u64, @ptrCast(&off)).,
	5908	.addr = @as(const u64, @ptrCast(&addr)).,
	5909	.len = 0,
	5910	.rw_flags = 0,
	5911	.user_data = @intFromPtr(cancel_region.fiber),
	5912	.buf_index = 0,
	5913	.personality = 0,
	5914	.splice_fd_in = @intCast(o.len),
	5915	.addr3 = @intFromPtr(o.ptr),
	5916	.resv = 0,
	5917	};
	5918	ev.yield(null, .nothing);
	5919	switch (cancel_region.errno()) {
	5920	.SUCCESS => return,
	5921	.INTR, .CANCELED => continue,
	5922	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5923	.NOTSOCK => \|err\| return errnoBug(err),
	5924	.INVAL => \|err\| return errnoBug(err),
	5925	.FAULT => \|err\| return errnoBug(err),
	5926	else => \|err\| return unexpectedErrno(err),
	5927	}
	5928	}
	5929	}
	5930
	5931	fn socket(
	5932	ev: *Evented,
	5933	cancel_region: *CancelRegion,
	5934	family: linux.sa_family_t,
	5935	options: net.IpAddress.BindOptions,
	5936	) error{
	5937	AddressFamilyUnsupported,
	5938	ProtocolUnsupportedBySystem,
	5939	ProcessFdQuotaExceeded,
	5940	SystemFdQuotaExceeded,
	5941	SystemResources,
	5942	ProtocolUnsupportedByAddressFamily,
	5943	SocketModeUnsupported,
	5944	OptionUnsupported,
	5945	Unexpected,
	5946	Canceled,
	5947	}!fd_t {
	5948	const mode = posixSocketMode(options.mode);
	5949	const protocol = posixProtocol(options.protocol);
	5950	const socket_fd = while (true) {
	5951	const thread = try cancel_region.awaitIoUring();
	5952	thread.enqueue().* = .{
	5953	.opcode = .SOCKET,
	5954	.flags = 0,
	5955	.ioprio = 0,
	5956	.fd = family,
	5957	.off = mode \| linux.SOCK.CLOEXEC,
	5958	.addr = 0,
	5959	.len = protocol,
	5960	.rw_flags = 0,
	5961	.user_data = @intFromPtr(cancel_region.fiber),
	5962	.buf_index = 0,
	5963	.personality = 0,
	5964	.splice_fd_in = 0,
	5965	.addr3 = 0,
	5966	.resv = 0,
	5967	};
	5968	ev.yield(null, .nothing);
	5969	const completion = cancel_region.completion();
	5970	switch (completion.errno()) {
	5971	.SUCCESS => break completion.result,
	5972	.INTR, .CANCELED => continue,
	5973	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
	5974	.INVAL => return error.ProtocolUnsupportedBySystem,
	5975	.MFILE => return error.ProcessFdQuotaExceeded,
	5976	.NFILE => return error.SystemFdQuotaExceeded,
	5977	.NOBUFS => return error.SystemResources,
	5978	.NOMEM => return error.SystemResources,
	5979	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
	5980	.PROTOTYPE => return error.SocketModeUnsupported,
	5981	else => \|err\| return unexpectedErrno(err),
	5982	}
	5983	};
	5984	errdefer ev.close(socket_fd);
	5985
	5986	if (options.ip6_only) {
	5987	if (linux.IPV6 == void) return error.OptionUnsupported;
	5988	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
	5989	}
	5990
	5991	return socket_fd;
	5992	}
	5993
	5994	fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
	5995	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
	5996	error.BadPathName, error.NameTooLong => unreachable, // path is empty
	5997	error.AccessDenied => return errnoBug(.ACCES),
	5998	error.SymLinkLoop => return errnoBug(.LOOP),
	5999	error.FileNotFound => return errnoBug(.NOENT),
	6000	error.NotDir => return errnoBug(.NOTDIR),
	6001	else => \|e\| return e,
	6002	};
	6003	}
	6004
	6005	fn statx(
	6006	ev: *Evented,
	6007	cancel_region: *CancelRegion,
	6008	dir: fd_t,
	6009	path: [*:0]const u8,
	6010	flags: u32,
	6011	) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
	6012	while (true) {
	6013	var statx_buf = std.mem.zeroes(linux.Statx);
	6014	const thread = try cancel_region.awaitIoUring();
	6015	thread.enqueue().* = .{
	6016	.opcode = .STATX,
	6017	.flags = 0,
	6018	.ioprio = 0,
	6019	.fd = dir,
	6020	.off = @intFromPtr(&statx_buf),
	6021	.addr = @intFromPtr(path),
	6022	.len = @bitCast(linux_statx_request),
	6023	.rw_flags = flags,
	6024	.user_data = @intFromPtr(cancel_region.fiber),
	6025	.buf_index = 0,
	6026	.personality = 0,
	6027	.splice_fd_in = 0,
	6028	.addr3 = 0,
	6029	.resv = 0,
	6030	};
	6031	ev.yield(null, .nothing);
	6032	switch (cancel_region.errno()) {
	6033	.SUCCESS => return statFromLinux(&statx_buf),
	6034	.INTR, .CANCELED => continue,
	6035	.ACCES => return error.AccessDenied,
	6036	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	6037	.FAULT => \|err\| return errnoBug(err),
	6038	.INVAL => \|err\| return errnoBug(err),
	6039	.LOOP => return error.SymLinkLoop,
	6040	.NAMETOOLONG => \|err\| return errnoBug(err),
	6041	.NOENT => return error.FileNotFound,
	6042	.NOTDIR => return error.NotDir,
	6043	.NOMEM => return error.SystemResources,
	6044	else => \|err\| return unexpectedErrno(err),
	6045	}
	6046	}
	6047	}
	6048
	6049	fn urandomReadAll(
	6050	ev: *Evented,
	6051	cancel_region: *CancelRegion,
	6052	buffer: []u8,
	6053	) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
	6054	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
	6055	.ACCMODE = .RDONLY,
	6056	.CLOEXEC = true,
	6057	}), buffer);
	6058	}
	6059
	6060	fn utimensat(
	6061	ev: *Evented,
	6062	cancel_region: *CancelRegion,
	6063	dir: fd_t,
	6064	path: [*:0]const u8,
	6065	times: ?*const [2]linux.timespec,
	6066	flags: u32,
	6067	) File.SetTimestampsError!void {
	6068	_ = ev;
	6069	while (true) {
	6070	try cancel_region.await(.nothing);
	6071	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
	6072	.SUCCESS => return,
	6073	.INTR => continue,
	6074	.BADF => \|err\| return errnoBug(err), // always a race condition
	6075	.FAULT => \|err\| return errnoBug(err),
	6076	.INVAL => \|err\| return errnoBug(err),
	6077	.ACCES => return error.AccessDenied,
	6078	.PERM => return error.PermissionDenied,
	6079	.ROFS => return error.ReadOnlyFileSystem,
	6080	else => \|err\| return unexpectedErrno(err),
	6081	}
	6082	}
	6083	}
	6084
	6085	fn writeAll(
	6086	ev: *Evented,
	6087	cancel_region: *CancelRegion,
	6088	fd: fd_t,
	6089	buffer: []const u8,
	6090	) (File.Writer.Error \|\| error{EndOfStream})!void {
	6091	var index: usize = 0;
	6092	while (buffer.len - index != 0) {
	6093	const len = try ev.pwritev(cancel_region, fd, &.{
	6094	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
	6095	}, null);
	6096	if (len == 0) return error.EndOfStream;
	6097	index += len;
	6098	}
	6099	}
	6100
	6101	test {
	6102	_ = Fiber.CancelProtection;
1497	6103	}

lib/std/Io/Threaded.zig+221-229

...	...	@@ -78,7 +78,7 @@ null_file: NullFile = .{},
78	78	random_file: RandomFile = .{},
79	79	pipe_file: PipeFile = .{},
80	80
81		csprng: Csprng = .{},
	81	csprng: Csprng = .uninitialized,
82	82
83	83	system_basic_information: SystemBasicInformation = .{},
84	84
...	...	@@ -88,10 +88,12 @@ const SystemBasicInformation = if (!is_windows) struct {} else struct {
88	88	};
89	89
90	90	pub const Csprng = struct {
91		rng: std.Random.DefaultCsprng = .{
	91	rng: std.Random.DefaultCsprng,
	92
	93	pub const uninitialized: Csprng = .{ .rng = .{
92	94	.state = undefined,
93	95	.offset = std.math.maxInt(usize),
94		},
	96	} };
95	97
96	98	pub const seed_len = std.Random.DefaultCsprng.secret_seed_length;
97	99
...	...	@@ -120,7 +122,7 @@ pub const Argv0 = switch (native_os) {
120	122	},
121	123	};
122	124
123		const Environ = struct {
	125	pub const Environ = struct {
124	126	/// Unmodified data directly from the OS.
125	127	process_environ: process.Environ,
126	128	/// Protected by `mutex`. Determines whether the other fields have been
...	...	@@ -157,6 +159,127 @@ const Environ = struct {
157	159	HOME: ?[:0]const u8 = null,
158	160	},
159	161	};
	162
	163	pub fn scan(environ: *Environ, allocator: std.mem.Allocator) void {
	164	if (environ.initialized) return;
	165	environ.initialized = true;
	166
	167	if (is_windows) {
	168	// This value expires with any call that modifies the environment,
	169	// which is outside of this Io implementation's control, so references
	170	// must be short-lived.
	171	const peb = windows.peb();
	172	assert(windows.ntdll.RtlEnterCriticalSection(peb.FastPebLock) == .SUCCESS);
	173	defer assert(windows.ntdll.RtlLeaveCriticalSection(peb.FastPebLock) == .SUCCESS);
	174	const ptr = peb.ProcessParameters.Environment;
	175
	176	var i: usize = 0;
	177	while (ptr[i] != 0) {
	178	// There are some special environment variables that start with =,
	179	// so we need a special case to not treat = as a key/value separator
	180	// if it's the first character.
	181	// https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
	182	const key_start = i;
	183	if (ptr[i] == '=') i += 1;
	184	while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
	185	const key_w = ptr[key_start..i];
	186
	187	const value_start = i + 1;
	188	while (ptr[i] != 0) : (i += 1) {} // skip over '=' and value
	189	const value_w = ptr[value_start..i];
	190	i += 1; // skip over null byte
	191
	192	if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'N', 'O', '_', 'C', 'O', 'L', 'O', 'R' })) {
	193	environ.exist.NO_COLOR = true;
	194	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'C', 'L', 'I', 'C', 'O', 'L', 'O', 'R', '_', 'F', 'O', 'R', 'C', 'E' })) {
	195	environ.exist.CLICOLOR_FORCE = true;
	196	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'Z', 'I', 'G', '_', 'P', 'R', 'O', 'G', 'R', 'E', 'S', 'S' })) {
	197	environ.zig_progress_file = file: {
	198	var value_buf: [std.fmt.count("{d}", .{std.math.maxInt(usize)})]u8 = undefined;
	199	const len = std.unicode.calcWtf8Len(value_w);
	200	if (len > value_buf.len) break :file error.UnrecognizedFormat;
	201	assert(std.unicode.wtf16LeToWtf8(&value_buf, value_w) == len);
	202	break :file .{
	203	.handle = @ptrFromInt(std.fmt.parseInt(usize, value_buf[0..len], 10) catch
	204	break :file error.UnrecognizedFormat),
	205	.flags = .{ .nonblocking = true },
	206	};
	207	};
	208	}
	209	comptime assert(@sizeOf(String) == 0);
	210	}
	211	} else if (native_os == .wasi and !builtin.link_libc) {
	212	var environ_size: usize = undefined;
	213	var environ_buf_size: usize = undefined;
	214
	215	switch (std.os.wasi.environ_sizes_get(&environ_size, &environ_buf_size)) {
	216	.SUCCESS => {},
	217	else => \|err\| {
	218	environ.err = posix.unexpectedErrno(err);
	219	return;
	220	},
	221	}
	222	if (environ_size == 0) return;
	223
	224	const wasi_environ = allocator.alloc([*:0]u8, environ_size) catch \|err\| {
	225	environ.err = err;
	226	return;
	227	};
	228	defer allocator.free(wasi_environ);
	229	const wasi_environ_buf = allocator.alloc(u8, environ_buf_size) catch \|err\| {
	230	environ.err = err;
	231	return;
	232	};
	233	defer allocator.free(wasi_environ_buf);
	234
	235	switch (std.os.wasi.environ_get(wasi_environ.ptr, wasi_environ_buf.ptr)) {
	236	.SUCCESS => {},
	237	else => \|err\| {
	238	environ.err = posix.unexpectedErrno(err);
	239	return;
	240	},
	241	}
	242
	243	for (wasi_environ) \|env\| {
	244	const pair = std.mem.sliceTo(env, 0);
	245	var parts = std.mem.splitScalar(u8, pair, '=');
	246	const key = parts.first();
	247	if (std.mem.eql(u8, key, "NO_COLOR")) {
	248	environ.exist.NO_COLOR = true;
	249	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
	250	environ.exist.CLICOLOR_FORCE = true;
	251	}
	252	comptime assert(@sizeOf(String) == 0);
	253	}
	254	} else {
	255	for (environ.process_environ.block.slice) \|opt_entry\| {
	256	const entry = opt_entry.?;
	257	var entry_i: usize = 0;
	258	while (entry[entry_i] != 0 and entry[entry_i] != '=') : (entry_i += 1) {}
	259	const key = entry[0..entry_i];
	260
	261	var end_i: usize = entry_i;
	262	while (entry[end_i] != 0) : (end_i += 1) {}
	263	const value = entry[entry_i + 1 .. end_i :0];
	264
	265	if (std.mem.eql(u8, key, "NO_COLOR")) {
	266	environ.exist.NO_COLOR = true;
	267	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
	268	environ.exist.CLICOLOR_FORCE = true;
	269	} else if (std.mem.eql(u8, key, "ZIG_PROGRESS")) {
	270	environ.zig_progress_file = file: {
	271	break :file .{
	272	.handle = std.fmt.parseInt(u31, value, 10) catch
	273	break :file error.UnrecognizedFormat,
	274	.flags = .{ .nonblocking = true },
	275	};
	276	};
	277	} else inline for (@typeInfo(String).@"struct".fields) \|field\| {
	278	if (std.mem.eql(u8, key, field.name)) @field(environ.string, field.name) = value;
	279	}
	280	}
	281	}
	282	}
160	283	};
161	284
162	285	pub const NullFile = switch (native_os) {
...	...	@@ -1397,13 +1520,13 @@ pub fn waitForApcOrAlert() void {
1397	1520	_ = windows.ntdll.NtDelayExecution(windows.TRUE, &infinite_timeout);
1398	1521	}
1399	1522
1400		const max_iovecs_len = 8;
1401		const splat_buffer_size = 64;
	1523	pub const max_iovecs_len = 8;
	1524	pub const splat_buffer_size = 64;
1402	1525	/// Happens to be the same number that matches maximum number of handles that
1403	1526	/// NtWaitForMultipleObjects accepts. We use this value also for poll() on
1404	1527	/// posix systems.
1405	1528	const poll_buffer_len = 64;
1406		const default_PATH = "/usr/local/bin:/bin/:/usr/bin";
	1529	pub const default_PATH = "/usr/local/bin:/bin/:/usr/bin";
1407	1530	/// There are multiple kernel bugs being worked around with retries.
1408	1531	const max_windows_kernel_bug_retries = 13;
1409	1532
...	...	@@ -1588,7 +1711,7 @@ fn worker(t: *Threaded) void {
1588	1711	.cancel_protection = .unblocked,
1589	1712	.futex_waiter = undefined,
1590	1713	.unpark_flag = unpark_flag_init,
1591		.csprng = .{},
	1714	.csprng = .uninitialized,
1592	1715	};
1593	1716	Thread.current = &thread;
1594	1717
...	...	@@ -2563,12 +2686,12 @@ fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Oper
2563	2686	fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2564	2687	const t: *Threaded = @ptrCast(@alignCast(userdata));
2565	2688	if (is_windows) {
2566		batchAwaitWindows(b, false) catch \|err\| switch (err) {
	2689	batchDrainSubmittedWindows(b, false) catch \|err\| switch (err) {
2567	2690	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2568	2691	else => \|e\| return e,
2569	2692	};
2570	2693	const alertable_syscall = try AlertableSyscall.start();
2571		while (b.pending.head != .none and b.completions.head == .none) waitForApcOrAlert();
	2694	while (b.pending.head != .none and b.completed.head == .none) waitForApcOrAlert();
2572	2695	alertable_syscall.finish();
2573	2696	return;
2574	2697	}
...	...	@@ -2576,7 +2699,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2576	2699	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2577	2700	var poll_len: u32 = 0;
2578	2701	{
2579		var index = b.submissions.head;
	2702	var index = b.submitted.head;
2580	2703	while (index != .none and poll_len < poll_buffer_len) {
2581	2704	const submission = &b.storage[index.toIndex()].submission;
2582	2705	switch (submission.operation) {
...	...	@@ -2605,7 +2728,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2605	2728	1 => {},
2606	2729	else => while (true) {
2607	2730	const timeout_ms: i32 = t: {
2608		if (b.completions.head != .none) {
	2731	if (b.completed.head != .none) {
2609	2732	// It is legal to call batchWait with already completed
2610	2733	// operations in the ring. In such case, we need to avoid
2611	2734	// blocking in the poll syscall, but we can still take this
...	...	@@ -2620,7 +2743,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2620	2743	switch (posix.errno(rc)) {
2621	2744	.SUCCESS => {
2622	2745	if (rc == 0) {
2623		if (b.completions.head != .none) {
	2746	if (b.completed.head != .none) {
2624	2747	// Since there are already completions available in the
2625	2748	// queue, this is neither a timeout nor a case for
2626	2749	// retrying.
...	...	@@ -2629,7 +2752,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2629	2752	continue;
2630	2753	}
2631	2754	var prev_index: Io.Operation.OptionalIndex = .none;
2632		var index = b.submissions.head;
	2755	var index = b.submitted.head;
2633	2756	for (poll_buffer[0..poll_len]) \|poll_entry\| {
2634	2757	const storage = &b.storage[index.toIndex()];
2635	2758	const submission = &storage.submission;
...	...	@@ -2638,17 +2761,17 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2638	2761	const result = try operate(t, submission.operation);
2639	2762
2640	2763	switch (prev_index) {
2641		.none => b.submissions.head = next_index,
	2764	.none => b.submitted.head = next_index,
2642	2765	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2643	2766	}
2644		if (next_index == .none) b.submissions.tail = prev_index;
	2767	if (next_index == .none) b.submitted.tail = prev_index;
2645	2768
2646		switch (b.completions.tail) {
2647		.none => b.completions.head = index,
	2769	switch (b.completed.tail) {
	2770	.none => b.completed.head = index,
2648	2771	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2649	2772	}
2650	2773	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2651		b.completions.tail = index;
	2774	b.completed.tail = index;
2652	2775	} else prev_index = index;
2653	2776	index = next_index;
2654	2777	}
...	...	@@ -2662,10 +2785,10 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2662	2785	}
2663	2786	}
2664	2787
2665		var tail_index = b.completions.tail;
2666		defer b.completions.tail = tail_index;
2667		var index = b.submissions.head;
2668		errdefer b.submissions.head = index;
	2788	var tail_index = b.completed.tail;
	2789	defer b.completed.tail = tail_index;
	2790	var index = b.submitted.head;
	2791	errdefer b.submitted.head = index;
2669	2792	while (index != .none) {
2670	2793	const storage = &b.storage[index.toIndex()];
2671	2794	const submission = &storage.submission;
...	...	@@ -2673,22 +2796,22 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2673	2796	const result = try operate(t, submission.operation);
2674	2797
2675	2798	switch (tail_index) {
2676		.none => b.completions.head = index,
	2799	.none => b.completed.head = index,
2677	2800	else => b.storage[tail_index.toIndex()].completion.node.next = index,
2678	2801	}
2679	2802	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2680	2803	tail_index = index;
2681	2804	index = next_index;
2682	2805	}
2683		b.submissions = .{ .head = .none, .tail = .none };
	2806	b.submitted = .{ .head = .none, .tail = .none };
2684	2807	}
2685	2808
2686	2809	fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {
2687	2810	const t: *Threaded = @ptrCast(@alignCast(userdata));
2688	2811	if (is_windows) {
2689	2812	const deadline: ?Io.Clock.Timestamp = timeout.toTimestamp(ioBasic(t));
2690		try batchAwaitWindows(b, true);
2691		while (b.pending.head != .none and b.completions.head == .none) {
	2813	try batchDrainSubmittedWindows(b, true);
	2814	while (b.pending.head != .none and b.completed.head == .none) {
2692	2815	var delay_interval: windows.LARGE_INTEGER = interval: {
2693	2816	const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);
2694	2817	break :interval timeoutToWindowsInterval(.{ .deadline = d }).?;
...	...	@@ -2701,7 +2824,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2701	2824	// The thread woke due to the timeout. Although spurious
2702	2825	// timeouts are OK, when no deadline is passed we must not
2703	2826	// return `error.Timeout`.
2704		if (timeout != .none and b.completions.head == .none) return error.Timeout;
	2827	if (timeout != .none and b.completed.head == .none) return error.Timeout;
2705	2828	},
2706	2829	else => {},
2707	2830	}
...	...	@@ -2743,7 +2866,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2743	2866	}
2744	2867	} = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };
2745	2868	{
2746		var index = b.submissions.head;
	2869	var index = b.submitted.head;
2747	2870	while (index != .none) {
2748	2871	const submission = &b.storage[index.toIndex()].submission;
2749	2872	switch (submission.operation) {
...	...	@@ -2757,18 +2880,18 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2757	2880	switch (poll_storage.len) {
2758	2881	0 => return,
2759	2882	1 => if (timeout == .none) {
2760		const index = b.submissions.head;
	2883	const index = b.submitted.head;
2761	2884	const storage = &b.storage[index.toIndex()];
2762	2885	const result = try operate(t, storage.submission.operation);
2763	2886
2764		b.submissions = .{ .head = .none, .tail = .none };
	2887	b.submitted = .{ .head = .none, .tail = .none };
2765	2888
2766		switch (b.completions.tail) {
2767		.none => b.completions.head = index,
	2889	switch (b.completed.tail) {
	2890	.none => b.completed.head = index,
2768	2891	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2769	2892	}
2770	2893	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2771		b.completions.tail = index;
	2894	b.completed.tail = index;
2772	2895	return;
2773	2896	},
2774	2897	else => {},
...	...	@@ -2777,7 +2900,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2777	2900	const deadline = timeout.toTimestamp(t_io);
2778	2901	while (true) {
2779	2902	const timeout_ms: i32 = t: {
2780		if (b.completions.head != .none) {
	2903	if (b.completed.head != .none) {
2781	2904	// It is legal to call batchWait with already completed
2782	2905	// operations in the ring. In such case, we need to avoid
2783	2906	// blocking in the poll syscall, but we can still take this
...	...	@@ -2794,7 +2917,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2794	2917	switch (posix.errno(rc)) {
2795	2918	.SUCCESS => {
2796	2919	if (rc == 0) {
2797		if (b.completions.head != .none) {
	2920	if (b.completed.head != .none) {
2798	2921	// Since there are already completions available in the
2799	2922	// queue, this is neither a timeout nor a case for
2800	2923	// retrying.
...	...	@@ -2806,7 +2929,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2806	2929	return error.Timeout;
2807	2930	}
2808	2931	var prev_index: Io.Operation.OptionalIndex = .none;
2809		var index = b.submissions.head;
	2932	var index = b.submitted.head;
2810	2933	for (poll_storage.slice[0..poll_storage.len]) \|poll_entry\| {
2811	2934	const submission = &b.storage[index.toIndex()].submission;
2812	2935	const next_index = submission.node.next;
...	...	@@ -2814,17 +2937,20 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2814	2937	const result = try operate(t, submission.operation);
2815	2938
2816	2939	switch (prev_index) {
2817		.none => b.submissions.head = next_index,
	2940	.none => b.submitted.head = next_index,
2818	2941	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2819	2942	}
2820		if (next_index == .none) b.submissions.tail = prev_index;
	2943	if (next_index == .none) b.submitted.tail = prev_index;
2821	2944
2822		switch (b.completions.tail) {
2823		.none => b.completions.head = index,
	2945	switch (b.completed.tail) {
	2946	.none => b.completed.head = index,
2824	2947	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2825	2948	}
2826		b.completions.tail = index;
2827		b.storage[index.toIndex()] = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2949	b.completed.tail = index;
	2950	b.storage[index.toIndex()] = .{ .completion = .{
	2951	.node = .{ .next = .none },
	2952	.result = result,
	2953	} };
2828	2954	} else prev_index = index;
2829	2955	index = next_index;
2830	2956	}
...	...	@@ -2841,7 +2967,7 @@ const WindowsBatchPendingOperationContext = extern struct {
2841	2967	file: windows.HANDLE,
2842	2968	iosb: windows.IO_STATUS_BLOCK,
2843	2969
2844		const Erased = [3]usize;
	2970	const Erased = Io.Operation.Storage.Pending.Context;
2845	2971
2846	2972	comptime {
2847	2973	assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));
...	...	@@ -2858,24 +2984,9 @@ const WindowsBatchPendingOperationContext = extern struct {
2858	2984
2859	2985	fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2860	2986	const t: *Threaded = @ptrCast(@alignCast(userdata));
2861		{
2862		var tail_index = b.unused.tail;
2863		defer b.unused.tail = tail_index;
2864		var index = b.submissions.head;
2865		errdefer b.submissions.head = index;
2866		while (index != .none) {
2867		const next_index = b.storage[index.toIndex()].submission.node.next;
2868		switch (tail_index) {
2869		.none => b.unused.head = index,
2870		else => b.storage[tail_index.toIndex()].unused.next = index,
2871		}
2872		b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
2873		tail_index = index;
2874		index = next_index;
2875		}
2876		b.submissions = .{ .head = .none, .tail = .none };
2877		}
2878	2987	if (is_windows) {
	2988	if (b.pending.head == .none) return;
	2989	waitForApcOrAlert();
2879	2990	var index = b.pending.head;
2880	2991	while (index != .none) {
2881	2992	const pending = &b.storage[index.toIndex()].pending;
...	...	@@ -2889,10 +3000,13 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2889	3000	t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);
2890	3001	b.context = null;
2891	3002	}
2892		assert(b.pending.head == .none);
2893	3003	}
2894	3004
2895		fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows.ULONG) callconv(.winapi) void {
	3005	fn batchApc(
	3006	apc_context: ?*anyopaque,
	3007	iosb: *windows.IO_STATUS_BLOCK,
	3008	_: windows.ULONG,
	3009	) callconv(.winapi) void {
2896	3010	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));
2897	3011	const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);
2898	3012	const erased_context = context.toErased();
...	...	@@ -2918,11 +3032,12 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows
2918	3032	b.unused.tail = .fromIndex(index);
2919	3033	},
2920	3034	else => {
2921		switch (b.completions.tail) {
2922		.none => b.completions.head = .fromIndex(index),
2923		else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = .fromIndex(index),
	3035	switch (b.completed.tail) {
	3036	.none => b.completed.head = .fromIndex(index),
	3037	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next =
	3038	.fromIndex(index),
2924	3039	}
2925		b.completions.tail = .fromIndex(index);
	3040	b.completed.tail = .fromIndex(index);
2926	3041	const result: Io.Operation.Result = switch (pending.tag) {
2927	3042	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
2928	3043	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },
...	...	@@ -2934,9 +3049,9 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows
2934	3049	}
2935	3050
2936	3051	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2937		fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, ConcurrencyUnavailable }!void {
2938		var index = b.submissions.head;
2939		errdefer b.submissions.head = index;
	3052	fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentError \|\| Io.Cancelable)!void {
	3053	var index = b.submitted.head;
	3054	errdefer b.submitted.head = index;
2940	3055	while (index != .none) {
2941	3056	const storage = &b.storage[index.toIndex()];
2942	3057	const submission = storage.submission;
...	...	@@ -2952,7 +3067,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
2952	3067	b.pending.tail = index;
2953	3068	const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);
2954	3069	errdefer {
2955		context.iosb.u.Status = .CANCELLED;
	3070	context.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };
2956	3071	batchApc(b, &context.iosb, 0);
2957	3072	}
2958	3073	switch (submission.operation) {
...	...	@@ -2960,10 +3075,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
2960	3075	var data_index: usize = 0;
2961	3076	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;
2962	3077	if (o.data.len - data_index == 0) {
2963		context.iosb = .{
2964		.u = .{ .Status = .SUCCESS },
2965		.Information = 0,
2966		};
	3078	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
2967	3079	batchApc(b, &context.iosb, 0);
2968	3080	break :o;
2969	3081	}
...	...	@@ -3023,10 +3135,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
3023	3135	.file_write_streaming => \|o\| o: {
3024	3136	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);
3025	3137	if (buffer.len == 0) {
3026		context.iosb = .{
3027		.u = .{ .Status = .SUCCESS },
3028		.Information = 0,
3029		};
	3138	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3030	3139	batchApc(b, &context.iosb, 0);
3031	3140	break :o;
3032	3141	}
...	...	@@ -3140,7 +3249,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
3140	3249	}
3141	3250	index = submission.node.next;
3142	3251	}
3143		b.submissions = .{ .head = .none, .tail = .none };
	3252	b.submitted = .{ .head = .none, .tail = .none };
3144	3253	}
3145	3254
3146	3255	/// Since Windows only supports writing one contiguous buffer, returns the
...	...	@@ -3155,7 +3264,7 @@ fn windowsWriteBuffer(header: []const u8, data: []const []const u8, splat: usize
3155	3264	if (splat == 0) return &.{};
3156	3265	break :b data[data.len - 1];
3157	3266	};
3158		return buffer[0..@min(buffer.len, std.math.maxInt(u32))];
	3267	return buffer[0..std.math.lossyCast(u32, buffer.len)];
3159	3268	}
3160	3269
3161	3270	fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {
...	...	@@ -4677,8 +4786,8 @@ fn atomicFileInit(
4677	4786	dir: Dir,
4678	4787	close_dir_on_deinit: bool,
4679	4788	) Dir.CreateFileAtomicError!File.Atomic {
4680		var random_integer: u64 = undefined;
4681	4789	while (true) {
	4790	var random_integer: u64 = undefined;
4682	4791	t_io.random(@ptrCast(&random_integer));
4683	4792	const tmp_sub_path = std.fmt.hex(random_integer);
4684	4793	const file = dir.createFile(t_io, &tmp_sub_path, .{
...	...	@@ -14317,11 +14426,11 @@ pub fn posixProtocol(protocol: ?net.Protocol) u32 {
14317	14426	return @intFromEnum(protocol orelse return 0);
14318	14427	}
14319	14428
14320		fn recoverableOsBugDetected() void {
	14429	pub fn recoverableOsBugDetected() void {
14321	14430	if (is_debug) unreachable;
14322	14431	}
14323	14432
14324		fn clockToPosix(clock: Io.Clock) posix.clockid_t {
	14433	pub fn clockToPosix(clock: Io.Clock) posix.clockid_t {
14325	14434	return switch (clock) {
14326	14435	.real => posix.CLOCK.REALTIME,
14327	14436	.awake => switch (native_os) {
...	...	@@ -14355,7 +14464,7 @@ fn clockToWasi(clock: Io.Clock) std.os.wasi.clockid_t {
14355	14464	};
14356	14465	}
14357	14466
14358		const linux_statx_request: std.os.linux.STATX = .{
	14467	pub const linux_statx_request: std.os.linux.STATX = .{
14359	14468	.TYPE = true,
14360	14469	.MODE = true,
14361	14470	.ATIME = true,
...	...	@@ -14367,7 +14476,7 @@ const linux_statx_request: std.os.linux.STATX = .{
14367	14476	.BLOCKS = true,
14368	14477	};
14369	14478
14370		const linux_statx_check: std.os.linux.STATX = .{
	14479	pub const linux_statx_check: std.os.linux.STATX = .{
14371	14480	.TYPE = true,
14372	14481	.MODE = true,
14373	14482	.ATIME = false,
...	...	@@ -14379,7 +14488,7 @@ const linux_statx_check: std.os.linux.STATX = .{
14379	14488	.BLOCKS = false,
14380	14489	};
14381	14490
14382		fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat {
	14491	pub fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat {
14383	14492	const actual_mask_int: u32 = @bitCast(stx.mask);
14384	14493	const wanted_mask_int: u32 = @bitCast(linux_statx_check);
14385	14494	if ((actual_mask_int \| wanted_mask_int) != actual_mask_int) return error.Unexpected;
...	...	@@ -14470,11 +14579,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) File.Stat {
14470	14579	};
14471	14580	}
14472	14581
14473		fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {
	14582	pub fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {
14474	14583	return .{ .nanoseconds = nanosecondsFromPosix(timespec) };
14475	14584	}
14476	14585
14477		fn nanosecondsFromPosix(timespec: *const posix.timespec) i96 {
	14586	pub fn nanosecondsFromPosix(timespec: *const posix.timespec) i96 {
14478	14587	return @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);
14479	14588	}
14480	14589
...	...	@@ -14492,7 +14601,7 @@ fn timestampToPosix(nanoseconds: i96) posix.timespec {
14492	14601	};
14493	14602	}
14494	14603
14495		fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
	14604	pub fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
14496	14605	return switch (set_ts) {
14497	14606	.unchanged => .OMIT,
14498	14607	.now => .NOW,
...	...	@@ -14500,7 +14609,7 @@ fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
14500	14609	};
14501	14610	}
14502	14611
14503		fn pathToPosix(file_path: []const u8, buffer: *[posix.PATH_MAX]u8) Dir.PathNameError![:0]u8 {
	14612	pub fn pathToPosix(file_path: []const u8, buffer: *[posix.PATH_MAX]u8) Dir.PathNameError![:0]u8 {
14504	14613	if (std.mem.containsAtLeastScalar2(u8, file_path, 0, 1)) return error.BadPathName;
14505	14614	// >= rather than > to make room for the null byte
14506	14615	if (file_path.len >= buffer.len) return error.NameTooLong;
...	...	@@ -14996,126 +15105,7 @@ const WindowsEnvironStrings = struct {
14996	15105	fn scanEnviron(t: *Threaded) void {
14997	15106	mutexLock(&t.mutex);
14998	15107	defer mutexUnlock(&t.mutex);
14999
15000		if (t.environ.initialized) return;
15001		t.environ.initialized = true;
15002
15003		if (is_windows) {
15004		// This value expires with any call that modifies the environment,
15005		// which is outside of this Io implementation's control, so references
15006		// must be short-lived.
15007		const peb = windows.peb();
15008		assert(windows.ntdll.RtlEnterCriticalSection(peb.FastPebLock) == .SUCCESS);
15009		defer assert(windows.ntdll.RtlLeaveCriticalSection(peb.FastPebLock) == .SUCCESS);
15010		const ptr = peb.ProcessParameters.Environment;
15011
15012		var i: usize = 0;
15013		while (ptr[i] != 0) {
15014
15015		// There are some special environment variables that start with =,
15016		// so we need a special case to not treat = as a key/value separator
15017		// if it's the first character.
15018		// https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
15019		const key_start = i;
15020		if (ptr[i] == '=') i += 1;
15021		while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
15022		const key_w = ptr[key_start..i];
15023
15024		const value_start = i + 1;
15025		while (ptr[i] != 0) : (i += 1) {} // skip over '=' and value
15026		const value_w = ptr[value_start..i];
15027		i += 1; // skip over null byte
15028
15029		if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'N', 'O', '_', 'C', 'O', 'L', 'O', 'R' })) {
15030		t.environ.exist.NO_COLOR = true;
15031		} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'C', 'L', 'I', 'C', 'O', 'L', 'O', 'R', '_', 'F', 'O', 'R', 'C', 'E' })) {
15032		t.environ.exist.CLICOLOR_FORCE = true;
15033		} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'Z', 'I', 'G', '_', 'P', 'R', 'O', 'G', 'R', 'E', 'S', 'S' })) {
15034		t.environ.zig_progress_file = file: {
15035		var value_buf: [std.fmt.count("{d}", .{std.math.maxInt(usize)})]u8 = undefined;
15036		const len = std.unicode.calcWtf8Len(value_w);
15037		if (len > value_buf.len) break :file error.UnrecognizedFormat;
15038		assert(std.unicode.wtf16LeToWtf8(&value_buf, value_w) == len);
15039		break :file .{
15040		.handle = @ptrFromInt(std.fmt.parseInt(usize, value_buf[0..len], 10) catch
15041		break :file error.UnrecognizedFormat),
15042		.flags = .{ .nonblocking = true },
15043		};
15044		};
15045		}
15046		comptime assert(@sizeOf(Environ.String) == 0);
15047		}
15048		} else if (native_os == .wasi and !builtin.link_libc) {
15049		var environ_count: usize = undefined;
15050		var environ_buf_size: usize = undefined;
15051
15052		switch (std.os.wasi.environ_sizes_get(&environ_count, &environ_buf_size)) {
15053		.SUCCESS => {},
15054		else => \|err\| {
15055		t.environ.err = posix.unexpectedErrno(err);
15056		return;
15057		},
15058		}
15059		if (environ_count == 0) return;
15060
15061		const environ = t.allocator.alloc([*:0]u8, environ_count) catch \|err\| {
15062		t.environ.err = err;
15063		return;
15064		};
15065		defer t.allocator.free(environ);
15066		const environ_buf = t.allocator.alloc(u8, environ_buf_size) catch \|err\| {
15067		t.environ.err = err;
15068		return;
15069		};
15070		defer t.allocator.free(environ_buf);
15071
15072		switch (std.os.wasi.environ_get(environ.ptr, environ_buf.ptr)) {
15073		.SUCCESS => {},
15074		else => \|err\| {
15075		t.environ.err = posix.unexpectedErrno(err);
15076		return;
15077		},
15078		}
15079
15080		for (environ) \|env\| {
15081		const pair = std.mem.sliceTo(env, 0);
15082		var parts = std.mem.splitScalar(u8, pair, '=');
15083		const key = parts.first();
15084		if (std.mem.eql(u8, key, "NO_COLOR")) {
15085		t.environ.exist.NO_COLOR = true;
15086		} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
15087		t.environ.exist.CLICOLOR_FORCE = true;
15088		}
15089		comptime assert(@sizeOf(Environ.String) == 0);
15090		}
15091		} else {
15092		for (t.environ.process_environ.block.slice) \|opt_entry\| {
15093		const entry = opt_entry.?;
15094		var entry_i: usize = 0;
15095		while (entry[entry_i] != 0 and entry[entry_i] != '=') : (entry_i += 1) {}
15096		const key = entry[0..entry_i];
15097
15098		var end_i: usize = entry_i;
15099		while (entry[end_i] != 0) : (end_i += 1) {}
15100		const value = entry[entry_i + 1 .. end_i :0];
15101
15102		if (std.mem.eql(u8, key, "NO_COLOR")) {
15103		t.environ.exist.NO_COLOR = true;
15104		} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
15105		t.environ.exist.CLICOLOR_FORCE = true;
15106		} else if (std.mem.eql(u8, key, "ZIG_PROGRESS")) {
15107		t.environ.zig_progress_file = file: {
15108		break :file .{
15109		.handle = std.fmt.parseInt(u31, value, 10) catch
15110		break :file error.UnrecognizedFormat,
15111		.flags = .{ .nonblocking = true },
15112		};
15113		};
15114		} else inline for (@typeInfo(Environ.String).@"struct".fields) \|field\| {
15115		if (std.mem.eql(u8, key, field.name)) @field(t.environ.string, field.name) = value;
15116		}
15117		}
15118		}
	15108	t.environ.scan(t.allocator);
15119	15109	}
15120	15110
15121	15111	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
...	...	@@ -15213,17 +15203,17 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15213	15203	const any_ignore = (options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore);
15214	15204	const dev_null_fd = if (any_ignore) try getDevNullFd(t) else undefined;
15215	15205
15216		const prog_pipe: [2]posix.fd_t = if (options.progress_node.index != .none)
	15206	const prog_pipe: [2]posix.fd_t = if (options.progress_node.index != .none) pipe: {
15217	15207	// We use CLOEXEC for the same reason as in `pipe_flags`.
15218		try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true })
15219		else
15220		.{ -1, -1 };
	15208	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
	15209	switch (native_os) {
	15210	.linux => _ = posix.system.fcntl(pipe[0], posix.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2)),
	15211	else => {},
	15212	}
	15213	break :pipe pipe;
	15214	} else .{ -1, -1 };
15221	15215	errdefer destroyPipe(prog_pipe);
15222	15216
15223		if (native_os == .linux and prog_pipe[0] != -1) {
15224		_ = posix.system.fcntl(prog_pipe[0], posix.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
15225		}
15226
15227	15217	var arena_allocator = std.heap.ArenaAllocator.init(t.allocator);
15228	15218	defer arena_allocator.deinit();
15229	15219	const arena = arena_allocator.allocator();
...	...	@@ -17241,16 +17231,7 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {
17241	17231
17242	17232	randomSecure(t, &seed) catch \|err\| switch (err) {
17243	17233	error.Canceled => unreachable,
17244		error.EntropyUnavailable => {
17245		@memset(&seed, 0);
17246		const aslr_addr = @intFromPtr(t);
17247		std.mem.writeInt(usize, seed[seed.len - @sizeOf(usize) ..][0..@sizeOf(usize)], aslr_addr, .native);
17248		switch (native_os) {
17249		.windows => fallbackSeedWindows(&seed),
17250		.wasi => if (builtin.link_libc) fallbackSeedPosix(&seed) else fallbackSeedWasi(&seed),
17251		else => fallbackSeedPosix(&seed),
17252		}
17253		},
	17234	error.EntropyUnavailable => fallbackSeed(t, &seed),
17254	17235	};
17255	17236	}
17256	17237	t.csprng.rng = .init(seed);
...	...	@@ -17259,6 +17240,17 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {
17259	17240	t.csprng.rng.fill(buffer);
17260	17241	}
17261	17242
	17243	pub fn fallbackSeed(aslr_addr: ?anyopaque, seed: [Csprng.seed_len]u8) void {
	17244	@memset(seed, 0);
	17245	std.mem.writeInt(usize, seed[seed.len - @sizeOf(usize) ..][0..@sizeOf(usize)], @intFromPtr(aslr_addr), .native);
	17246	const fallbackSeedImpl = switch (native_os) {
	17247	.windows => fallbackSeedWindows,
	17248	.wasi => if (builtin.link_libc) fallbackSeedPosix else fallbackSeedWasi,
	17249	else => fallbackSeedPosix,
	17250	};
	17251	fallbackSeedImpl(seed);
	17252	}
	17253
17262	17254	fn fallbackSeedPosix(seed: *[Csprng.seed_len]u8) void {
17263	17255	std.mem.writeInt(posix.pid_t, seed[0..@sizeOf(posix.pid_t)], posix.system.getpid(), .native);
17264	17256	const i_1 = @sizeOf(posix.pid_t);

lib/std/os/linux.zig+6-3

...	...	@@ -6717,9 +6717,10 @@ pub const IORING_ACCEPT_MULTISHOT = 1 << 0;
6717	6717	/// IORING_OP_MSG_RING command types, stored in sqe->addr
6718	6718	pub const IORING_MSG_RING_COMMAND = enum(u8) {
6719	6719	/// pass sqe->len as 'res' and off as user_data
6720		DATA,
	6720	DATA = 0,
6721	6721	/// send a registered fd to another ring
6722		SEND_FD,
	6722	SEND_FD = 1,
	6723	_,
6723	6724	};
6724	6725
6725	6726	// io_uring_sqe.msg_ring_flags (rw_flags in the Zig struct)
...	...	@@ -6772,6 +6773,8 @@ pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
6772	6773	pub const IORING_CQE_F_NOTIF = 1 << 3;
6773	6774	/// If set, the buffer ID set in the completion will get more completions.
6774	6775	pub const IORING_CQE_F_BUF_MORE = 1 << 4;
	6776	pub const IORING_CQE_F_SKIP = 1 << 5;
	6777	pub const IORING_CQE_F_32 = 1 << 15;
6775	6778
6776	6779	pub const IORING_CQE_BUFFER_SHIFT = 16;
6777	6780
...	...	@@ -7068,7 +7071,7 @@ pub const IORING_RESTRICTION = enum(u16) {
7068	7071	_,
7069	7072	};
7070	7073
7071		pub const IO_URING_SOCKET_OP = enum(u16) {
	7074	pub const IO_URING_SOCKET_OP = enum(u32) {
7072	7075	SIOCIN = 0,
7073	7076	SIOCOUTQ = 1,
7074	7077	GETSOCKOPT = 2,

lib/std/process.zig+3-3

...	...	@@ -60,7 +60,7 @@ pub const CurrentPathError = error{
60	60	NameTooLong,
61	61	/// Not possible on Windows. Always returned on WASI.
62	62	CurrentDirUnlinked,
63		} \|\| Io.UnexpectedError;
	63	} \|\| Io.Cancelable \|\| Io.UnexpectedError;
64	64
65	65	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).
66	66	/// On other platforms, the result is an opaque sequence of bytes with no
...	...	@@ -72,7 +72,7 @@ pub fn currentPath(io: Io, buffer: []u8) CurrentPathError!usize {
72	72	pub const CurrentPathAllocError = Allocator.Error \|\| error{
73	73	/// Not possible on Windows. Always returned on WASI.
74	74	CurrentDirUnlinked,
75		} \|\| Io.UnexpectedError;
	75	} \|\| Io.Cancelable \|\| Io.UnexpectedError;
76	76
77	77	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).
78	78	/// On other platforms, the result is an opaque sequence of bytes with no
...	...	@@ -355,7 +355,7 @@ pub const SpawnError = error{
355	355	/// On Windows, the volume does not contain a recognized file system. File
356	356	/// system drivers might not be loaded, or the volume may be corrupt.
357	357	UnrecognizedVolume,
358		} \|\| Io.Dir.PathNameError \|\| Io.Cancelable \|\| Io.UnexpectedError;
	358	} \|\| Io.File.OpenError \|\| Io.Dir.PathNameError \|\| Io.Cancelable \|\| Io.UnexpectedError;
359	359
360	360	pub const SpawnOptions = struct {
361	361	argv: []const []const u8,

lib/std/tar.zig+4-4

...	...	@@ -1128,10 +1128,10 @@ fn filePermissions(mode: u32, options: PipeOptions) Io.File.Permissions {
1128	1128
1129	1129	test filePermissions {
1130	1130	if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;
1131		try testing.expectEqual(.default_file, filePermissions(0o744, .{ .mode_mode = .ignore }));
1132		try testing.expectEqual(.executable_file, filePermissions(0o744, .{}));
1133		try testing.expectEqual(.default_file, filePermissions(0o644, .{}));
1134		try testing.expectEqual(.default_file, filePermissions(0o655, .{}));
	1131	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o744, .{ .mode_mode = .ignore }));
	1132	try testing.expectEqual(Io.File.Permissions.executable_file, filePermissions(0o744, .{}));
	1133	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o644, .{}));
	1134	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o655, .{}));
1135	1135	}
1136	1136
1137	1137	test "executable bit" {

src/Compilation.zig+13-33

...	...	@@ -4891,11 +4891,7 @@ fn performAllTheWork(
4891	4891
4892	4892	work: while (true) {
4893	4893	for (&comp.work_queues) \|*work_queue\| if (work_queue.popFront()) \|job\| {
4894		try processOneJob(
4895		@intFromEnum(Zcu.PerThread.Id.main),
4896		comp,
4897		job,
4898		);
	4894	try processOneJob(.main, comp, job);
4899	4895	continue :work;
4900	4896	};
4901	4897	if (comp.zcu) \|zcu\| {
...	...	@@ -5160,11 +5156,7 @@ pub fn queueJobs(comp: *Compilation, jobs: []const Job) !void {
5160	5156	for (jobs) \|job\| try comp.queueJob(job);
5161	5157	}
5162	5158
5163		fn processOneJob(
5164		tid: usize,
5165		comp: *Compilation,
5166		job: Job,
5167		) JobError!void {
	5159	fn processOneJob(tid: Zcu.PerThread.Id, comp: *Compilation, job: Job) JobError!void {
5168	5160	switch (job) {
5169	5161	.codegen_func => \|func\| {
5170	5162	const zcu = comp.zcu.?;
...	...	@@ -5232,7 +5224,7 @@ fn processOneJob(
5232	5224	const named_frame = tracy.namedFrame("analyze_func");
5233	5225	defer named_frame.end();
5234	5226
5235		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5227	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5236	5228	defer pt.deactivate();
5237	5229
5238	5230	pt.ensureFuncBodyUpToDate(func) catch \|err\| switch (err) {
...	...	@@ -5245,7 +5237,7 @@ fn processOneJob(
5245	5237	const named_frame = tracy.namedFrame("analyze_comptime_unit");
5246	5238	defer named_frame.end();
5247	5239
5248		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5240	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5249	5241	defer pt.deactivate();
5250	5242
5251	5243	const maybe_err: Zcu.SemaError!void = switch (unit.unwrap()) {
...	...	@@ -5285,7 +5277,7 @@ fn processOneJob(
5285	5277	const named_frame = tracy.namedFrame("resolve_type_fully");
5286	5278	defer named_frame.end();
5287	5279
5288		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5280	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5289	5281	defer pt.deactivate();
5290	5282	Type.fromInterned(ty).resolveFully(pt) catch \|err\| switch (err) {
5291	5283	error.OutOfMemory, error.Canceled => \|e\| return e,
...	...	@@ -5296,7 +5288,7 @@ fn processOneJob(
5296	5288	const named_frame = tracy.namedFrame("analyze_mod");
5297	5289	defer named_frame.end();
5298	5290
5299		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5291	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5300	5292	defer pt.deactivate();
5301	5293	pt.semaMod(mod) catch \|err\| switch (err) {
5302	5294	error.OutOfMemory, error.Canceled => \|e\| return e,
...	...	@@ -5642,13 +5634,14 @@ fn workerUpdateFile(
5642	5634	prog_node: std.Progress.Node,
5643	5635	group: *Io.Group,
5644	5636	) void {
5645		const tid = Compilation.getTid();
5646	5637	const io = comp.io;
	5638	const tid: Zcu.PerThread.Id = .acquire(io);
	5639	defer tid.release(io);
5647	5640
5648	5641	const child_prog_node = prog_node.start(fs.path.basename(file.path.sub_path), 0);
5649	5642	defer child_prog_node.end();
5650	5643
5651		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5644	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5652	5645	defer pt.deactivate();
5653	5646	pt.updateFile(file_index, file) catch \|err\| {
5654	5647	pt.reportRetryableFileError(file_index, "unable to load '{s}': {s}", .{ fs.path.basename(file.path.sub_path), @errorName(err) }) catch \|oom\| switch (oom) {
...	...	@@ -5708,9 +5701,10 @@ fn workerUpdateBuiltinFile(comp: Compilation, file: Zcu.File) void {
5708	5701	}
5709	5702
5710	5703	fn workerUpdateEmbedFile(comp: Compilation, ef_index: Zcu.EmbedFile.Index, ef: Zcu.EmbedFile) void {
5711		const tid = Compilation.getTid();
5712	5704	const io = comp.io;
5713		comp.detectEmbedFileUpdate(@enumFromInt(tid), ef_index, ef) catch \|err\| switch (err) {
	5705	const tid: Zcu.PerThread.Id = .acquire(io);
	5706	defer tid.release(io);
	5707	comp.detectEmbedFileUpdate(tid, ef_index, ef) catch \|err\| switch (err) {
5714	5708	error.OutOfMemory => {
5715	5709	comp.mutex.lockUncancelable(io);
5716	5710	defer comp.mutex.unlock(io);
...	...	@@ -5868,7 +5862,7 @@ pub fn translateC(
5868	5862	}
5869	5863
5870	5864	var stdout: []u8 = undefined;
5871		try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, &stdout);
	5865	try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, comp.thread_limit, &stdout);
5872	5866
5873	5867	if (out_dep_path) \|dep_file_path\| add_deps: {
5874	5868	if (comp.verbose_cimport) log.info("processing dep file at {s}", .{dep_file_path});
...	...	@@ -8394,17 +8388,3 @@ pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
8394	8388	pub fn compilerRtStrip(comp: Compilation) bool {
8395	8389	return comp.root_mod.strip;
8396	8390	}
8397
8398		/// This is a temporary workaround put in place to migrate from `std.Thread.Pool`
8399		/// to `std.Io.Threaded` for asynchronous/concurrent work. The eventual solution
8400		/// will likely involve significant changes to the `InternPool` implementation.
8401		pub fn getTid() usize {
8402		if (my_tid == null) my_tid = next_tid.fetchAdd(1, .monotonic);
8403		return my_tid.?;
8404		}
8405		pub fn setMainThread() void {
8406		my_tid = 0;
8407		}
8408		/// TID 0 is reserved for the main thread.
8409		var next_tid: std.atomic.Value(usize) = .init(1);
8410		threadlocal var my_tid: ?usize = null;

src/InternPool.zig+6-7

...	...	@@ -3,6 +3,7 @@
3	3	const InternPool = @This();
4	4
5	5	const builtin = @import("builtin");
	6	const build_options = @import("build_options");
6	7
7	8	const std = @import("std");
8	9	const Io = std.Io;
...	...	@@ -86,13 +87,11 @@ dep_entries: std.ArrayList(DepEntry),
86	87	/// garbage collection pass.
87	88	free_dep_entries: std.ArrayList(DepEntry.Index),
88	89
89		/// Whether a multi-threaded intern pool is useful.
90		/// Currently `false` until the intern pool is actually accessed
91		/// from multiple threads to reduce the cost of this data structure.
92		const want_multi_threaded = true;
93
94	90	/// Whether a single-threaded intern pool impl is in use.
95		pub const single_threaded = builtin.single_threaded or !want_multi_threaded;
	91	pub const single_threaded = switch (build_options.io_mode) {
	92	.threaded => builtin.single_threaded,
	93	.evented => false, // even without threads, evented can be access from multiple tasks at a time
	94	};
96	95
97	96	pub const empty: InternPool = .{
98	97	.locals = &.{},
...	...	@@ -6915,7 +6914,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, io: Io, available_threads: usize) !
6915	6914	assert(ip.locals.len == 0 and ip.shards.len == 0);
6916	6915	assert(available_threads > 0 and available_threads <= std.math.maxInt(u8));
6917	6916
6918		const used_threads = if (single_threaded) 1 else available_threads;
	6917	const used_threads = if (single_threaded) 1 else @max(available_threads, 2);
6919	6918	ip.locals = try gpa.alloc(Local, used_threads);
6920	6919	@memset(ip.locals, .{
6921	6920	.shared = .{

src/Zcu.zig+8-4

...	...	@@ -4954,8 +4954,10 @@ pub const CodegenTaskPool = struct {
4954	4954	// We own `air` now, so we are responsbile for freeing it.
4955	4955	var air = orig_air;
4956	4956	defer air.deinit(zcu.comp.gpa);
4957		const tid = Compilation.getTid();
4958		const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
	4957	const io = zcu.comp.io;
	4958	const tid: Zcu.PerThread.Id = .acquire(io);
	4959	defer tid.release(io);
	4960	const pt: Zcu.PerThread = .activate(zcu, tid);
4959	4961	defer pt.deactivate();
4960	4962	return pt.runCodegen(func_index, &air);
4961	4963	}
...	...	@@ -4964,8 +4966,10 @@ pub const CodegenTaskPool = struct {
4964	4966	func_index: InternPool.Index,
4965	4967	air: *Air,
4966	4968	) CodegenResult {
4967		const tid = Compilation.getTid();
4968		const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
	4969	const io = zcu.comp.io;
	4970	const tid: Zcu.PerThread.Id = .acquire(io);
	4971	defer tid.release(io);
	4972	const pt: Zcu.PerThread = .activate(zcu, tid);
4969	4973	defer pt.deactivate();
4970	4974	return pt.runCodegen(func_index, air);
4971	4975	}

src/Zcu/PerThread.zig+75-2

...	...	@@ -41,13 +41,86 @@ zcu: *Zcu,
41	41	tid: Id,
42	42
43	43	pub const IdBacking = u7;
44		pub const Id = if (InternPool.single_threaded) enum { main } else enum(IdBacking) { main, _ };
	44	pub const Id = if (InternPool.single_threaded) enum {
	45	main,
	46
	47	pub fn allocate(arena: Allocator, n: usize) Allocator.Error!void {
	48	_ = arena;
	49	_ = n;
	50	}
	51	pub fn acquire(io: std.Io) Id {
	52	_ = io;
	53	return .main;
	54	}
	55	pub fn release(tid: Id, io: std.Io) void {
	56	_ = io;
	57	_ = tid;
	58	}
	59	} else enum(IdBacking) {
	60	main,
	61	_,
	62
	63	var tid_mutex: std.Io.Mutex = .init;
	64	var tid_cond: std.Io.Condition = .init;
	65	/// This is a temporary workaround put in place to migrate from `std.Thread.Pool`
	66	/// to `std.Io.Threaded` for asynchronous/concurrent work. The eventual solution
	67	/// will likely involve significant changes to the `InternPool` implementation.
	68	var available_tids: std.ArrayList(Id) = .empty;
	69	threadlocal var recursive_depth: usize = 0;
	70	threadlocal var recursive_tid: Id = .main;
	71
	72	pub fn allocate(arena: Allocator, n: usize) Allocator.Error!void {
	73	assert(available_tids.items.len == 0);
	74	try available_tids.ensureTotalCapacityPrecise(arena, n - 1);
	75	for (1..n) \|tid\| available_tids.appendAssumeCapacity(@enumFromInt(tid));
	76	}
	77	pub fn acquire(io: std.Io) Id {
	78	switch (build_options.io_mode) {
	79	.threaded => {
	80	recursive_depth += 1;
	81	if (recursive_depth > 1) {
	82	assert(recursive_tid != .main);
	83	return recursive_tid;
	84	}
	85	},
	86	.evented => {},
	87	}
	88	tid_mutex.lockUncancelable(io);
	89	defer tid_mutex.unlock(io);
	90	while (true) {
	91	if (available_tids.pop()) \|tid\| {
	92	switch (build_options.io_mode) {
	93	.threaded => recursive_tid = tid,
	94	.evented => {},
	95	}
	96	return tid;
	97	}
	98	tid_cond.waitUncancelable(io, &tid_mutex);
	99	}
	100	}
	101	pub fn release(tid: Id, io: std.Io) void {
	102	switch (build_options.io_mode) {
	103	.threaded => {
	104	assert(recursive_tid == tid);
	105	recursive_depth -= 1;
	106	if (recursive_depth > 0) return;
	107	recursive_tid = .main;
	108	},
	109	.evented => {},
	110	}
	111	{
	112	tid_mutex.lockUncancelable(io);
	113	defer tid_mutex.unlock(io);
	114	available_tids.appendAssumeCapacity(tid);
	115	}
	116	tid_cond.signal(io);
	117	}
	118	};
45	119
46	120	pub fn activate(zcu: *Zcu, tid: Id) Zcu.PerThread {
47	121	zcu.intern_pool.activate();
48	122	return .{ .zcu = zcu, .tid = tid };
49	123	}
50
51	124	pub fn deactivate(pt: Zcu.PerThread) void {
52	125	pt.zcu.intern_pool.deactivate();
53	126	}

src/crash_report.zig+9-38

...	...	@@ -1,34 +1,14 @@
1		/// We override the panic implementation to our own one, so we can print our own information before
2		/// calling the default panic handler. This declaration must be re-exposed from `@import("root")`.
3		pub const panic = if (dev.env == .bootstrap)
4		std.debug.simple_panic
5		else
6		std.debug.FullPanic(panicImpl);
7
8		/// We let std install its segfault handler, but we override the target-agnostic handler it calls,
9		/// so we can print our own information before calling the default segfault logic. This declaration
10		/// must be re-exposed from `@import("root")`.
11		pub const debug = struct {
12		pub const handleSegfault = handleSegfaultImpl;
13		};
14
15	1	/// Printed in panic messages when suggesting a command to run, allowing copy-pasting the command.
16	2	/// Set by `main` as soon as arguments are known. The value here is a default in case we somehow
17	3	/// crash earlier than that.
18	4	pub var zig_argv0: []const u8 = "zig";
19	5
20		fn handleSegfaultImpl(addr: ?usize, name: []const u8, opt_ctx: ?std.debug.CpuContextPtr) noreturn {
21		@branchHint(.cold);
22		dumpCrashContext() catch {};
23		std.debug.defaultHandleSegfault(addr, name, opt_ctx);
24		}
25		fn panicImpl(msg: []const u8, first_trace_addr: ?usize) noreturn {
26		@branchHint(.cold);
27		dumpCrashContext() catch {};
28		std.debug.defaultPanic(msg, first_trace_addr orelse @returnAddress());
29		}
	6	const enabled = switch (build_options.io_mode) {
	7	.threaded => build_options.enable_debug_extensions,
	8	.evented => false, // would use threadlocals in a way incompatible with evented
	9	};
30	10
31		pub const AnalyzeBody = if (build_options.enable_debug_extensions) struct {
	11	pub const AnalyzeBody = if (enabled) struct {
32	12	parent: ?*AnalyzeBody,
33	13	sema: *Sema,
34	14	block: *Sema.Block,
...	...	@@ -63,7 +43,7 @@ pub const AnalyzeBody = if (build_options.enable_debug_extensions) struct {
63	43	pub inline fn setBodyIndex(_: @This(), _: usize) void {}
64	44	};
65	45
66		pub const CodegenFunc = if (build_options.enable_debug_extensions) struct {
	46	pub const CodegenFunc = if (enabled) struct {
67	47	zcu: *const Zcu,
68	48	func_index: InternPool.Index,
69	49	threadlocal var current: ?CodegenFunc = null;
...	...	@@ -82,23 +62,14 @@ pub const CodegenFunc = if (build_options.enable_debug_extensions) struct {
82	62	pub fn stop(_: InternPool.Index) void {}
83	63	};
84	64
85		fn dumpCrashContext() Io.Writer.Error!void {
	65	pub fn dumpCrashContext(terminal: Io.Terminal) Io.Writer.Error!void {
86	66	const S = struct {
87		/// In the case of recursive panics or segfaults, don't print the context for a second time.
88		threadlocal var already_dumped = false;
89	67	/// TODO: make this unnecessary. It exists because `print_zir` currently needs an allocator,
90	68	/// but that shouldn't be necessary---it's already only used in one place.
91		threadlocal var crash_heap: [64 * 1024]u8 = undefined;
	69	var crash_heap: [64 * 1024]u8 = undefined;
92	70	};
93		if (S.already_dumped) return;
94		S.already_dumped = true;
95
96		// TODO: this does mean that a different thread could grab the stderr mutex between the context
97		// and the actual panic printing, which would be quite confusing.
98		const stderr = std.debug.lockStderr(&.{});
99		defer std.debug.unlockStderr();
100		const w = &stderr.file_writer.interface;
101	71
	72	const w = terminal.writer;
102	73	try w.writeAll("Compiler crash context:\n");
103	74
104	75	if (CodegenFunc.current) \|*cg\| {

src/introspect.zig+1-5

...	...	@@ -54,11 +54,7 @@ pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
54	54	/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
55	55	/// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
56	56	/// On WASI, "" is returned instead of ".".
57		pub fn getResolvedCwd(io: Io, gpa: Allocator) error{
58		OutOfMemory,
59		CurrentDirUnlinked,
60		Unexpected,
61		}![]u8 {
	57	pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
62	58	if (builtin.target.os.tag == .wasi) {
63	59	if (std.debug.runtime_safety) {
64	60	const cwd = try std.process.currentPathAlloc(io, gpa);

src/link.zig+4-4

...	...	@@ -1500,12 +1500,12 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
1500	1500	},
1501	1501	}
1502	1502	}
1503		pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
	1503	pub fn doZcuTask(comp: *Compilation, tid: Zcu.PerThread.Id, task: ZcuTask) void {
1504	1504	const io = comp.io;
1505	1505	const diags = &comp.link_diags;
1506	1506	const zcu = comp.zcu.?;
1507	1507	const ip = &zcu.intern_pool;
1508		const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
	1508	const pt: Zcu.PerThread = .activate(zcu, tid);
1509	1509	defer pt.deactivate();
1510	1510
1511	1511	var timer = comp.startTimer();
...	...	@@ -1610,8 +1610,8 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
1610	1610	}
1611	1611	}
1612	1612	}
1613		pub fn doIdleTask(comp: *Compilation, tid: usize) error{ OutOfMemory, LinkFailure }!bool {
1614		return if (comp.bin_file) \|lf\| lf.idle(@enumFromInt(tid)) else false;
	1613	pub fn doIdleTask(comp: *Compilation, tid: Zcu.PerThread.Id) error{ OutOfMemory, LinkFailure }!bool {
	1614	return if (comp.bin_file) \|lf\| lf.idle(tid) else false;
1615	1615	}
1616	1616	/// After the main pipeline is done, but before flush, the compilation may need to link one final
1617	1617	/// `Nav` into the binary: the `builtin.test_functions` value. Since the link thread isn't running

src/link/Queue.zig+7-5

...	...	@@ -96,12 +96,12 @@ pub fn enqueuePrelink(q: Queue, comp: Compilation, tasks: []const PrelinkTask)
96	96	pub fn enqueueZcu(
97	97	q: *Queue,
98	98	comp: *Compilation,
99		tid: usize,
	99	tid: Zcu.PerThread.Id,
100	100	task: ZcuTask,
101	101	) Io.Cancelable!void {
102	102	const io = comp.io;
103	103
104		assert(tid == 0);
	104	assert(tid == .main);
105	105
106	106	if (q.future != null) {
107	107	if (q.zcu_queue.putOne(io, task)) \|_\| {
...	...	@@ -148,8 +148,9 @@ pub fn finishZcuQueue(q: Queue, comp: Compilation) void {
148	148	}
149	149
150	150	fn runLinkTasks(q: Queue, comp: Compilation) void {
151		const tid = Compilation.getTid();
152	151	const io = comp.io;
	152	const tid: Zcu.PerThread.Id = .acquire(io);
	153	defer tid.release(io);
153	154
154	155	var have_idle_tasks = true;
155	156
...	...	@@ -198,7 +199,7 @@ fn runLinkTasks(q: Queue, comp: Compilation) void {
198	199	}
199	200	}
200	201	}
201		fn runIdleTask(comp: *Compilation, tid: usize) bool {
	202	fn runIdleTask(comp: *Compilation, tid: Zcu.PerThread.Id) bool {
202	203	return link.doIdleTask(comp, tid) catch \|err\| switch (err) {
203	204	error.OutOfMemory => have_more: {
204	205	comp.link_diags.setAllocFailure();
...	...	@@ -217,5 +218,6 @@ const Compilation = @import("../Compilation.zig");
217	218	const InternPool = @import("../InternPool.zig");
218	219	const link = @import("../link.zig");
219	220	const PrelinkTask = link.PrelinkTask;
220		const ZcuTask = link.ZcuTask;
221	221	const Queue = @This();
	222	const Zcu = @import("../Zcu.zig");
	223	const ZcuTask = link.ZcuTask;

src/main.zig+93-49

...	...	@@ -52,8 +52,11 @@ pub const std_options: std.Options = .{
52	52	};
53	53	pub const std_options_cwd = if (native_os == .wasi) wasi_cwd else null;
54	54
55		pub const panic = crash_report.panic;
56		pub const debug = crash_report.debug;
	55	pub const debug = struct {
	56	pub fn printCrashContext(terminal: Io.Terminal) void {
	57	crash_report.dumpCrashContext(terminal) catch {};
	58	}
	59	};
57	60
58	61	var preopens: std.process.Preopens = .empty;
59	62	pub fn wasi_cwd() Io.Dir {
...	...	@@ -158,25 +161,55 @@ pub fn log(
158	161	std.log.defaultLog(level, scope, format, args);
159	162	}
160	163
161		var debug_allocator: std.heap.DebugAllocator(.{
162		.stack_trace_frames = build_options.mem_leak_frames,
163		}) = .init;
164
165	164	const use_debug_allocator = build_options.debug_gpa or
166	165	(native_os != .wasi and !builtin.link_libc and switch (builtin.mode) {
167	166	.Debug, .ReleaseSafe => true,
168	167	.ReleaseFast, .ReleaseSmall => false,
169	168	});
170	169
	170	const RootAllocator = if (use_debug_allocator) std.heap.DebugAllocator(.{
	171	.stack_trace_frames = build_options.mem_leak_frames,
	172	.thread_safe = switch (build_options.io_mode) {
	173	.threaded => true,
	174	.evented => false,
	175	},
	176	}) else struct {
	177	pub const init: RootAllocator = .{};
	178	pub fn allocator(_: RootAllocator) Allocator {
	179	if (native_os == .wasi) return std.heap.wasm_allocator;
	180	if (builtin.link_libc) return std.heap.c_allocator;
	181	return std.heap.smp_allocator;
	182	}
	183	pub fn deinit(_: RootAllocator) std.heap.Check {
	184	return .ok;
	185	}
	186	};
	187
171	188	pub fn main(init: std.process.Init.Minimal) anyerror!void {
172		const gpa = gpa: {
173		if (use_debug_allocator) break :gpa debug_allocator.allocator();
174		if (native_os == .wasi) break :gpa std.heap.wasm_allocator;
175		if (builtin.link_libc) break :gpa std.heap.c_allocator;
176		break :gpa std.heap.smp_allocator;
177		};
178		defer if (use_debug_allocator) {
179		_ = debug_allocator.deinit();
	189	var root_allocator: RootAllocator = .init;
	190	defer _ = root_allocator.deinit();
	191	const root_gpa = root_allocator.allocator();
	192	var io_impl: IoImpl = undefined;
	193	switch (build_options.io_mode) {
	194	.threaded => io_impl = .init(root_gpa, .{
	195	.stack_size = thread_stack_size,
	196
	197	.argv0 = .init(init.args),
	198	.environ = init.environ,
	199	}),
	200	.evented => try io_impl.init(root_gpa, .{
	201	.argv0 = .init(init.args),
	202	.environ = init.environ,
	203
	204	.backing_allocator_needs_mutex = use_debug_allocator,
	205	}),
	206	}
	207	defer io_impl.deinit();
	208	io_impl_ptr = &io_impl;
	209	const io = io_impl.io();
	210	const gpa = switch (build_options.io_mode) {
	211	.threaded => root_gpa,
	212	.evented => io_impl.allocator(),
180	213	};
181	214	var arena_instance = std.heap.ArenaAllocator.init(gpa);
182	215	defer arena_instance.deinit();
...	...	@@ -193,17 +226,6 @@ pub fn main(init: std.process.Init.Minimal) anyerror!void {
193	226
194	227	var environ_map = init.environ.createMap(arena) catch \|err\| fatal("failed to parse environment: {t}", .{err});
195	228
196		Compilation.setMainThread();
197
198		var threaded: Io.Threaded = .init(gpa, .{
199		.argv0 = .init(init.args),
200		.environ = init.environ,
201		});
202		defer threaded.deinit();
203		threaded_impl_ptr = &threaded;
204		threaded.stack_size = thread_stack_size;
205		const io = threaded.io();
206
207	229	if (tracy.enable_allocation) {
208	230	var gpa_tracy = tracy.tracyAllocator(gpa);
209	231	return mainArgs(gpa_tracy.allocator(), arena, io, args, &environ_map);
...	...	@@ -3400,7 +3422,7 @@ fn buildOutputType(
3400	3422	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
3401	3423	std.math.maxInt(Zcu.PerThread.IdBacking),
3402	3424	);
3403		setThreadLimit(thread_limit);
	3425	try setThreadLimit(arena, thread_limit);
3404	3426
3405	3427	for (create_module.c_source_files.items) \|*src\| {
3406	3428	dev.check(.c_compiler);
...	...	@@ -4731,13 +4753,13 @@ pub fn translateC(
4731	4753	argv: []const []const u8,
4732	4754	environ_map: *const process.Environ.Map,
4733	4755	prog_node: std.Progress.Node,
	4756	thread_limit: usize,
4734	4757	capture: ?*[]u8,
4735	4758	) !void {
4736		try jitCmd(gpa, arena, io, argv, environ_map, .{
	4759	try jitCmdInner(gpa, arena, io, argv, environ_map, prog_node, thread_limit, .{
4737	4760	.cmd_name = "translate-c",
4738	4761	.root_src_path = "translate-c/main.zig",
4739	4762	.depend_on_aro = true,
4740		.progress_node = prog_node,
4741	4763	.capture = capture,
4742	4764	});
4743	4765	}
...	...	@@ -5187,7 +5209,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5187	5209	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
5188	5210	std.math.maxInt(Zcu.PerThread.IdBacking),
5189	5211	);
5190		setThreadLimit(thread_limit);
	5212	try setThreadLimit(arena, thread_limit);
5191	5213
5192	5214	// Dummy http client that is not actually used when fetch_command is unsupported.
5193	5215	// Prevents bootstrap from depending on a bunch of unnecessary stuff.
...	...	@@ -5651,7 +5673,7 @@ const JitCmdOptions = struct {
5651	5673	capture: ?*[]u8 = null,
5652	5674	/// Send error bundles via std.zig.Server over stdout
5653	5675	server: bool = false,
5654		progress_node: ?std.Progress.Node = null,
	5676	color: Color = .auto,
5655	5677	};
5656	5678
5657	5679	fn jitCmd(
...	...	@@ -5664,12 +5686,30 @@ fn jitCmd(
5664	5686	) !void {
5665	5687	dev.check(.jit_command);
5666	5688
5667		const color: Color = .auto;
5668		const root_prog_node = if (options.progress_node) \|node\| node else std.Progress.start(io, .{
5669		.disable_printing = (color == .off),
	5689	const root_prog_node = std.Progress.start(io, .{
	5690	.disable_printing = (options.color == .off),
5670	5691	});
5671	5692	defer root_prog_node.end();
5672	5693
	5694	const thread_limit = @min(
	5695	@max(std.Thread.getCpuCount() catch 1, 1),
	5696	std.math.maxInt(Zcu.PerThread.IdBacking),
	5697	);
	5698	try setThreadLimit(arena, thread_limit);
	5699
	5700	return jitCmdInner(gpa, arena, io, args, environ_map, root_prog_node, thread_limit, options);
	5701	}
	5702
	5703	fn jitCmdInner(
	5704	gpa: Allocator,
	5705	arena: Allocator,
	5706	io: Io,
	5707	args: []const []const u8,
	5708	environ_map: *const process.Environ.Map,
	5709	root_prog_node: std.Progress.Node,
	5710	thread_limit: usize,
	5711	options: JitCmdOptions,
	5712	) !void {
5673	5713	const target_query: std.Target.Query = .{};
5674	5714	const resolved_target: Package.Module.ResolvedTarget = .{
5675	5715	.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
...	...	@@ -5702,12 +5742,6 @@ fn jitCmd(
5702	5742	);
5703	5743	defer dirs.deinit(io);
5704	5744
5705		const thread_limit = @min(
5706		@max(std.Thread.getCpuCount() catch 1, 1),
5707		std.math.maxInt(Zcu.PerThread.IdBacking),
5708		);
5709		setThreadLimit(thread_limit);
5710
5711	5745	var child_argv: std.ArrayList([]const u8) = .empty;
5712	5746	try child_argv.ensureUnusedCapacity(arena, args.len + 4);
5713	5747
...	...	@@ -5795,7 +5829,7 @@ fn jitCmd(
5795	5829	process.exit(2);
5796	5830	}
5797	5831	} else {
5798		updateModule(comp, color, root_prog_node) catch \|err\| switch (err) {
	5832	updateModule(comp, options.color, root_prog_node) catch \|err\| switch (err) {
5799	5833	error.CompileErrorsReported => process.exit(2),
5800	5834	else => \|e\| return e,
5801	5835	};
...	...	@@ -7777,15 +7811,25 @@ fn addLibDirectoryWarn2(
7777	7811	});
7778	7812	}
7779	7813
7780		var threaded_impl_ptr: *Io.Threaded = undefined;
7781		fn setThreadLimit(n: usize) void {
7782		// We want a maximum of n total threads to keep the InternPool happy, but
7783		// the main thread doesn't count towards the limits, so use n-1. Also, the
7784		// linker can run concurrently, so we need to set both the async and the
7785		// concurrency limit.
7786		const limit: Io.Limit = .limited(n - 1);
7787		threaded_impl_ptr.setAsyncLimit(limit);
7788		threaded_impl_ptr.concurrent_limit = limit;
	7814	const IoImpl = switch (build_options.io_mode) {
	7815	.threaded => Io.Threaded,
	7816	.evented => Io.Evented,
	7817	};
	7818	var io_impl_ptr: *IoImpl = undefined;
	7819	fn setThreadLimit(arena: std.mem.Allocator, n: usize) Allocator.Error!void {
	7820	switch (build_options.io_mode) {
	7821	.threaded => {
	7822	// We want a maximum of n total threads to keep the InternPool happy, but
	7823	// the main thread doesn't count towards the limits, so use n-1. Also, the
	7824	// linker can run concurrently, so we need to set both the async and the
	7825	// concurrency limit.
	7826	const limit: Io.Limit = .limited(n - 1);
	7827	io_impl_ptr.setAsyncLimit(limit);
	7828	io_impl_ptr.concurrent_limit = limit;
	7829	},
	7830	.evented => {},
	7831	}
	7832	try Zcu.PerThread.Id.allocate(arena, @max(n, 2));
7789	7833	}
7790	7834
7791	7835	fn randInt(io: Io, comptime T: type) T {

stage1/config.zig.in+1

...	...	@@ -13,4 +13,5 @@ pub const value_tracing = false;
13	13	pub const skip_non_native = false;
14	14	pub const debug_gpa = false;
15	15	pub const dev = .core;
	16	pub const io_mode: enum { threaded, evented } = .threaded;
16	17	pub const value_interpret_mode = .direct;