Commit b607b0c27af71a541811955d3012f5346c997b09 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-02-10 00:43:51+01:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-02-10 00:43:51+01:00
log	b607b0c27af71a541811955d3012f5346c997b09
tree	9b11b9fcf148b21c3bcf656c334dde5119e04935
parent	04c180c8e57fecdf3478de966f32b8a1f65ad202
parent	7e8ee985e20f34ba6afb815a7cded443aca297d3

Merge pull request 'IoUring: update to new Io APIs' (#31158) from io-uring-update into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31158
Reviewed-by: Andrew Kelley <andrew@ziglang.org>

21 files changed, 5997 insertions(+), 1231 deletions(-)

bootstrap.c+1

...	...	@@ -143,6 +143,7 @@ int main(int argc, char **argv) {
143	143	"pub const skip_non_native = false;\n"
144	144	"pub const debug_gpa = false;\n"
145	145	"pub const dev = .core;\n"
	146	"pub const io_mode: enum { threaded, evented } = .threaded;\n"
146	147	"pub const value_interpret_mode = .direct;\n"
147	148	, zig_version);
148	149	if (written < 100)

build.zig+17-5

...	...	@@ -13,6 +13,7 @@ const DevEnv = @import("src/dev.zig").Env;
13	13	const zig_version: std.SemanticVersion = .{ .major = 0, .minor = 16, .patch = 0 };
14	14	const stack_size = 46 * 1024 * 1024;
15	15
	16	const IoMode = enum { threaded, evented };
16	17	const ValueInterpretMode = enum { direct, by_name };
17	18
18	19	pub fn build(b: *std.Build) !void {
...	...	@@ -188,6 +189,7 @@ pub fn build(b: *std.Build) !void {
188	189	const strip = b.option(bool, "strip", "Omit debug information");
189	190	const valgrind = b.option(bool, "valgrind", "Enable valgrind integration");
190	191	const pie = b.option(bool, "pie", "Produce a Position Independent Executable");
	192	const io_mode = b.option(IoMode, "io-mode", "How the compiler performs IO") orelse .threaded;
191	193	const value_interpret_mode = b.option(ValueInterpretMode, "value-interpret-mode", "How the compiler translates between 'std.builtin' types and its internal datastructures") orelse .direct;
192	194	const value_tracing = b.option(bool, "value-tracing", "Enable extra state tracking to help troubleshoot bugs in the compiler (using the std.debug.Trace API)") orelse false;
193	195
...	...	@@ -236,6 +238,7 @@ pub fn build(b: *std.Build) !void {
236	238	exe_options.addOption(bool, "llvm_has_xtensa", llvm_has_xtensa);
237	239	exe_options.addOption(bool, "debug_gpa", debug_gpa);
238	240	exe_options.addOption(DevEnv, "dev", b.option(DevEnv, "dev", "Build a compiler with a reduced feature set for development of specific features") orelse if (only_c) .bootstrap else .full);
	241	exe_options.addOption(IoMode, "io_mode", io_mode);
239	242	exe_options.addOption(ValueInterpretMode, "value_interpret_mode", value_interpret_mode);
240	243
241	244	if (link_libc) {
...	...	@@ -367,14 +370,22 @@ pub fn build(b: *std.Build) !void {
367	370	&[_][]const u8{ tracy_path, "public", "TracyClient.cpp" },
368	371	);
369	372
370		const tracy_c_flags: []const []const u8 = &.{ "-DTRACY_ENABLE=1", "-fno-sanitize=undefined" };
	373	const tracy_c_flags: []const []const u8 = &.{
	374	"-DTRACY_ENABLE=1",
	375	"-fno-sanitize=undefined",
	376	"-DTRACY_FIBERS",
	377	};
371	378
372	379	exe.root_module.addIncludePath(.{ .cwd_relative = tracy_path });
373		exe.root_module.addCSourceFile(.{ .file = .{ .cwd_relative = client_cpp }, .flags = tracy_c_flags });
374		if (!enable_llvm) {
375		exe.root_module.linkSystemLibrary("c++", .{ .use_pkg_config = .no });
376		}
	380	exe.root_module.addCSourceFile(.{
	381	.file = .{ .cwd_relative = client_cpp },
	382	.flags = tracy_c_flags[0..switch (io_mode) {
	383	.threaded => 2,
	384	.evented => 3,
	385	}],
	386	});
377	387	exe.root_module.link_libc = true;
	388	exe.root_module.link_libcpp = true;
378	389
379	390	if (target.result.os.tag == .windows) {
380	391	exe.root_module.linkSystemLibrary("dbghelp", .{});
...	...	@@ -712,6 +723,7 @@ fn addWasiUpdateStep(b: *std.Build, version: [:0]const u8) !void {
712	723	exe_options.addOption(u32, "tracy_callstack_depth", 0);
713	724	exe_options.addOption(bool, "value_tracing", false);
714	725	exe_options.addOption(DevEnv, "dev", .bootstrap);
	726	exe_options.addOption(IoMode, "io_mode", .threaded);
715	727
716	728	// zig1 chooses to interpret values by name. The tradeoff is as follows:
717	729	//

lib/std/Io.zig+41-20

...	...	@@ -378,7 +378,9 @@ pub const Operation = union(enum) {
378	378	pub const Pending = struct {
379	379	node: List.DoubleNode,
380	380	tag: Tag,
381		context: [3]usize,
	381	context: Context align(@max(@alignOf(usize), 4)),
	382
	383	pub const Context = [3]usize;
382	384	};
383	385
384	386	pub const Completion = struct {
...	...	@@ -426,10 +428,10 @@ pub fn operate(io: Io, operation: Operation) Cancelable!Operation.Result {
426	428	pub const Batch = struct {
427	429	storage: []Operation.Storage,
428	430	unused: Operation.List,
429		submissions: Operation.List,
	431	submitted: Operation.List,
430	432	pending: Operation.List,
431		completions: Operation.List,
432		context: ?*anyopaque,
	433	completed: Operation.List,
	434	context: ?anyopaque align(@max(@alignOf(?anyopaque), 4)),
433	435
434	436	/// After calling this, it is safe to unconditionally defer a call to
435	437	/// `cancel`. `storage` is a pre-allocated buffer of undefined memory that
...	...	@@ -448,9 +450,9 @@ pub const Batch = struct {
448	450	.head = .fromIndex(0),
449	451	.tail = .fromIndex(storage.len - 1),
450	452	},
451		.submissions = .empty,
	453	.submitted = .empty,
452	454	.pending = .empty,
453		.completions = .empty,
	455	.completed = .empty,
454	456	.context = null,
455	457	};
456	458	}
...	...	@@ -471,20 +473,20 @@ pub const Batch = struct {
471	473	const storage = &b.storage[index];
472	474	const unused = storage.unused;
473	475	switch (unused.prev) {
474		.none => b.unused.head = .none,
	476	.none => b.unused.head = unused.next,
475	477	else => \|prev_index\| b.storage[prev_index.toIndex()].unused.next = unused.next,
476	478	}
477	479	switch (unused.next) {
478		.none => b.unused.tail = .none,
	480	.none => b.unused.tail = unused.prev,
479	481	else => \|next_index\| b.storage[next_index.toIndex()].unused.prev = unused.prev,
480	482	}
481	483
482		switch (b.submissions.tail) {
483		.none => b.submissions.head = .fromIndex(index),
	484	switch (b.submitted.tail) {
	485	.none => b.submitted.head = .fromIndex(index),
484	486	else => \|tail_index\| b.storage[tail_index.toIndex()].submission.node.next = .fromIndex(index),
485	487	}
486	488	storage.* = .{ .submission = .{ .node = .{ .next = .none }, .operation = operation } };
487		b.submissions.tail = .fromIndex(index);
	489	b.submitted.tail = .fromIndex(index);
488	490	}
489	491
490	492	pub const Completion = struct {
...	...	@@ -501,13 +503,13 @@ pub const Batch = struct {
501	503	/// Each completion returned from this function dequeues from the `Batch`.
502	504	/// It is not required to dequeue all completions before awaiting again.
503	505	pub fn next(b: *Batch) ?Completion {
504		const index = b.completions.head;
	506	const index = b.completed.head;
505	507	if (index == .none) return null;
506	508	const storage = &b.storage[index.toIndex()];
507	509	const completion = storage.completion;
508	510	const next_index = completion.node.next;
509		b.completions.head = next_index;
510		if (next_index == .none) b.completions.tail = .none;
	511	b.completed.head = next_index;
	512	if (next_index == .none) b.completed.tail = .none;
511	513
512	514	const tail_index = b.unused.tail;
513	515	switch (tail_index) {
...	...	@@ -551,7 +553,27 @@ pub const Batch = struct {
551	553	/// may have successfully completed regardless of the cancel request and
552	554	/// will appear in the iteration.
553	555	pub fn cancel(b: *Batch, io: Io) void {
554		return io.vtable.batchCancel(io.userdata, b);
	556	{ // abort pending submissions
	557	var tail_index = b.unused.tail;
	558	defer b.unused.tail = tail_index;
	559	var index = b.submitted.head;
	560	errdefer b.submissions.head = index;
	561	while (index != .none) {
	562	const next_index = b.storage[index.toIndex()].submission.node.next;
	563	switch (tail_index) {
	564	.none => b.unused.head = index,
	565	else => b.storage[tail_index.toIndex()].unused.next = index,
	566	}
	567	b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
	568	tail_index = index;
	569	index = next_index;
	570	}
	571	b.submitted = .{ .head = .none, .tail = .none };
	572	}
	573	io.vtable.batchCancel(io.userdata, b);
	574	assert(b.submitted.head == .none and b.submitted.tail == .none);
	575	assert(b.pending.head == .none and b.pending.tail == .none);
	576	assert(b.context == null); // that was the last chance to deallocate resources
555	577	}
556	578	};
557	579
...	...	@@ -1117,13 +1139,13 @@ pub fn recancel(io: Io) void {
1117	1139	/// To modify a task's cancel protection state, see `swapCancelProtection`.
1118	1140	///
1119	1141	/// For a description of cancelation and cancelation points, see `Future.cancel`.
1120		pub const CancelProtection = enum {
	1142	pub const CancelProtection = enum(u1) {
1121	1143	/// Any call to an `Io` function with `error.Canceled` in its error set is a cancelation point.
1122	1144	///
1123	1145	/// This is the default state, which all tasks are created in.
1124		unblocked,
	1146	unblocked = 0,
1125	1147	/// No `Io` function introduces a cancelation point (`error.Canceled` will never be returned).
1126		blocked,
	1148	blocked = 1,
1127	1149	};
1128	1150	/// Updates the current task's cancel protection state (see `CancelProtection`).
1129	1151	///
...	...	@@ -1292,8 +1314,7 @@ pub fn futexWake(io: Io, comptime T: type, ptr: *align(@alignOf(u32)) const T, m
1292	1314	/// shared region of code known as the "critical section".
1293	1315	///
1294	1316	/// Mutex is an extern struct so that it may be used as a field inside another
1295		/// extern struct. Having a guaranteed memory layout including mutexes is
1296		/// important for IPC over shared memory (mmap).
	1317	/// extern struct.
1297	1318	pub const Mutex = extern struct {
1298	1319	state: std.atomic.Value(State),
1299	1320

lib/std/Io/File.zig+7-2

...	...	@@ -477,12 +477,17 @@ pub const Permissions = std.Options.FilePermissions orelse if (is_windows) enum(
477	477	/// libc implementations use `0o666` inside `fopen` and then rely on the
478	478	/// process-scoped "umask" setting to adjust this number for file creation.
479	479	default_file = 0o666,
480		default_dir = 0o755,
481		executable_file = 0o777,
	480	/// This is the default mode given to POSIX operating systems for creating
	481	/// directories. `0o777` is "-rwxrwxrwx" which is counter-intuitive at first,
	482	/// since most people would expect "-rwxr-xr-x", for example, when using
	483	/// the `touch` command, which would correspond to `0o755`.
	484	default_dir = 0o777,
482	485	_,
483	486
484	487	pub const has_executable_bit = native_os != .wasi;
485	488
	489	pub const executable_file: @This() = .default_dir;
	490
486	491	pub fn toMode(self: @This()) std.posix.mode_t {
487	492	return @intFromEnum(self);
488	493	}

lib/std/Io/IoUring.zig+5425-775

...	...	@@ -1,21 +1,88 @@
1		const EventLoop = @This();
	1	const addressFromPosix = Io.Threaded.addressFromPosix;
	2	const addressToPosix = Io.Threaded.addressToPosix;
	3	const Alignment = std.mem.Alignment;
	4	const Allocator = std.mem.Allocator;
	5	const Argv0 = Io.Threaded.Argv0;
	6	const assert = std.debug.assert;
2	7	const builtin = @import("builtin");
3
4		const std = @import("../std.zig");
	8	const ChdirError = Io.Threaded.ChdirError;
	9	const clockToPosix = Io.Threaded.clockToPosix;
	10	const Csprng = Io.Threaded.Csprng;
	11	const default_PATH = Io.Threaded.default_PATH;
	12	const Dir = Io.Dir;
	13	const Environ = Io.Threaded.Environ;
	14	const errnoBug = Io.Threaded.errnoBug;
	15	const Evented = @This();
	16	const fallbackSeed = Io.Threaded.fallbackSeed;
	17	const fd_t = linux.fd_t;
	18	const File = Io.File;
5	19	const Io = std.Io;
6		const assert = std.debug.assert;
7		const Allocator = std.mem.Allocator;
8		const Alignment = std.mem.Alignment;
9		const IoUring = std.os.linux.IoUring;
	20	const IoUring = linux.IoUring;
	21	const iovec = std.posix.iovec;
	22	const iovec_const = std.posix.iovec_const;
	23	const linux = std.os.linux;
	24	const linux_statx_request = Io.Threaded.linux_statx_request;
	25	const LOCK = std.posix.LOCK;
	26	const log = std.log.scoped(.@"io-uring");
	27	const max_iovecs_len = Io.Threaded.max_iovecs_len;
	28	const nanosecondsFromPosix = Io.Threaded.nanosecondsFromPosix;
	29	const net = Io.net;
	30	const PATH_MAX = linux.PATH_MAX;
	31	const pathToPosix = Io.Threaded.pathToPosix;
	32	const pid_t = linux.pid_t;
	33	const PosixAddress = Io.Threaded.PosixAddress;
	34	const posixAddressFamily = Io.Threaded.posixAddressFamily;
	35	const posixProtocol = Io.Threaded.posixProtocol;
	36	const posixSocketMode = Io.Threaded.posixSocketMode;
	37	const process = std.process;
	38	const recoverableOsBugDetected = Io.Threaded.recoverableOsBugDetected;
	39	const setTimestampToPosix = Io.Threaded.setTimestampToPosix;
	40	const splat_buffer_size = Io.Threaded.splat_buffer_size;
	41	const statFromLinux = Io.Threaded.statFromLinux;
	42	const std = @import("../std.zig");
	43	const timestampFromPosix = Io.Threaded.timestampFromPosix;
	44	const unexpectedErrno = std.posix.unexpectedErrno;
	45	const winsize = std.posix.winsize;
	46
	47	const tracy = if (@hasDecl(@import("root"), "tracy")) @import("root").tracy else struct {
	48	const enable = false;
	49	inline fn fiberEnter(fiber: [*:0]const u8) void {
	50	_ = fiber;
	51	}
	52	inline fn fiberLeave() void {}
	53	};
10	54
11		/// Must be a thread-safe allocator.
12		gpa: Allocator,
13		mutex: Io.Mutex,
14		main_fiber_buffer: [@sizeOf(Fiber) + Fiber.max_result_size]u8 align(@alignOf(Fiber)),
	55	backing_allocator_needs_mutex: bool,
	56	backing_allocator_mutex: Io.Mutex,
	57	/// Does not need to be thread-safe if not used elsewhere.
	58	backing_allocator: Allocator,
	59	main_fiber_buffer: [
	60	std.mem.alignForward(usize, @sizeOf(Fiber), @alignOf(Completion)) + @sizeOf(Completion)
	61	]u8 align(@max(@alignOf(Fiber), @alignOf(Completion))),
15	62	threads: Thread.List,
16	63
	64	stderr_mutex: Io.Mutex,
	65	stderr_writer: File.Writer = .{
	66	.io = undefined,
	67	.interface = Io.File.Writer.initInterface(&.{}),
	68	.file = .stderr(),
	69	.mode = .streaming,
	70	},
	71	stderr_mode: Io.Terminal.Mode = .no_color,
	72	stderr_writer_initialized: bool = false,
	73
	74	environ_mutex: Io.Mutex,
	75	environ: Environ,
	76
	77	null_fd: CachedFd,
	78	random_fd: CachedFd,
	79
	80	csprng_mutex: Io.Mutex,
	81	csprng: Csprng,
	82
17	83	/// Empirically saw >128KB being used by the self-hosted backend to panic.
18		const idle_stack_size = 256 * 1024;
	84	/// Empirically saw glibc complain about 256KB.
	85	const idle_stack_size = 512 * 1024;
19	86
20	87	const max_idle_search = 4;
21	88	const max_steal_ready_search = 4;
...	...	@@ -23,6 +90,7 @@ const max_steal_ready_search = 4;
23	90	const io_uring_entries = 64;
24	91
25	92	const Thread = struct {
	93	required_align: void align(4),
26	94	thread: std.Thread,
27	95	idle_context: Context,
28	96	current_context: *Context,
...	...	@@ -30,19 +98,34 @@ const Thread = struct {
30	98	io_uring: IoUring,
31	99	idle_search_index: u32,
32	100	steal_ready_search_index: u32,
	101	name_arena: if (tracy.enable) std.heap.ArenaAllocator.State else struct {},
	102	csprng: Csprng,
33	103
34		const canceling: ?*Thread = @ptrFromInt(@alignOf(Thread));
35
36		threadlocal var self: *Thread = undefined;
	104	threadlocal var self: ?*Thread = null;
37	105
38		fn current() *Thread {
39		return self;
	106	noinline fn current() *Thread {
	107	return self.?;
40	108	}
41	109
42	110	fn currentFiber(thread: Thread) Fiber {
	111	assert(thread.current_context != &thread.idle_context);
43	112	return @fieldParentPtr("context", thread.current_context);
44	113	}
45	114
	115	fn enqueue(thread: Thread) linux.io_uring_sqe {
	116	while (true) return thread.io_uring.get_sqe() catch {
	117	thread.submit();
	118	continue;
	119	};
	120	}
	121
	122	fn submit(thread: *Thread) void {
	123	_ = thread.io_uring.submit() catch \|err\| switch (err) {
	124	error.SignalInterrupt => {},
	125	else => \|e\| @panic(@errorName(e)),
	126	};
	127	}
	128
46	129	const List = struct {
47	130	allocated: []Thread,
48	131	reserved: u32,
...	...	@@ -53,18 +136,112 @@ const Thread = struct {
53	136	const Fiber = struct {
54	137	required_align: void align(4),
55	138	context: Context,
56		awaiter: ?*Fiber,
57		queue_next: ?*Fiber,
58		cancel_thread: ?*Thread,
59		awaiting_completions: std.StaticBitSet(3),
	139	await_count: i32,
	140	link: union {
	141	awaiter: ?*Fiber,
	142	group: struct { prev: ?Fiber, next: ?Fiber },
	143	},
	144	status: union(enum) {
	145	queue_next: ?*Fiber,
	146	awaiting_group: Group,
	147	},
	148	cancel_status: CancelStatus,
	149	cancel_protection: CancelProtection,
	150	name: if (tracy.enable) [*:0]const u8 else void,
	151
	152	var next_name: u64 = 0;
	153
	154	const CancelStatus = packed struct(u32) {
	155	requested: bool,
	156	awaiting: Awaiting,
	157
	158	const unrequested: CancelStatus = .{ .requested = false, .awaiting = .nothing };
	159
	160	const Awaiting = enum(u31) {
	161	nothing = std.math.maxInt(u31),
	162	group = std.math.maxInt(u31) - 1,
	163	select = std.math.maxInt(u31) - 2,
	164	/// An io_uring fd.
	165	_,
	166
	167	fn subWrap(lhs: Awaiting, rhs: Awaiting) Awaiting {
	168	return @enumFromInt(@intFromEnum(lhs) -% @intFromEnum(rhs));
	169	}
	170
	171	fn fromIoUringFd(fd: fd_t) Awaiting {
	172	const awaiting: Awaiting = @enumFromInt(fd);
	173	switch (awaiting) {
	174	.nothing, .group, .select => unreachable,
	175	_ => return awaiting,
	176	}
	177	}
	178
	179	fn toIoUringFd(awaiting: Awaiting) fd_t {
	180	switch (awaiting) {
	181	.nothing, .group => unreachable,
	182	_ => return @intFromEnum(awaiting),
	183	}
	184	}
	185	};
	186
	187	fn changeAwaiting(
	188	cancel_status: *CancelStatus,
	189	old_awaiting: Awaiting,
	190	new_awaiting: Awaiting,
	191	) bool {
	192	const old_cancel_status = @atomicRmw(CancelStatus, cancel_status, .Add, .{
	193	.requested = false,
	194	.awaiting = new_awaiting.subWrap(old_awaiting),
	195	}, .monotonic);
	196	assert(old_cancel_status.awaiting == old_awaiting);
	197	return old_cancel_status.requested;
	198	}
	199	};
	200
	201	const CancelProtection = packed struct {
	202	user: Io.CancelProtection,
	203	acknowledged: bool,
	204
	205	const unblocked: CancelProtection = .{ .user = .unblocked, .acknowledged = false };
	206
	207	fn check(cancel_protection: CancelProtection) Io.CancelProtection {
	208	return @enumFromInt(@intFromBool(cancel_protection != unblocked));
	209	}
	210
	211	fn acknowledge(cancel_protection: *CancelProtection) void {
	212	assert(!cancel_protection.acknowledged);
	213	cancel_protection.acknowledged = true;
	214	}
	215
	216	fn recancel(cancel_protection: *CancelProtection) void {
	217	assert(cancel_protection.acknowledged);
	218	cancel_protection.acknowledged = false;
	219	}
	220
	221	test check {
	222	try std.testing.expectEqual(Io.CancelProtection.unblocked, check(.unblocked));
	223	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	224	.user = .unblocked,
	225	.acknowledged = true,
	226	}));
	227	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	228	.user = .blocked,
	229	.acknowledged = false,
	230	}));
	231	try std.testing.expectEqual(Io.CancelProtection.blocked, check(.{
	232	.user = .blocked,
	233	.acknowledged = true,
	234	}));
	235	}
	236	};
60	237
61	238	const finished: ?*Fiber = @ptrFromInt(@alignOf(Thread));
62	239
63	240	const max_result_align: Alignment = .@"16";
64		const max_result_size = max_result_align.forward(64);
	241	const max_result_size = max_result_align.forward(512);
65	242	/// This includes any stack realignments that need to happen, and also the
66	243	/// initial frame return address slot and argument frame, depending on target.
67		const min_stack_size = 4 * 1024 * 1024;
	244	const min_stack_size = 60 * 1024 * 1024;
68	245	const max_context_align: Alignment = .@"16";
69	246	const max_context_size = max_context_align.forward(1024);
70	247	const max_closure_size: usize = @sizeOf(AsyncClosure);
...	...	@@ -76,9 +253,19 @@ const Fiber = struct {
76	253	) + max_closure_size + max_context_size,
77	254	std.heap.page_size_max,
78	255	);
	256	comptime {
	257	assert(max_result_align.compare(.gte, .of(Completion)));
	258	assert(max_result_size >= @sizeOf(Completion));
	259	}
	260
	261	fn create(ev: Evented) error{OutOfMemory}!Fiber {
	262	return @ptrCast(try ev.allocator().alignedAlloc(u8, .of(Fiber), allocation_size));
	263	}
79	264
80		fn allocate(el: EventLoop) error{OutOfMemory}!Fiber {
81		return @ptrCast(try el.gpa.alignedAlloc(u8, .of(Fiber), allocation_size));
	265	fn destroy(fiber: *Fiber, gpa: std.mem.Allocator) void {
	266	log.debug("destroying {*}", .{fiber});
	267	assert(fiber.status.queue_next == null);
	268	gpa.free(fiber.allocatedSlice());
82	269	}
83	270
84	271	fn allocatedSlice(f: *Fiber) []align(@alignOf(Fiber)) u8 {
...	...	@@ -98,98 +285,514 @@ const Fiber = struct {
98	285	return @ptrFromInt(alignment.forward(@intFromPtr(f) + @sizeOf(Fiber)));
99	286	}
100	287
101		fn enterCancelRegion(fiber: Fiber, thread: Thread) error{Canceled}!void {
102		if (@cmpxchgStrong(
103		?*Thread,
104		&fiber.cancel_thread,
105		null,
106		thread,
	288	const Queue = struct { head: Fiber, tail: Fiber };
	289
	290	/// Like a `*Fiber`, but 2 bits smaller than a pointer (because the LSBs are always 0 due to
	291	/// alignment) so that those two bits can be used in a `packed struct`.
	292	const PackedPtr = enum(@Int(.unsigned, @bitSizeOf(usize) - 2)) {
	293	null = 0,
	294	all_ones = std.math.maxInt(@Int(.unsigned, @bitSizeOf(usize) - 2)),
	295	_,
	296
	297	const Split = packed struct(usize) { low: u2, high: PackedPtr };
	298	fn pack(ptr: ?*Fiber) PackedPtr {
	299	const split: Split = @bitCast(@intFromPtr(ptr));
	300	assert(split.low == 0);
	301	return split.high;
	302	}
	303	fn unpack(ptr: PackedPtr) ?*Fiber {
	304	const split: Split = .{ .low = 0, .high = ptr };
	305	return @ptrFromInt(@as(usize, @bitCast(split)));
	306	}
	307	};
	308
	309	fn requestCancel(fiber: Fiber, ev: Evented) void {
	310	const cancel_status = @atomicRmw(
	311	Fiber.CancelStatus,
	312	&fiber.cancel_status,
	313	.Or,
	314	.{ .requested = true, .awaiting = @enumFromInt(0) },
107	315	.acq_rel,
108		.acquire,
109		)) \|cancel_thread\| {
110		assert(cancel_thread == Thread.canceling);
	316	);
	317	assert(!cancel_status.requested);
	318	switch (cancel_status.awaiting) {
	319	.nothing => {},
	320	.group => {
	321	// The awaiter received a cancelation request while awaiting a group,
	322	// so propagate the cancelation to the group.
	323	if (fiber.status.awaiting_group.cancel(ev, null)) {
	324	fiber.status = .{ .queue_next = null };
	325	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
	326	}
	327	},
	328	.select => if (@atomicRmw(i32, &fiber.await_count, .Add, 1, .monotonic) == -1) {
	329	_ = ev.schedule(.current(), .{ .head = fiber, .tail = fiber });
	330	},
	331	_ => \|cancel_io_uring_fd\| {
	332	const thread: *Thread = .current();
	333	thread.enqueue().* = if (thread.io_uring.fd == @intFromEnum(cancel_io_uring_fd)) .{
	334	.opcode = .ASYNC_CANCEL,
	335	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	336	.ioprio = 0,
	337	.fd = 0,
	338	.off = 0,
	339	.addr = @intFromPtr(fiber),
	340	.len = 0,
	341	.rw_flags = 0,
	342	.user_data = @intFromEnum(Completion.UserData.wakeup),
	343	.buf_index = 0,
	344	.personality = 0,
	345	.splice_fd_in = 0,
	346	.addr3 = 0,
	347	.resv = 0,
	348	} else .{
	349	.opcode = .MSG_RING,
	350	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	351	.ioprio = 0,
	352	.fd = @intFromEnum(cancel_io_uring_fd),
	353	.off = @intFromPtr(fiber) \| 0b01,
	354	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
	355	.len = 0,
	356	.rw_flags = 0,
	357	.user_data = @intFromEnum(Completion.UserData.cleanup),
	358	.buf_index = 0,
	359	.personality = 0,
	360	.splice_fd_in = 0,
	361	.addr3 = 0,
	362	.resv = 0,
	363	};
	364	},
	365	}
	366	}
	367	};
	368
	369	const CancelRegion = struct {
	370	fiber: *Fiber,
	371	status: Fiber.CancelStatus,
	372	fn init() CancelRegion {
	373	const fiber = Thread.current().currentFiber();
	374	return .{
	375	.fiber = fiber,
	376	.status = .{
	377	.requested = fiber.cancel_protection.check() == .unblocked,
	378	.awaiting = .nothing,
	379	},
	380	};
	381	}
	382	fn initBlocked() CancelRegion {
	383	return .{
	384	.fiber = Thread.current().currentFiber(),
	385	.status = .{ .requested = false, .awaiting = .nothing },
	386	};
	387	}
	388	fn deinit(cancel_region: *CancelRegion) void {
	389	if (cancel_region.status.requested) _ = cancel_region.fiber.cancel_status.changeAwaiting(
	390	cancel_region.status.awaiting,
	391	.nothing,
	392	);
	393	cancel_region.* = undefined;
	394	}
	395	fn await(cancel_region: *CancelRegion, awaiting: Fiber.CancelStatus.Awaiting) Io.Cancelable!void {
	396	if (!cancel_region.status.requested) return;
	397	const status: Fiber.CancelStatus = .{ .requested = true, .awaiting = awaiting };
	398	if (cancel_region.fiber.cancel_status.changeAwaiting(
	399	cancel_region.status.awaiting,
	400	status.awaiting,
	401	)) {
	402	cancel_region.fiber.cancel_protection.acknowledge();
	403	cancel_region.status = .unrequested;
111	404	return error.Canceled;
112	405	}
	406	cancel_region.status = status;
	407	}
	408	fn awaitIoUring(cancel_region: CancelRegion) Io.Cancelable!Thread {
	409	const thread: *Thread = .current();
	410	try cancel_region.await(.fromIoUringFd(thread.io_uring.fd));
	411	return thread;
113	412	}
	413	fn completion(cancel_region: *const CancelRegion) Completion {
	414	return cancel_region.fiber.resultPointer(Completion).*;
	415	}
	416	fn errno(cancel_region: *const CancelRegion) linux.E {
	417	return cancel_region.completion().errno();
	418	}
	419	};
114	420
115		fn exitCancelRegion(fiber: Fiber, thread: Thread) void {
116		if (@cmpxchgStrong(
117		?*Thread,
118		&fiber.cancel_thread,
119		thread,
120		null,
121		.acq_rel,
122		.acquire,
123		)) \|cancel_thread\| assert(cancel_thread == Thread.canceling);
	421	const CachedFd = struct {
	422	once: Once,
	423
	424	const Once = enum(fd_t) {
	425	uninitialized = -1,
	426	initializing = -2,
	427	/// fd
	428	_,
	429
	430	fn fromFd(fd: fd_t) Once {
	431	return @enumFromInt(@as(u31, @intCast(fd)));
	432	}
	433
	434	fn toFd(once: Once) fd_t {
	435	return @as(u31, @intCast(@intFromEnum(once)));
	436	}
	437	};
	438
	439	const init: CachedFd = .{ .once = .uninitialized };
	440
	441	fn close(cached_fd: *CachedFd) void {
	442	switch (cached_fd.once) {
	443	.uninitialized => {},
	444	.initializing => unreachable,
	445	_ => \|fd\| {
	446	assert(@intFromEnum(fd) >= 0);
	447	std.posix.close(@intFromEnum(fd));
	448	cached_fd.* = .init;
	449	},
	450	}
124	451	}
125	452
126		const Queue = struct { head: Fiber, tail: Fiber };
	453	fn open(
	454	cached_fd: *CachedFd,
	455	ev: *Evented,
	456	cancel_region: *CancelRegion,
	457	path: [*:0]const u8,
	458	flags: linux.O,
	459	) File.OpenError!fd_t {
	460	var once = @atomicLoad(Once, &cached_fd.once, .monotonic);
	461	while (true) {
	462	switch (once) {
	463	.uninitialized => {},
	464	.initializing => try futexWait(
	465	ev,
	466	@ptrCast(&cached_fd.once),
	467	@bitCast(@intFromEnum(once)),
	468	.none,
	469	),
	470	_ => \|fd\| {
	471	@branchHint(.likely);
	472	return fd.toFd();
	473	},
	474	}
	475	once = @cmpxchgWeak(
	476	Once,
	477	&cached_fd.once,
	478	.uninitialized,
	479	.initializing,
	480	.monotonic,
	481	.monotonic,
	482	) orelse {
	483	errdefer {
	484	@atomicStore(Once, &cached_fd.once, .uninitialized, .monotonic);
	485	futexWake(ev, @ptrCast(&cached_fd.once), 1);
	486	}
	487	const fd = try ev.openat(cancel_region, linux.AT.FDCWD, path, flags, 0);
	488	@atomicStore(Once, &cached_fd.once, .fromFd(fd), .monotonic);
	489	futexWake(ev, @ptrCast(&cached_fd.once), std.math.maxInt(u32));
	490	return fd;
	491	};
	492	}
	493	}
127	494	};
128	495
129		fn recycle(el: EventLoop, fiber: Fiber) void {
130		std.log.debug("recyling {*}", .{fiber});
131		assert(fiber.queue_next == null);
132		el.gpa.free(fiber.allocatedSlice());
	496	pub fn allocator(ev: *Evented) std.mem.Allocator {
	497	return if (ev.backing_allocator_needs_mutex) .{
	498	.ptr = ev,
	499	.vtable = &.{
	500	.alloc = alloc,
	501	.resize = resize,
	502	.remap = remap,
	503	.free = free,
	504	},
	505	} else ev.backing_allocator;
	506	}
	507
	508	fn alloc(userdata: anyopaque, len: usize, alignment: std.mem.Alignment, ret_addr: usize) ?[]u8 {
	509	const ev: *Evented = @ptrCast(@alignCast(userdata));
	510	const ev_io = ev.io();
	511	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	512	defer ev.backing_allocator_mutex.unlock(ev_io);
	513	return ev.backing_allocator.rawAlloc(len, alignment, ret_addr);
133	514	}
134	515
135		pub fn io(el: *EventLoop) Io {
	516	fn resize(
	517	userdata: *anyopaque,
	518	memory: []u8,
	519	alignment: std.mem.Alignment,
	520	new_len: usize,
	521	ret_addr: usize,
	522	) bool {
	523	const ev: *Evented = @ptrCast(@alignCast(userdata));
	524	const ev_io = ev.io();
	525	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	526	defer ev.backing_allocator_mutex.unlock(ev_io);
	527	return ev.backing_allocator.rawResize(memory, alignment, new_len, ret_addr);
	528	}
	529
	530	fn remap(
	531	userdata: *anyopaque,
	532	memory: []u8,
	533	alignment: Alignment,
	534	new_len: usize,
	535	ret_addr: usize,
	536	) ?[*]u8 {
	537	const ev: *Evented = @ptrCast(@alignCast(userdata));
	538	const ev_io = ev.io();
	539	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	540	defer ev.backing_allocator_mutex.unlock(ev_io);
	541	return ev.backing_allocator.rawRemap(memory, alignment, new_len, ret_addr);
	542	}
	543
	544	fn free(userdata: *anyopaque, memory: []u8, alignment: std.mem.Alignment, ret_addr: usize) void {
	545	const ev: *Evented = @ptrCast(@alignCast(userdata));
	546	const ev_io = ev.io();
	547	ev.backing_allocator_mutex.lockUncancelable(ev_io);
	548	defer ev.backing_allocator_mutex.unlock(ev_io);
	549	return ev.backing_allocator.rawFree(memory, alignment, ret_addr);
	550	}
	551
	552	pub fn io(ev: *Evented) Io {
136	553	return .{
137		.userdata = el,
	554	.userdata = ev,
138	555	.vtable = &.{
139	556	.async = async,
140	557	.concurrent = concurrent,
141	558	.await = await,
142		.select = select,
143	559	.cancel = cancel,
144		.cancelRequested = cancelRequested,
145	560
146		.mutexLock = mutexLock,
147		.mutexUnlock = mutexUnlock,
	561	.groupAsync = groupAsync,
	562	.groupConcurrent = groupConcurrent,
	563	.groupAwait = groupAwait,
	564	.groupCancel = groupCancel,
	565
	566	.recancel = recancel,
	567	.swapCancelProtection = swapCancelProtection,
	568	.checkCancel = checkCancel,
148	569
149		.conditionWait = conditionWait,
150		.conditionWake = conditionWake,
	570	.select = select,
	571
	572	.futexWait = futexWait,
	573	.futexWaitUncancelable = futexWaitUncancelable,
	574	.futexWake = futexWake,
	575
	576	.operate = operate,
	577	.batchAwaitAsync = batchAwaitAsync,
	578	.batchAwaitConcurrent = batchAwaitConcurrent,
	579	.batchCancel = batchCancel,
	580
	581	.dirCreateDir = dirCreateDir,
	582	.dirCreateDirPath = dirCreateDirPath,
	583	.dirCreateDirPathOpen = dirCreateDirPathOpen,
	584	.dirOpenDir = dirOpenDir,
	585	.dirStat = dirStat,
	586	.dirStatFile = dirStatFile,
	587	.dirAccess = dirAccess,
	588	.dirCreateFile = dirCreateFile,
	589	.dirCreateFileAtomic = dirCreateFileAtomic,
	590	.dirOpenFile = dirOpenFile,
	591	.dirClose = dirClose,
	592	.dirRead = dirRead,
	593	.dirRealPath = dirRealPath,
	594	.dirRealPathFile = dirRealPathFile,
	595	.dirDeleteFile = dirDeleteFile,
	596	.dirDeleteDir = dirDeleteDir,
	597	.dirRename = dirRename,
	598	.dirRenamePreserve = dirRenamePreserve,
	599	.dirSymLink = dirSymLink,
	600	.dirReadLink = dirReadLink,
	601	.dirSetOwner = dirSetOwner,
	602	.dirSetFileOwner = dirSetFileOwner,
	603	.dirSetPermissions = dirSetPermissions,
	604	.dirSetFilePermissions = dirSetFilePermissions,
	605	.dirSetTimestamps = dirSetTimestamps,
	606	.dirHardLink = dirHardLink,
151	607
152		.createFile = createFile,
153		.fileOpen = fileOpen,
	608	.fileStat = fileStat,
	609	.fileLength = fileLength,
154	610	.fileClose = fileClose,
155		.pread = pread,
156		.pwrite = pwrite,
	611	.fileWritePositional = fileWritePositional,
	612	.fileWriteFileStreaming = fileWriteFileStreaming,
	613	.fileWriteFilePositional = fileWriteFilePositional,
	614	.fileReadPositional = fileReadPositional,
	615	.fileSeekBy = fileSeekBy,
	616	.fileSeekTo = fileSeekTo,
	617	.fileSync = fileSync,
	618	.fileIsTty = fileIsTty,
	619	.fileEnableAnsiEscapeCodes = fileEnableAnsiEscapeCodes,
	620	.fileSupportsAnsiEscapeCodes = fileIsTty,
	621	.fileSetLength = fileSetLength,
	622	.fileSetOwner = fileSetOwner,
	623	.fileSetPermissions = fileSetPermissions,
	624	.fileSetTimestamps = fileSetTimestamps,
	625	.fileLock = fileLock,
	626	.fileTryLock = fileTryLock,
	627	.fileUnlock = fileUnlock,
	628	.fileDowngradeLock = fileDowngradeLock,
	629	.fileRealPath = fileRealPath,
	630	.fileHardLink = fileHardLink,
	631
	632	.fileMemoryMapCreate = fileMemoryMapCreate,
	633	.fileMemoryMapDestroy = fileMemoryMapDestroy,
	634	.fileMemoryMapSetLength = fileMemoryMapSetLength,
	635	.fileMemoryMapRead = fileMemoryMapRead,
	636	.fileMemoryMapWrite = fileMemoryMapWrite,
	637
	638	.processExecutableOpen = processExecutableOpen,
	639	.processExecutablePath = processExecutablePath,
	640	.lockStderr = lockStderr,
	641	.tryLockStderr = tryLockStderr,
	642	.unlockStderr = unlockStderr,
	643	.processCurrentPath = processCurrentPath,
	644	.processSetCurrentDir = processSetCurrentDir,
	645	.processReplace = processReplace,
	646	.processReplacePath = processReplacePath,
	647	.processSpawn = processSpawn,
	648	.processSpawnPath = processSpawnPath,
	649	.childWait = childWait,
	650	.childKill = childKill,
	651
	652	.progressParentFile = progressParentFile,
157	653
158	654	.now = now,
	655	.clockResolution = clockResolution,
159	656	.sleep = sleep,
	657
	658	.random = random,
	659	.randomSecure = randomSecure,
	660
	661	.netListenIp = netListenIpUnavailable,
	662	.netAccept = netAcceptUnavailable,
	663	.netBindIp = netBindIp,
	664	.netConnectIp = netConnectIpUnavailable,
	665	.netListenUnix = netListenUnixUnavailable,
	666	.netConnectUnix = netConnectUnixUnavailable,
	667	.netSocketCreatePair = netSocketCreatePairUnavailable,
	668	.netSend = netSendUnavailable,
	669	.netReceive = netReceive,
	670	.netRead = netReadUnavailable,
	671	.netWrite = netWriteUnavailable,
	672	.netWriteFile = netWriteFileUnavailable,
	673	.netClose = netClose,
	674	.netShutdown = netShutdown,
	675	.netInterfaceNameResolve = netInterfaceNameResolveUnavailable,
	676	.netInterfaceName = netInterfaceNameUnavailable,
	677	.netLookup = netLookupUnavailable,
160	678	},
161	679	};
162	680	}
163	681
164		pub fn init(el: *EventLoop, gpa: Allocator) !void {
	682	fn fileMemoryMapSetLength(
	683	userdata: ?*anyopaque,
	684	mm: *File.MemoryMap,
	685	new_len: usize,
	686	) File.MemoryMap.SetLengthError!void {
	687	const ev: *Evented = @ptrCast(@alignCast(userdata));
	688	_ = ev;
	689	const page_size = std.heap.pageSize();
	690	const alignment: Alignment = .fromByteUnits(page_size);
	691	const page_align = std.heap.page_size_min;
	692	const old_memory = mm.memory;
	693
	694	if (alignment.forward(new_len) == alignment.forward(old_memory.len)) {
	695	mm.memory.len = new_len;
	696	return;
	697	}
	698	var cancel_region: CancelRegion = .init();
	699	defer cancel_region.deinit();
	700	const flags: linux.MREMAP = .{ .MAYMOVE = true };
	701	const addr_hint: ?[*]const u8 = null;
	702	const new_memory = while (true) {
	703	try cancel_region.await(.nothing);
	704	const rc = linux.mremap(old_memory.ptr, old_memory.len, new_len, flags, addr_hint);
	705	switch (linux.errno(rc)) {
	706	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..new_len],
	707	.INTR => continue,
	708	.AGAIN => return error.LockedMemoryLimitExceeded,
	709	.NOMEM => return error.OutOfMemory,
	710	.INVAL => \|err\| return errnoBug(err),
	711	.FAULT => \|err\| return errnoBug(err),
	712	else => \|err\| return unexpectedErrno(err),
	713	}
	714	};
	715	mm.memory = new_memory;
	716	}
	717
	718	fn fileMemoryMapRead(userdata: ?anyopaque, mm: File.MemoryMap) File.ReadPositionalError!void {
	719	const ev: *Evented = @ptrCast(@alignCast(userdata));
	720	_ = ev;
	721	_ = mm;
	722	}
	723
	724	fn fileMemoryMapWrite(userdata: ?anyopaque, mm: File.MemoryMap) File.WritePositionalError!void {
	725	const ev: *Evented = @ptrCast(@alignCast(userdata));
	726	_ = ev;
	727	_ = mm;
	728	}
	729
	730	pub const InitOptions = struct {
	731	backing_allocator_needs_mutex: bool = true,
	732
	733	/// Affects the following operations:
	734	/// * `processExecutablePath` on OpenBSD and Haiku.
	735	argv0: Argv0 = .empty,
	736	/// Affects the following operations:
	737	/// * `fileIsTty`
	738	/// * `processSpawn`, `processSpawnPath`, `processReplace`, `processReplacePath`
	739	environ: process.Environ,
	740	};
	741
	742	pub fn init(ev: *Evented, backing_allocator: Allocator, options: InitOptions) !void {
165	743	const threads_size = @max(std.Thread.getCpuCount() catch 1, 1) * @sizeOf(Thread);
166		const idle_stack_end_offset = std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
167		const allocated_slice = try gpa.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
168		errdefer gpa.free(allocated_slice);
169		el.* = .{
170		.gpa = gpa,
171		.mutex = .{},
	744	const idle_stack_end_offset =
	745	std.mem.alignForward(usize, threads_size + idle_stack_size, std.heap.page_size_max);
	746	const allocated_slice = try backing_allocator.alignedAlloc(u8, .of(Thread), idle_stack_end_offset);
	747	errdefer backing_allocator.free(allocated_slice);
	748	ev.* = .{
	749	.backing_allocator_needs_mutex = options.backing_allocator_needs_mutex,
	750	.backing_allocator_mutex = .init,
	751	.backing_allocator = backing_allocator,
172	752	.main_fiber_buffer = undefined,
173	753	.threads = .{
174	754	.allocated = @ptrCast(allocated_slice[0..threads_size]),
175	755	.reserved = 1,
176	756	.active = 1,
177	757	},
	758
	759	.stderr_mutex = .init,
	760	.stderr_writer = .{
	761	.io = ev.io(),
	762	.interface = Io.File.Writer.initInterface(&.{}),
	763	.file = .stderr(),
	764	.mode = .streaming,
	765	},
	766	.stderr_mode = .no_color,
	767	.stderr_writer_initialized = false,
	768
	769	.environ_mutex = .init,
	770	.environ = .{ .process_environ = options.environ },
	771
	772	.null_fd = .init,
	773	.random_fd = .init,
	774
	775	.csprng_mutex = .init,
	776	.csprng = .uninitialized,
178	777	};
179		const main_fiber: *Fiber = @ptrCast(&el.main_fiber_buffer);
	778	const main_fiber: *Fiber = @ptrCast(&ev.main_fiber_buffer);
180	779	main_fiber.* = .{
181	780	.required_align = {},
182	781	.context = undefined,
183		.awaiter = null,
184		.queue_next = null,
185		.cancel_thread = null,
186		.awaiting_completions = .initEmpty(),
	782	.await_count = 0,
	783	.link = .{ .awaiter = null },
	784	.status = .{ .queue_next = null },
	785	.cancel_status = .unrequested,
	786	.cancel_protection = .unblocked,
	787	.name = if (tracy.enable) "main task",
187	788	};
188		const main_thread = &el.threads.allocated[0];
	789	const main_thread = &ev.threads.allocated[0];
189	790	Thread.self = main_thread;
190		const idle_stack_end: [*]align(16) usize = @ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
191		(idle_stack_end - 1)[0..1].* = .{@intFromPtr(el)};
	791	const idle_stack_end: [*]align(16) usize =
	792	@ptrCast(@alignCast(allocated_slice[idle_stack_end_offset..].ptr));
	793	(idle_stack_end - 1)[0..1].* = .{@intFromPtr(ev)};
192	794	main_thread.* = .{
	795	.required_align = {},
193	796	.thread = undefined,
194	797	.idle_context = switch (builtin.cpu.arch) {
195	798	.aarch64 => .{
...	...	@@ -206,42 +809,58 @@ pub fn init(el: *EventLoop, gpa: Allocator) !void {
206	809	},
207	810	.current_context = &main_fiber.context,
208	811	.ready_queue = null,
209		.io_uring = try IoUring.init(io_uring_entries, 0),
	812	.io_uring = try .init(
	813	io_uring_entries,
	814	linux.IORING_SETUP_COOP_TASKRUN \| linux.IORING_SETUP_SINGLE_ISSUER,
	815	),
210	816	.idle_search_index = 1,
211	817	.steal_ready_search_index = 1,
	818	.name_arena = .{},
	819	.csprng = .uninitialized,
212	820	};
213	821	errdefer main_thread.io_uring.deinit();
214		std.log.debug("created main idle {*}", .{&main_thread.idle_context});
215		std.log.debug("created main {*}", .{main_fiber});
	822	log.debug("created main idle {*}", .{&main_thread.idle_context});
	823	log.debug("created main {*}", .{main_fiber});
	824	if (tracy.enable) tracy.fiberEnter(main_fiber.name);
216	825	}
217	826
218		pub fn deinit(el: *EventLoop) void {
219		const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);
220		for (el.threads.allocated[0..active_threads]) \|*thread\| {
	827	pub fn deinit(ev: *Evented) void {
	828	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
	829	for (ev.threads.allocated[0..active_threads]) \|*thread\| {
221	830	const ready_fiber = @atomicLoad(?*Fiber, &thread.ready_queue, .monotonic);
222	831	assert(ready_fiber == null or ready_fiber == Fiber.finished); // pending async
223	832	}
224		el.yield(null, .exit);
225		const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(el.threads.allocated.ptr));
226		const idle_stack_end_offset = std.mem.alignForward(usize, el.threads.allocated.len * @sizeOf(Thread) + idle_stack_size, std.heap.page_size_max);
227		for (el.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
228		el.gpa.free(allocated_ptr[0..idle_stack_end_offset]);
229		el.* = undefined;
	833	ev.yield(null, .exit);
	834	ev.threads.allocated[0].io_uring.deinit();
	835	ev.null_fd.close();
	836	ev.random_fd.close();
	837	const allocated_ptr: [*]align(@alignOf(Thread)) u8 = @ptrCast(@alignCast(ev.threads.allocated.ptr));
	838	const idle_stack_end_offset = std.mem.alignForward(
	839	usize,
	840	ev.threads.allocated.len * @sizeOf(Thread) + idle_stack_size,
	841	std.heap.page_size_max,
	842	);
	843	for (ev.threads.allocated[1..active_threads]) \|*thread\| thread.thread.join();
	844	assert(active_threads == ev.threads.active); // spawned threads while there was no pending async?
	845	ev.backing_allocator.free(allocated_ptr[0..idle_stack_end_offset]);
	846	ev.* = undefined;
230	847	}
231	848
232		fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
	849	fn findReadyFiber(ev: Evented, thread: Thread) ?*Fiber {
233	850	if (@atomicRmw(?*Fiber, &thread.ready_queue, .Xchg, Fiber.finished, .acquire)) \|ready_fiber\| {
234		@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release);
235		ready_fiber.queue_next = null;
	851	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
	852	ready_fiber.status.queue_next = null;
236	853	return ready_fiber;
237	854	}
238		const active_threads = @atomicLoad(u32, &el.threads.active, .acquire);
	855	const active_threads = @atomicLoad(u32, &ev.threads.active, .acquire);
239	856	for (0..@min(max_steal_ready_search, active_threads)) \|_\| {
240	857	defer thread.steal_ready_search_index += 1;
241	858	if (thread.steal_ready_search_index == active_threads) thread.steal_ready_search_index = 0;
242		const steal_ready_search_thread = &el.threads.allocated[0..active_threads][thread.steal_ready_search_index];
	859	const steal_ready_search_thread =
	860	&ev.threads.allocated[0..active_threads][thread.steal_ready_search_index];
243	861	if (steal_ready_search_thread == thread) continue;
244		const ready_fiber = @atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue;
	862	const ready_fiber =
	863	@atomicLoad(?*Fiber, &steal_ready_search_thread.ready_queue, .acquire) orelse continue;
245	864	if (ready_fiber == Fiber.finished) continue;
246	865	if (@cmpxchgWeak(
247	866	?*Fiber,
...	...	@@ -251,8 +870,8 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
251	870	.acquire,
252	871	.monotonic,
253	872	)) \|_\| continue;
254		@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.queue_next, .release);
255		ready_fiber.queue_next = null;
	873	@atomicStore(?*Fiber, &thread.ready_queue, ready_fiber.status.queue_next, .release);
	874	ready_fiber.status.queue_next = null;
256	875	return ready_fiber;
257	876	}
258	877	// couldn't find anything to do, so we are now open for business
...	...	@@ -260,9 +879,9 @@ fn findReadyFiber(el: EventLoop, thread: Thread) ?*Fiber {
260	879	return null;
261	880	}
262	881
263		fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
	882	fn yield(ev: Evented, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage.PendingTask) void {
264	883	const thread: *Thread = .current();
265		const ready_context = if (maybe_ready_fiber orelse el.findReadyFiber(thread)) \|ready_fiber\|
	884	const ready_context = if (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\|
266	885	&ready_fiber.context
267	886	else
268	887	&thread.idle_context;
...	...	@@ -273,25 +892,25 @@ fn yield(el: EventLoop, maybe_ready_fiber: ?Fiber, pending_task: SwitchMessage
273	892	},
274	893	.pending_task = pending_task,
275	894	};
276		std.log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });
277		contextSwitch(&message).handle(el);
	895	log.debug("switching from {} to {}", .{ message.contexts.prev, message.contexts.ready });
	896	contextSwitch(&message).handle(ev);
278	897	}
279	898
280		fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
	899	fn schedule(ev: Evented, thread: Thread, ready_queue: Fiber.Queue) bool {
281	900	{
282	901	var fiber = ready_queue.head;
283	902	while (true) {
284		std.log.debug("scheduling {*}", .{fiber});
285		fiber = fiber.queue_next orelse break;
	903	log.debug("scheduling {*}", .{fiber});
	904	fiber = fiber.status.queue_next orelse break;
286	905	}
287	906	assert(fiber == ready_queue.tail);
288	907	}
289	908	// shared fields of previous `Thread` must be initialized before later ones are marked as active
290		const new_thread_index = @atomicLoad(u32, &el.threads.active, .acquire);
	909	const new_thread_index = @atomicLoad(u32, &ev.threads.active, .acquire);
291	910	for (0..@min(max_idle_search, new_thread_index)) \|_\| {
292	911	defer thread.idle_search_index += 1;
293	912	if (thread.idle_search_index == new_thread_index) thread.idle_search_index = 0;
294		const idle_search_thread = &el.threads.allocated[0..new_thread_index][thread.idle_search_index];
	913	const idle_search_thread = &ev.threads.allocated[0..new_thread_index][thread.idle_search_index];
295	914	if (idle_search_thread == thread) continue;
296	915	if (@cmpxchgWeak(
297	916	?*Fiber,
...	...	@@ -301,13 +920,13 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
301	920	.release,
302	921	.monotonic,
303	922	)) \|_\| continue;
304		getSqe(&thread.io_uring).* = .{
	923	thread.enqueue().* = .{
305	924	.opcode = .MSG_RING,
306		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
	925	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
307	926	.ioprio = 0,
308	927	.fd = idle_search_thread.io_uring.fd,
309	928	.off = @intFromEnum(Completion.UserData.wakeup),
310		.addr = 0,
	929	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
311	930	.len = 0,
312	931	.rw_flags = 0,
313	932	.user_data = @intFromEnum(Completion.UserData.wakeup),
...	...	@@ -317,145 +936,222 @@ fn schedule(el: EventLoop, thread: Thread, ready_queue: Fiber.Queue) void {
317	936	.addr3 = 0,
318	937	.resv = 0,
319	938	};
320		return;
	939	return true;
321	940	}
322	941	spawn_thread: {
323	942	// previous failed reservations must have completed before retrying
324		if (new_thread_index == el.threads.allocated.len or @cmpxchgWeak(
	943	if (new_thread_index == ev.threads.allocated.len or @cmpxchgWeak(
325	944	u32,
326		&el.threads.reserved,
	945	&ev.threads.reserved,
327	946	new_thread_index,
328	947	new_thread_index + 1,
329	948	.acquire,
330	949	.monotonic,
331	950	) != null) break :spawn_thread;
332		const new_thread = &el.threads.allocated[new_thread_index];
	951	const new_thread = &ev.threads.allocated[new_thread_index];
333	952	const next_thread_index = new_thread_index + 1;
	953	var params = std.mem.zeroInit(linux.io_uring_params, .{
	954	.flags = linux.IORING_SETUP_ATTACH_WQ \|
	955	linux.IORING_SETUP_R_DISABLED \|
	956	linux.IORING_SETUP_COOP_TASKRUN \|
	957	linux.IORING_SETUP_SINGLE_ISSUER,
	958	.wq_fd = @as(u32, @intCast(ev.threads.allocated[0].io_uring.fd)),
	959	});
334	960	new_thread.* = .{
	961	.required_align = {},
335	962	.thread = undefined,
336	963	.idle_context = undefined,
337	964	.current_context = &new_thread.idle_context,
338	965	.ready_queue = ready_queue.head,
339		.io_uring = IoUring.init(io_uring_entries, 0) catch \|err\| {
340		@atomicStore(u32, &el.threads.reserved, new_thread_index, .release);
	966	.io_uring = IoUring.init_params(io_uring_entries, &params) catch \|err\| {
	967	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
341	968	// no more access to `thread` after giving up reservation
342		std.log.warn("unable to create worker thread due to io_uring init failure: {s}", .{@errorName(err)});
	969	log.warn("unable to create worker thread due to io_uring init failure: {s}", .{
	970	@errorName(err),
	971	});
343	972	break :spawn_thread;
344	973	},
345	974	.idle_search_index = 0,
346	975	.steal_ready_search_index = 0,
	976	.name_arena = .{},
	977	.csprng = .uninitialized,
347	978	};
348	979	new_thread.thread = std.Thread.spawn(.{
349	980	.stack_size = idle_stack_size,
350		.allocator = el.gpa,
351		}, threadEntry, .{ el, new_thread_index }) catch \|err\| {
	981	.allocator = ev.allocator(),
	982	}, threadEntry, .{ ev, new_thread_index }) catch \|err\| {
352	983	new_thread.io_uring.deinit();
353		@atomicStore(u32, &el.threads.reserved, new_thread_index, .release);
	984	@atomicStore(u32, &ev.threads.reserved, new_thread_index, .release);
354	985	// no more access to `thread` after giving up reservation
355		std.log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
	986	log.warn("unable to create worker thread due spawn failure: {s}", .{@errorName(err)});
356	987	break :spawn_thread;
357	988	};
358	989	// shared fields of `Thread` must be initialized before being marked active
359		@atomicStore(u32, &el.threads.active, next_thread_index, .release);
360		return;
	990	@atomicStore(u32, &ev.threads.active, next_thread_index, .release);
	991	return false;
361	992	}
362	993	// nobody wanted it, so just queue it on ourselves
363	994	while (@cmpxchgWeak(
364	995	?*Fiber,
365	996	&thread.ready_queue,
366		ready_queue.tail.queue_next,
	997	ready_queue.tail.status.queue_next,
367	998	ready_queue.head,
368	999	.acq_rel,
369	1000	.acquire,
370		)) \|old_head\| ready_queue.tail.queue_next = old_head;
	1001	)) \|old_head\| ready_queue.tail.status.queue_next = old_head;
	1002	return false;
371	1003	}
372	1004
373		fn mainIdle(el: EventLoop, message: const SwitchMessage) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
374		message.handle(el);
375		el.idle(&el.threads.allocated[0]);
376		el.yield(@ptrCast(&el.main_fiber_buffer), .nothing);
	1005	fn mainIdle(
	1006	ev: *Evented,
	1007	message: *const SwitchMessage,
	1008	) callconv(.withStackAlign(.c, @max(@alignOf(Thread), @alignOf(Context)))) noreturn {
	1009	message.handle(ev);
	1010	ev.idle(&ev.threads.allocated[0]);
	1011	ev.yield(@ptrCast(&ev.main_fiber_buffer), .nothing);
377	1012	unreachable; // switched to dead fiber
378	1013	}
379	1014
380		fn threadEntry(el: *EventLoop, index: u32) void {
381		const thread: *Thread = &el.threads.allocated[index];
	1015	fn threadEntry(ev: *Evented, index: u32) void {
	1016	const thread: *Thread = &ev.threads.allocated[index];
382	1017	Thread.self = thread;
383		std.log.debug("created thread idle {*}", .{&thread.idle_context});
384		el.idle(thread);
	1018	defer thread.io_uring.deinit();
	1019	log.debug("created thread idle {*}", .{&thread.idle_context});
	1020	switch (linux.errno(linux.io_uring_register(thread.io_uring.fd, .REGISTER_ENABLE_RINGS, null, 0))) {
	1021	.SUCCESS => ev.idle(thread),
	1022	else => \|err\| @panic(@tagName(err)),
	1023	}
385	1024	}
386	1025
387	1026	const Completion = struct {
	1027	result: i32,
	1028	flags: u32,
	1029
388	1030	const UserData = enum(usize) {
389	1031	unused,
390	1032	wakeup,
	1033	futex_wake,
391	1034	cleanup,
392	1035	exit,
393		/// *Fiber
	1036	/// If bit 0 is 1, a pointer to the `context` field of `Io.Batch.Storage.Pending`.
	1037	/// If bits 0 and 1 are 0, a `*Fiber`.
394	1038	_,
395	1039	};
396		result: i32,
397		flags: u32,
	1040
	1041	fn errno(completion: Completion) linux.E {
	1042	return linux.errno(@bitCast(@as(isize, completion.result)));
	1043	}
398	1044	};
399	1045
400		fn idle(el: EventLoop, thread: Thread) void {
	1046	fn idle(ev: Evented, thread: Thread) void {
401	1047	var maybe_ready_fiber: ?*Fiber = null;
402	1048	while (true) {
403		while (maybe_ready_fiber orelse el.findReadyFiber(thread)) \|ready_fiber\| {
404		el.yield(ready_fiber, .nothing);
	1049	while (maybe_ready_fiber orelse ev.findReadyFiber(thread)) \|ready_fiber\| {
	1050	ev.yield(ready_fiber, .nothing);
405	1051	maybe_ready_fiber = null;
406	1052	}
407	1053	_ = thread.io_uring.submit_and_wait(1) catch \|err\| switch (err) {
408		error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}),
	1054	error.SignalInterrupt => {},
409	1055	else => \|e\| @panic(@errorName(e)),
410	1056	};
411		var cqes_buffer: [io_uring_entries]std.os.linux.io_uring_cqe = undefined;
	1057	var cqes_buffer: [io_uring_entries]linux.io_uring_cqe = undefined;
412	1058	var maybe_ready_queue: ?Fiber.Queue = null;
413	1059	for (cqes_buffer[0 .. thread.io_uring.copy_cqes(&cqes_buffer, 0) catch \|err\| switch (err) {
414		error.SignalInterrupt => cqes_len: {
415		std.log.warn("copy_cqes failed with SignalInterrupt", .{});
416		break :cqes_len 0;
417		},
	1060	error.SignalInterrupt => 0,
418	1061	else => \|e\| @panic(@errorName(e)),
419		}]) \|cqe\| switch (@as(Completion.UserData, @enumFromInt(cqe.user_data))) {
	1062	}]) \|cqe\| if (cqe.flags & linux.IORING_CQE_F_SKIP == 0) switch (@as(
	1063	Completion.UserData,
	1064	@enumFromInt(cqe.user_data),
	1065	)) {
420	1066	.unused => unreachable, // bad submission queued?
421	1067	.wakeup => {},
	1068	.futex_wake => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1069	.SUCCESS => recoverableOsBugDetected(), // success is skipped
	1070	.INVAL => {}, // invalid futex_wait() on ptr done elsewhere
	1071	.INTR, .CANCELED => recoverableOsBugDetected(), // `Completion.UserData.futex_wake` is not cancelable
	1072	.FAULT => {}, // pointer became invalid while doing the wake
	1073	else => recoverableOsBugDetected(), // deadlock due to operating system bug
	1074	},
422	1075	.cleanup => @panic("failed to notify other threads that we are exiting"),
423	1076	.exit => {
424	1077	assert(maybe_ready_fiber == null and maybe_ready_queue == null); // pending async
425	1078	return;
426	1079	},
427		_ => switch (errno(cqe.res)) {
428		.INTR => getSqe(&thread.io_uring).* = .{
429		.opcode = .ASYNC_CANCEL,
430		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
431		.ioprio = 0,
432		.fd = 0,
433		.off = 0,
434		.addr = cqe.user_data,
435		.len = 0,
436		.rw_flags = 0,
437		.user_data = @intFromEnum(Completion.UserData.wakeup),
438		.buf_index = 0,
439		.personality = 0,
440		.splice_fd_in = 0,
441		.addr3 = 0,
442		.resv = 0,
443		},
444		else => {
445		const fiber: *Fiber = @ptrFromInt(cqe.user_data);
446		assert(fiber.queue_next == null);
447		fiber.resultPointer(Completion).* = .{
	1080	_ => if (@as(?*Fiber, ready_fiber: switch (@as(u2, @truncate(cqe.user_data))) {
	1081	0b00 => {
	1082	const ready_fiber: *Fiber = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1083	ready_fiber.resultPointer(Completion).* = .{
448	1084	.result = cqe.res,
449	1085	.flags = cqe.flags,
450	1086	};
451		if (maybe_ready_fiber == null) maybe_ready_fiber = fiber else if (maybe_ready_queue) \|*ready_queue\| {
452		ready_queue.tail.queue_next = fiber;
453		ready_queue.tail = fiber;
454		} else maybe_ready_queue = .{ .head = fiber, .tail = fiber };
	1087	break :ready_fiber ready_fiber;
	1088	},
	1089	0b01 => {
	1090	thread.enqueue().* = .{
	1091	.opcode = .ASYNC_CANCEL,
	1092	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	1093	.ioprio = 0,
	1094	.fd = 0,
	1095	.off = 0,
	1096	.addr = cqe.user_data & ~@as(usize, 0b11),
	1097	.len = 0,
	1098	.rw_flags = 0,
	1099	.user_data = @intFromEnum(Completion.UserData.wakeup),
	1100	.buf_index = 0,
	1101	.personality = 0,
	1102	.splice_fd_in = 0,
	1103	.addr3 = 0,
	1104	.resv = 0,
	1105	};
	1106	break :ready_fiber null;
	1107	},
	1108	0b10 => {
	1109	const context: *Io.Operation.Storage.Pending.Context =
	1110	@ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1111	const batch: *Io.Batch = @ptrFromInt(context[0]);
	1112	var next: usize = 0b00;
	1113	context[0..3].* = .{ next, @as(u32, @bitCast(cqe.res)), cqe.flags };
	1114	while (true) {
	1115	next = @cmpxchgWeak(
	1116	usize,
	1117	@as(*usize, @ptrCast(&batch.context)),
	1118	next,
	1119	cqe.user_data,
	1120	.release,
	1121	.acquire,
	1122	) orelse break;
	1123	context[0] = next;
	1124	}
	1125	break :ready_fiber switch (@as(u2, @truncate(next))) {
	1126	0b00, 0b01 => @ptrFromInt(next & ~@as(usize, 0b11)),
	1127	0b10, 0b11 => null,
	1128	};
	1129	},
	1130	0b11 => switch (Completion.errno(.{ .result = cqe.res, .flags = cqe.flags })) {
	1131	.SUCCESS => unreachable, // no event count specified
	1132	.TIME => {
	1133	const context: *usize = @ptrFromInt(cqe.user_data & ~@as(usize, 0b11));
	1134	const fiber = @atomicRmw(usize, context, .Add, 0b01, .acquire);
	1135	break :ready_fiber switch (@as(u2, @truncate(fiber))) {
	1136	else => unreachable, // timeout completed multiple times
	1137	0b00 => @ptrFromInt(fiber & ~@as(usize, 0b11)),
	1138	0b10 => null,
	1139	};
	1140	},
	1141	.CANCELED => null, // user data may have been invalidated
	1142	else => \|err\| unexpectedErrno(err) catch null,
455	1143	},
	1144	})) \|ready_fiber\| {
	1145	assert(ready_fiber.status.queue_next == null);
	1146	if (maybe_ready_fiber == null) {
	1147	maybe_ready_fiber = ready_fiber;
	1148	} else if (maybe_ready_queue) \|*ready_queue\| {
	1149	ready_queue.tail.status.queue_next = ready_fiber;
	1150	ready_queue.tail = ready_fiber;
	1151	} else maybe_ready_queue = .{ .head = ready_fiber, .tail = ready_fiber };
456	1152	},
457	1153	};
458		if (maybe_ready_queue) \|ready_queue\| el.schedule(thread, ready_queue);
	1154	if (maybe_ready_queue) \|ready_queue\| _ = ev.schedule(thread, ready_queue);
459	1155	}
460	1156	}
461	1157
...	...	@@ -469,113 +1165,74 @@ const SwitchMessage = struct {
469	1165	const PendingTask = union(enum) {
470	1166	nothing,
471	1167	reschedule,
472		recycle: *Fiber,
473		register_awaiter: ?Fiber,
474		register_select: []const *Io.AnyFuture,
475		mutex_lock: struct {
476		prev_state: Io.Mutex.State,
477		mutex: *Io.Mutex,
478		},
479		condition_wait: struct {
480		cond: *Io.Condition,
481		mutex: *Io.Mutex,
482		},
	1168	await: u31,
	1169	group_await: Group,
	1170	group_cancel: Group,
	1171	batch_await: *Io.Batch,
	1172	destroy,
483	1173	exit,
484	1174	};
485	1175
486		fn handle(message: const SwitchMessage, el: EventLoop) void {
	1176	fn handle(message: const SwitchMessage, ev: Evented) void {
487	1177	const thread: *Thread = .current();
488	1178	thread.current_context = message.contexts.ready;
	1179	if (tracy.enable) {
	1180	if (message.contexts.ready != &thread.idle_context) {
	1181	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.ready));
	1182	tracy.fiberEnter(fiber.name);
	1183	} else tracy.fiberLeave();
	1184	}
489	1185	switch (message.pending_task) {
490	1186	.nothing => {},
491	1187	.reschedule => if (message.contexts.prev != &thread.idle_context) {
492		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
493		assert(prev_fiber.queue_next == null);
494		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
	1188	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1189	assert(fiber.status.queue_next == null);
	1190	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
495	1191	},
496		.recycle => \|fiber\| {
497		el.recycle(fiber);
	1192	.await => \|count\| {
	1193	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1194	if (@atomicRmw(i32, &fiber.await_count, .Sub, count, .monotonic) > 0)
	1195	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
498	1196	},
499		.register_awaiter => \|awaiter\| {
500		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
501		assert(prev_fiber.queue_next == null);
502		if (@atomicRmw(?*Fiber, awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished)
503		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
	1197	.group_await => \|group\| {
	1198	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1199	if (group.await(ev, fiber))
	1200	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
504	1201	},
505		.register_select => \|futures\| {
506		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
507		assert(prev_fiber.queue_next == null);
508		for (futures) \|any_future\| {
509		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
510		if (@atomicRmw(?*Fiber, &future_fiber.awaiter, .Xchg, prev_fiber, .acq_rel) == Fiber.finished) {
511		const closure: *AsyncClosure = .fromFiber(future_fiber);
512		if (!@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .seq_cst)) {
513		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
514		}
515		}
516		}
517		},
518		.mutex_lock => \|mutex_lock\| {
519		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
520		assert(prev_fiber.queue_next == null);
521		var prev_state = mutex_lock.prev_state;
522		while (switch (prev_state) {
523		else => next_state: {
524		prev_fiber.queue_next = @ptrFromInt(@intFromEnum(prev_state));
525		break :next_state @cmpxchgWeak(
526		Io.Mutex.State,
527		&mutex_lock.mutex.state,
528		prev_state,
529		@enumFromInt(@intFromPtr(prev_fiber)),
530		.release,
531		.acquire,
532		);
533		},
534		.unlocked => @cmpxchgWeak(
535		Io.Mutex.State,
536		&mutex_lock.mutex.state,
537		.unlocked,
538		.locked_once,
539		.acquire,
540		.acquire,
541		) orelse {
542		prev_fiber.queue_next = null;
543		el.schedule(thread, .{ .head = prev_fiber, .tail = prev_fiber });
544		return;
545		},
546		}) \|next_state\| prev_state = next_state;
	1202	.group_cancel => \|group\| {
	1203	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1204	if (group.cancel(ev, fiber))
	1205	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
547	1206	},
548		.condition_wait => \|condition_wait\| {
549		const prev_fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
550		assert(prev_fiber.queue_next == null);
551		const cond_impl = prev_fiber.resultPointer(ConditionImpl);
552		cond_impl.* = .{
553		.tail = prev_fiber,
554		.event = .queued,
555		};
	1207	.batch_await => \|batch\| {
	1208	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
556	1209	if (@cmpxchgStrong(
557		?*Fiber,
558		@as(?Fiber, @ptrCast(&condition_wait.cond.state)),
	1210	?*anyopaque,
	1211	&batch.context,
559	1212	null,
560		prev_fiber,
	1213	fiber,
561	1214	.release,
562		.acquire,
563		)) \|waiting_fiber\| {
564		const waiting_cond_impl = waiting_fiber.?.resultPointer(ConditionImpl);
565		assert(waiting_cond_impl.tail.queue_next == null);
566		waiting_cond_impl.tail.queue_next = prev_fiber;
567		waiting_cond_impl.tail = prev_fiber;
	1215	.monotonic,
	1216	)) \|head\| {
	1217	assert(@as(u2, @truncate(@intFromPtr(head))) != 0b00);
	1218	_ = ev.schedule(thread, .{ .head = fiber, .tail = fiber });
568	1219	}
569		condition_wait.mutex.unlock(el.io());
570	1220	},
571		.exit => for (el.threads.allocated[0..@atomicLoad(u32, &el.threads.active, .acquire)]) \|*each_thread\| {
572		getSqe(&thread.io_uring).* = .{
	1221	.destroy => {
	1222	const fiber: *Fiber = @alignCast(@fieldParentPtr("context", message.contexts.prev));
	1223	fiber.destroy(ev.backing_allocator);
	1224	ev.backing_allocator_mutex.unlock(ev.io());
	1225	},
	1226	.exit => for (
	1227	ev.threads.allocated[0..@atomicLoad(u32, &ev.threads.active, .acquire)],
	1228	) \|*each_thread\| {
	1229	thread.enqueue().* = .{
573	1230	.opcode = .MSG_RING,
574		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
	1231	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
575	1232	.ioprio = 0,
576	1233	.fd = each_thread.io_uring.fd,
577	1234	.off = @intFromEnum(Completion.UserData.exit),
578		.addr = 0,
	1235	.addr = @intFromEnum(linux.IORING_MSG_RING_COMMAND.DATA),
579	1236	.len = 0,
580	1237	.rw_flags = 0,
581	1238	.user_data = @intFromEnum(Completion.UserData.cleanup),
...	...	@@ -784,66 +1441,74 @@ inline fn contextSwitch(message: const SwitchMessage) const SwitchMessage {
784	1441
785	1442	fn mainIdleEntry() callconv(.naked) void {
786	1443	switch (builtin.cpu.arch) {
787		.x86_64 => asm volatile (
788		\\ movq (%%rsp), %%rdi
789		\\ jmp %[mainIdle:P]
790		:
791		: [mainIdle] "X" (&mainIdle),
792		),
793	1444	.aarch64 => asm volatile (
794	1445	\\ ldr x0, [sp, #-8]
795	1446	\\ b %[mainIdle]
796	1447	:
797	1448	: [mainIdle] "X" (&mainIdle),
798	1449	),
799		else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
800		}
801		}
802
803		fn fiberEntry() callconv(.naked) void {
804		switch (builtin.cpu.arch) {
805	1450	.x86_64 => asm volatile (
806		\\ leaq 8(%%rsp), %%rdi
807		\\ jmp %[AsyncClosure_call:P]
	1451	\\ movq (%%rsp), %%rdi
	1452	\\ jmp %[mainIdle:P]
808	1453	:
809		: [AsyncClosure_call] "X" (&AsyncClosure.call),
	1454	: [mainIdle] "X" (&mainIdle),
810	1455	),
811	1456	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
812	1457	}
813	1458	}
814	1459
815	1460	const AsyncClosure = struct {
816		event_loop: *EventLoop,
	1461	ev: *Evented,
817	1462	fiber: *Fiber,
818	1463	start: const fn (context: const anyopaque, result: *anyopaque) void,
819	1464	result_align: Alignment,
820		already_awaited: bool,
	1465
	1466	fn fromFiber(fiber: Fiber) AsyncClosure {
	1467	return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
	1468	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1469	) - @sizeOf(AsyncClosure));
	1470	}
821	1471
822	1472	fn contextPointer(closure: AsyncClosure) []align(Fiber.max_context_align.toByteUnits()) u8 {
823	1473	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(AsyncClosure));
824	1474	}
825	1475
826		fn call(closure: AsyncClosure, message: const SwitchMessage) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
827		message.handle(closure.event_loop);
	1476	fn entry() callconv(.naked) void {
	1477	switch (builtin.cpu.arch) {
	1478	.aarch64 => asm volatile (
	1479	\\ mov x0, sp
	1480	\\ b %[call]
	1481	:
	1482	: [call] "X" (&call),
	1483	),
	1484	.x86_64 => asm volatile (
	1485	\\ leaq 8(%%rsp), %%rdi
	1486	\\ jmp %[call:P]
	1487	:
	1488	: [call] "X" (&call),
	1489	),
	1490	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1491	}
	1492	}
	1493
	1494	fn call(
	1495	closure: *AsyncClosure,
	1496	message: *const SwitchMessage,
	1497	) callconv(.withStackAlign(.c, @alignOf(AsyncClosure))) noreturn {
	1498	message.handle(closure.ev);
828	1499	const fiber = closure.fiber;
829		std.log.debug("{*} performing async", .{fiber});
	1500	log.debug("{*} performing async", .{fiber});
830	1501	closure.start(closure.contextPointer(), fiber.resultBytes(closure.result_align));
831		const awaiter = @atomicRmw(?*Fiber, &fiber.awaiter, .Xchg, Fiber.finished, .acq_rel);
832		const ready_awaiter = r: {
833		const a = awaiter orelse break :r null;
834		if (@atomicRmw(bool, &closure.already_awaited, .Xchg, true, .acq_rel)) break :r null;
835		break :r a;
836		};
837		closure.event_loop.yield(ready_awaiter, .nothing);
	1502	closure.ev.yield(
	1503	if (@atomicRmw(?*Fiber, &fiber.link.awaiter, .Xchg, Fiber.finished, .acq_rel)) \|awaiter\|
	1504	if (@atomicRmw(i32, &awaiter.await_count, .Add, 1, .monotonic) == -1) awaiter else null
	1505	else
	1506	null,
	1507	.nothing,
	1508	);
838	1509	unreachable; // switched to dead fiber
839	1510	}
840
841		fn fromFiber(fiber: Fiber) AsyncClosure {
842		return @ptrFromInt(Fiber.max_context_align.max(.of(AsyncClosure)).backward(
843		@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
844		) - @sizeOf(AsyncClosure));
845		}
846		};
	1511	};
847	1512
848	1513	fn async(
849	1514	userdata: ?*anyopaque,
...	...	@@ -853,7 +1518,8 @@ fn async(
853	1518	context_alignment: Alignment,
854	1519	start: const fn (context: const anyopaque, result: *anyopaque) void,
855	1520	) ?*std.Io.AnyFuture {
856		return concurrent(userdata, result.len, result_alignment, context, context_alignment, start) catch {
	1521	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1522	return concurrent(ev, result.len, result_alignment, context, context_alignment, start) catch {
857	1523	start(context.ptr, result.ptr);
858	1524	return null;
859	1525	};
...	...	@@ -872,626 +1538,4610 @@ fn concurrent(
872	1538	assert(result_len <= Fiber.max_result_size); // TODO
873	1539	assert(context.len <= Fiber.max_context_size); // TODO
874	1540
875		const event_loop: *EventLoop = @ptrCast(@alignCast(userdata));
876		const fiber = try Fiber.allocate(event_loop);
877		std.log.debug("allocated {*}", .{fiber});
	1541	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1542	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1543	error.OutOfMemory => return error.ConcurrencyUnavailable,
	1544	};
	1545	log.debug("allocated {*}", .{fiber});
878	1546
879	1547	const closure: *AsyncClosure = .fromFiber(fiber);
880	1548	fiber.* = .{
881	1549	.required_align = {},
882	1550	.context = switch (builtin.cpu.arch) {
883		.x86_64 => .{
884		.rsp = @intFromPtr(closure) - @sizeOf(usize),
885		.rbp = 0,
886		.rip = @intFromPtr(&fiberEntry),
887		},
888	1551	.aarch64 => .{
889	1552	.sp = @intFromPtr(closure),
890	1553	.fp = 0,
891		.pc = @intFromPtr(&fiberEntry),
	1554	.pc = @intFromPtr(&AsyncClosure.entry),
	1555	},
	1556	.x86_64 => .{
	1557	.rsp = @intFromPtr(closure) - @sizeOf(usize),
	1558	.rbp = 0,
	1559	.rip = @intFromPtr(&AsyncClosure.entry),
892	1560	},
893	1561	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
894	1562	},
895		.awaiter = null,
896		.queue_next = null,
897		.cancel_thread = null,
898		.awaiting_completions = .initEmpty(),
	1563	.await_count = 0,
	1564	.link = .{ .awaiter = null },
	1565	.status = .{ .queue_next = null },
	1566	.cancel_status = .unrequested,
	1567	.cancel_protection = .unblocked,
	1568	.name = if (tracy.enable) name: {
	1569	const thread: *Thread = .current();
	1570	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
	1571	defer thread.name_arena = name_arena.state;
	1572	break :name std.fmt.allocPrintSentinel(
	1573	name_arena.allocator(),
	1574	"task {d}",
	1575	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
	1576	0,
	1577	) catch return error.ConcurrencyUnavailable;
	1578	},
899	1579	};
900	1580	closure.* = .{
901		.event_loop = event_loop,
	1581	.ev = ev,
902	1582	.fiber = fiber,
903	1583	.start = start,
904	1584	.result_align = result_alignment,
905		.already_awaited = false,
906	1585	};
907	1586	@memcpy(closure.contextPointer(), context);
908	1587
909		event_loop.schedule(.current(), .{ .head = fiber, .tail = fiber });
	1588	const thread: *Thread = .current();
	1589	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
910	1590	return @ptrCast(fiber);
911	1591	}
912	1592
913	1593	fn await(
914	1594	userdata: ?*anyopaque,
915		any_future: *std.Io.AnyFuture,
	1595	future: *std.Io.AnyFuture,
916	1596	result: []u8,
917	1597	result_alignment: Alignment,
918	1598	) void {
919		const event_loop: *EventLoop = @ptrCast(@alignCast(userdata));
920		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
921		if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) != Fiber.finished)
922		event_loop.yield(null, .{ .register_awaiter = &future_fiber.awaiter });
	1599	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1600	const fiber = Thread.current().currentFiber();
	1601	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1602	if (@atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, fiber, .acq_rel)) \|awaiter\| {
	1603	assert(awaiter == Fiber.finished);
	1604	} else while (true) {
	1605	ev.yield(null, .{ .await = 1 });
	1606	const awaiter = @atomicLoad(?*Fiber, &future_fiber.link.awaiter, .acquire);
	1607	if (awaiter == Fiber.finished) break;
	1608	assert(awaiter == fiber); // spurious wakeup
	1609	}
923	1610	@memcpy(result, future_fiber.resultBytes(result_alignment));
924		event_loop.recycle(future_fiber);
	1611	future_fiber.destroy(ev.allocator());
925	1612	}
926	1613
927		fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) usize {
928		const el: *EventLoop = @ptrCast(@alignCast(userdata));
	1614	fn cancel(
	1615	userdata: ?*anyopaque,
	1616	future: *std.Io.AnyFuture,
	1617	result: []u8,
	1618	result_alignment: Alignment,
	1619	) void {
	1620	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1621	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	1622	future_fiber.requestCancel(ev);
	1623	await(ev, future, result, result_alignment);
	1624	}
	1625
	1626	const Group = struct {
	1627	ptr: *Io.Group,
	1628
	1629	const List = packed struct(usize) {
	1630	cancel_requested: bool,
	1631	awaiter_delayed: bool,
	1632	fibers: Fiber.PackedPtr,
	1633	};
	1634	fn listPtr(group: Group) *List {
	1635	return @ptrCast(&group.ptr.token);
	1636	}
	1637
	1638	const Mutex = packed struct(u32) {
	1639	locked: bool,
	1640	contended: bool,
	1641	shared2: u30,
	1642	};
	1643	fn mutexPtr(group: Group) *Mutex {
	1644	return switch (comptime builtin.cpu.arch.endian()) {
	1645	.little => @ptrCast(&group.ptr.state),
	1646	.big => @ptrCast(@alignCast(
	1647	@as([*]u8, @ptrCast(&group.ptr.state)) + @sizeOf(usize) - @sizeOf(u32),
	1648	)),
	1649	};
	1650	}
	1651
	1652	const Awaiter = packed struct(usize) {
	1653	locked: bool,
	1654	contended: bool,
	1655	awaiter: Fiber.PackedPtr,
	1656	};
	1657	fn awaiterPtr(group: Group) *Awaiter {
	1658	return @ptrCast(&group.ptr.state);
	1659	}
	1660
	1661	fn lock(group: Group, ev: *Evented) void {
	1662	const mutex = group.mutexPtr();
	1663	{
	1664	const old_state = @atomicRmw(
	1665	Mutex,
	1666	mutex,
	1667	.Or,
	1668	.{ .locked = true, .contended = false, .shared2 = 0 },
	1669	.acquire,
	1670	);
	1671	if (!old_state.locked) {
	1672	@branchHint(.likely);
	1673	return;
	1674	}
	1675	if (old_state.contended) {
	1676	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1677	}
	1678	}
	1679	while (true) {
	1680	var old_state = @atomicRmw(
	1681	Mutex,
	1682	mutex,
	1683	.Or,
	1684	.{ .locked = true, .contended = true, .shared2 = 0 },
	1685	.acquire,
	1686	);
	1687	if (!old_state.locked) {
	1688	@branchHint(.likely);
	1689	return;
	1690	}
	1691	old_state.contended = true;
	1692	futexWaitUncancelable(ev, @ptrCast(mutex), @bitCast(old_state));
	1693	}
	1694	}
	1695
	1696	fn unlock(group: Group, ev: *Evented) void {
	1697	const mutex = group.mutexPtr();
	1698	const old_state = @atomicRmw(
	1699	Mutex,
	1700	mutex,
	1701	.And,
	1702	.{ .locked = false, .contended = false, .shared2 = std.math.maxInt(u30) },
	1703	.release,
	1704	);
	1705	assert(old_state.locked);
	1706	if (old_state.contended) futexWake(ev, @ptrCast(mutex), 1);
	1707	}
929	1708
930		// Optimization to avoid the yield below.
931		for (futures, 0..) \|any_future, i\| {
932		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
933		if (@atomicLoad(?*Fiber, &future_fiber.awaiter, .acquire) == Fiber.finished)
934		return i;
	1709	fn addFiber(group: Group, ev: Evented, fiber: Fiber) void {
	1710	group.lock(ev);
	1711	defer group.unlock(ev);
	1712	const list_ptr = group.listPtr();
	1713	const list = @atomicLoad(List, list_ptr, .monotonic);
	1714	if (list.cancel_requested) fiber.cancel_status = .{ .requested = true, .awaiting = .nothing };
	1715	const old_head = list.fibers.unpack();
	1716	if (old_head) \|head\| head.link.group.prev = fiber;
	1717	fiber.link.group.next = old_head;
	1718	@atomicStore(List, list_ptr, .{
	1719	.cancel_requested = list.cancel_requested,
	1720	.awaiter_delayed = list.awaiter_delayed,
	1721	.fibers = .pack(fiber),
	1722	}, .monotonic);
935	1723	}
936	1724
937		el.yield(null, .{ .register_select = futures });
	1725	fn removeFiber(group: Group, ev: Evented, fiber: Fiber) ?*Fiber {
	1726	group.lock(ev);
	1727	defer group.unlock(ev);
	1728	const list_ptr = group.listPtr();
	1729	const list = @atomicLoad(List, list_ptr, .monotonic);
	1730	if (fiber.link.group.next) \|next\| next.link.group.prev = fiber.link.group.prev;
	1731	if (fiber.link.group.prev) \|prev\| {
	1732	prev.link.group.next = fiber.link.group.next;
	1733	} else if (fiber.link.group.next) \|new_head\| {
	1734	@atomicStore(List, list_ptr, .{
	1735	.cancel_requested = list.cancel_requested,
	1736	.awaiter_delayed = list.awaiter_delayed,
	1737	.fibers = .pack(new_head),
	1738	}, .monotonic);
	1739	} else if (@atomicLoad(Awaiter, group.awaiterPtr(), .monotonic).awaiter.unpack()) \|awaiter\| {
	1740	if (!awaiter.cancel_status.changeAwaiting(.group, .nothing) or list.cancel_requested) {
	1741	@atomicStore(List, list_ptr, .{
	1742	.cancel_requested = false,
	1743	.awaiter_delayed = false,
	1744	.fibers = .null,
	1745	}, .release);
	1746	assert(awaiter.status.awaiting_group.ptr == group.ptr);
	1747	awaiter.status = .{ .queue_next = null };
	1748	return awaiter;
	1749	}
	1750	// Race with `Fiber.requestCancel`
	1751	@atomicStore(List, list_ptr, .{
	1752	.cancel_requested = false,
	1753	.awaiter_delayed = true,
	1754	.fibers = .null,
	1755	}, .monotonic);
	1756	} else @atomicStore(List, list_ptr, .{
	1757	.cancel_requested = false,
	1758	.awaiter_delayed = false,
	1759	.fibers = .null,
	1760	}, .release);
	1761	return null;
	1762	}
938	1763
939		std.log.debug("back from select yield", .{});
	1764	fn await(group: Group, ev: Evented, awaiter: Fiber) bool {
	1765	group.lock(ev);
	1766	defer group.unlock(ev);
	1767	if (@atomicLoad(List, group.listPtr(), .monotonic).fibers.unpack()) \|_\| {
	1768	if (group.registerAwaiter(awaiter) and awaiter.cancel_protection.check() == .unblocked) {
	1769	// The awaiter already had an unacknowledged cancelation request before
	1770	// attempting to await a group, so propagate the cancelation to the group.
	1771	assert(!group.cancelLocked(ev, null));
	1772	}
	1773	return false;
	1774	}
	1775	return true;
	1776	}
940	1777
941		const my_thread: *Thread = .current();
942		const my_fiber = my_thread.currentFiber();
943		var result: ?usize = null;
	1778	fn cancel(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1779	group.lock(ev);
	1780	defer group.unlock(ev);
	1781	return group.cancelLocked(ev, maybe_awaiter);
	1782	}
944	1783
945		for (futures, 0..) \|any_future, i\| {
946		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
947		if (@cmpxchgStrong(?*Fiber, &future_fiber.awaiter, my_fiber, null, .seq_cst, .seq_cst)) \|awaiter\| {
948		if (awaiter == Fiber.finished) {
949		if (result == null) result = i;
950		} else if (awaiter) \|a\| {
951		const closure: *AsyncClosure = .fromFiber(a);
952		closure.already_awaited = false;
	1784	/// Assumes the mutex is held.
	1785	fn cancelLocked(group: Group, ev: Evented, maybe_awaiter: ?Fiber) bool {
	1786	const list_ptr = group.listPtr();
	1787	const list = @atomicRmw(
	1788	List,
	1789	list_ptr,
	1790	.Add,
	1791	.{ .cancel_requested = true, .awaiter_delayed = false, .fibers = .null },
	1792	.monotonic,
	1793	);
	1794	assert(!list.cancel_requested);
	1795	if (list.fibers.unpack()) \|head\| {
	1796	var maybe_fiber: ?*Fiber = head;
	1797	while (maybe_fiber) \|fiber\| {
	1798	fiber.requestCancel(ev);
	1799	maybe_fiber = fiber.link.group.next;
953	1800	}
954		} else {
955		const closure: *AsyncClosure = .fromFiber(my_fiber);
956		closure.already_awaited = false;
	1801	if (maybe_awaiter) \|awaiter\| _ = group.registerAwaiter(awaiter);
	1802	return false;
957	1803	}
	1804	@atomicStore(
	1805	List,
	1806	list_ptr,
	1807	.{ .cancel_requested = false, .awaiter_delayed = false, .fibers = .null },
	1808	.release,
	1809	);
	1810	return if (maybe_awaiter) \|_\| true else list.awaiter_delayed;
958	1811	}
959	1812
960		return result.?;
961		}
	1813	/// Assumes the mutex is held.
	1814	fn registerAwaiter(group: Group, awaiter: *Fiber) bool {
	1815	assert(awaiter.status.queue_next == null);
	1816	awaiter.status = .{ .awaiting_group = group };
	1817	assert(@atomicRmw(
	1818	Awaiter,
	1819	group.awaiterPtr(),
	1820	.Add,
	1821	.{ .locked = false, .contended = false, .awaiter = .pack(awaiter) },
	1822	.monotonic,
	1823	).awaiter == .null);
	1824	return awaiter.cancel_status.changeAwaiting(.nothing, .group);
	1825	}
962	1826
963		fn cancel(
	1827	const AsyncClosure = struct {
	1828	ev: *Evented,
	1829	group: Group,
	1830	fiber: *Fiber,
	1831	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1832
	1833	fn fromFiber(fiber: Fiber) Group.AsyncClosure {
	1834	return @ptrFromInt(Fiber.max_context_align.max(.of(Group.AsyncClosure)).backward(
	1835	@intFromPtr(fiber.allocatedEnd()) - Fiber.max_context_size,
	1836	) - @sizeOf(Group.AsyncClosure));
	1837	}
	1838
	1839	fn contextPointer(
	1840	closure: *Group.AsyncClosure,
	1841	) [*]align(Fiber.max_context_align.toByteUnits()) u8 {
	1842	return @alignCast(@as([*]u8, @ptrCast(closure)) + @sizeOf(Group.AsyncClosure));
	1843	}
	1844
	1845	fn entry() callconv(.naked) void {
	1846	switch (builtin.cpu.arch) {
	1847	.aarch64 => asm volatile (
	1848	\\ mov x0, sp
	1849	\\ b %[call]
	1850	:
	1851	: [call] "X" (&call),
	1852	),
	1853	.x86_64 => asm volatile (
	1854	\\ leaq 8(%%rsp), %%rdi
	1855	\\ jmp %[call:P]
	1856	:
	1857	: [call] "X" (&call),
	1858	),
	1859	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
	1860	}
	1861	}
	1862
	1863	fn call(
	1864	closure: *Group.AsyncClosure,
	1865	message: *const SwitchMessage,
	1866	) callconv(.withStackAlign(.c, @alignOf(Group.AsyncClosure))) noreturn {
	1867	message.handle(closure.ev);
	1868	assert(closure.fiber.status.queue_next == null);
	1869	log.debug("{*} performing group async", .{closure.fiber});
	1870	const result = closure.start(closure.contextPointer());
	1871	const ev = closure.ev;
	1872	const group = closure.group;
	1873	const fiber = closure.fiber;
	1874	const cancel_acknowledged = fiber.cancel_protection.acknowledged;
	1875	if (result) {
	1876	assert(!cancel_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1877	} else \|err\| switch (err) {
	1878	error.Canceled => assert(cancel_acknowledged), // group task returned `error.Canceled` but was never canceled
	1879	}
	1880	const awaiter = group.removeFiber(ev, fiber);
	1881	ev.backing_allocator_mutex.lockUncancelable(ev.io());
	1882	ev.yield(awaiter, .destroy);
	1883	unreachable; // switched to dead fiber
	1884	}
	1885	};
	1886	};
	1887
	1888	fn groupAsync(
964	1889	userdata: ?*anyopaque,
965		any_future: *std.Io.AnyFuture,
966		result: []u8,
967		result_alignment: Alignment,
	1890	type_erased: *Io.Group,
	1891	context: []const u8,
	1892	context_alignment: Alignment,
	1893	start: const fn (context: const anyopaque) Io.Cancelable!void,
968	1894	) void {
969		const future_fiber: *Fiber = @ptrCast(@alignCast(any_future));
970		if (@atomicRmw(
971		?*Thread,
972		&future_fiber.cancel_thread,
973		.Xchg,
974		Thread.canceling,
975		.acq_rel,
976		)) \|cancel_thread\| if (cancel_thread != Thread.canceling) {
977		getSqe(&Thread.current().io_uring).* = .{
978		.opcode = .MSG_RING,
979		.flags = std.os.linux.IOSQE_CQE_SKIP_SUCCESS,
980		.ioprio = 0,
981		.fd = cancel_thread.io_uring.fd,
982		.off = @intFromPtr(future_fiber),
983		.addr = 0,
984		.len = @bitCast(-@as(i32, @intFromEnum(std.os.linux.E.INTR))),
985		.rw_flags = 0,
986		.user_data = @intFromEnum(Completion.UserData.cleanup),
987		.buf_index = 0,
988		.personality = 0,
989		.splice_fd_in = 0,
990		.addr3 = 0,
991		.resv = 0,
992		};
	1895	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1896	return groupConcurrent(ev, type_erased, context, context_alignment, start) catch {
	1897	const fiber = Thread.current().currentFiber();
	1898	const pre_acknowledged = fiber.cancel_protection.acknowledged;
	1899	const result = start(context.ptr);
	1900	const post_acknowledged = fiber.cancel_protection.acknowledged;
	1901	if (result) {
	1902	if (pre_acknowledged) {
	1903	assert(post_acknowledged); // group task called `recancel` but was not canceled
	1904	} else {
	1905	assert(!post_acknowledged); // group task acknowledged cancelation but did not return `error.Canceled`
	1906	}
	1907	} else \|err\| switch (err) {
	1908	// Don't swallow the cancelation: make it visible to the `Group.async` caller.
	1909	error.Canceled => {
	1910	assert(!pre_acknowledged); // group task called `recancel` but was not canceled
	1911	assert(post_acknowledged); // group task returned `error.Canceled` but was never canceled
	1912	recancel(userdata);
	1913	},
	1914	}
993	1915	};
994		await(userdata, any_future, result, result_alignment);
995		}
996
997		fn cancelRequested(userdata: ?*anyopaque) bool {
998		_ = userdata;
999		return @atomicLoad(?*Thread, &Thread.current().currentFiber().cancel_thread, .acquire) == Thread.canceling;
1000	1916	}
1001	1917
1002		fn createFile(
	1918	fn groupConcurrent(
1003	1919	userdata: ?*anyopaque,
1004		dir: Io.Dir,
1005		sub_path: []const u8,
1006		flags: Io.File.CreateFlags,
1007		) Io.File.OpenError!Io.File {
1008		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1009		const thread: *Thread = .current();
1010		const iou = &thread.io_uring;
1011		const fiber = thread.currentFiber();
1012		try fiber.enterCancelRegion(thread);
1013
1014		const posix = std.posix;
1015		const sub_path_c = try posix.toPosixPath(sub_path);
	1920	type_erased: *Io.Group,
	1921	context: []const u8,
	1922	context_alignment: Alignment,
	1923	start: const fn (context: const anyopaque) Io.Cancelable!void,
	1924	) Io.ConcurrentError!void {
	1925	assert(context_alignment.compare(.lte, Fiber.max_context_align)); // TODO
	1926	assert(context.len <= Fiber.max_context_size); // TODO
1016	1927
1017		var os_flags: posix.O = .{
1018		.ACCMODE = if (flags.read) .RDWR else .WRONLY,
1019		.CREAT = true,
1020		.TRUNC = flags.truncate,
1021		.EXCL = flags.exclusive,
	1928	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1929	const group: Group = .{ .ptr = type_erased };
	1930	const fiber = Fiber.create(ev) catch \|err\| switch (err) {
	1931	error.OutOfMemory => return error.ConcurrencyUnavailable,
1022	1932	};
1023		if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true;
1024		if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true;
	1933	log.debug("allocated {*}", .{fiber});
1025	1934
1026		// Use the O locking flags if the os supports them to acquire the lock
1027		// atomically. Note that the NONBLOCK flag is removed after the openat()
1028		// call is successful.
1029		const has_flock_open_flags = @hasField(posix.O, "EXLOCK");
1030		if (has_flock_open_flags) switch (flags.lock) {
1031		.none => {},
1032		.shared => {
1033		os_flags.SHLOCK = true;
1034		os_flags.NONBLOCK = flags.lock_nonblocking;
	1935	const closure: *Group.AsyncClosure = .fromFiber(fiber);
	1936	fiber.* = .{
	1937	.required_align = {},
	1938	.context = switch (builtin.cpu.arch) {
	1939	.aarch64 => .{
	1940	.sp = @intFromPtr(closure),
	1941	.fp = 0,
	1942	.pc = @intFromPtr(&Group.AsyncClosure.entry),
	1943	},
	1944	.x86_64 => .{
	1945	.rsp = @intFromPtr(closure) - @sizeOf(usize),
	1946	.rbp = 0,
	1947	.rip = @intFromPtr(&Group.AsyncClosure.entry),
	1948	},
	1949	else => \|arch\| @compileError("unimplemented architecture: " ++ @tagName(arch)),
1035	1950	},
1036		.exclusive => {
1037		os_flags.EXLOCK = true;
1038		os_flags.NONBLOCK = flags.lock_nonblocking;
	1951	.await_count = 0,
	1952	.link = .{ .group = .{ .prev = null, .next = null } },
	1953	.status = .{ .queue_next = null },
	1954	.cancel_status = .unrequested,
	1955	.cancel_protection = .unblocked,
	1956	.name = if (tracy.enable) name: {
	1957	const thread: *Thread = .current();
	1958	var name_arena = thread.name_arena.promote(std.heap.page_allocator);
	1959	defer thread.name_arena = name_arena.state;
	1960	break :name std.fmt.allocPrintSentinel(
	1961	name_arena.allocator(),
	1962	"group task {d}",
	1963	.{@atomicRmw(u64, &Fiber.next_name, .Add, 1, .monotonic)},
	1964	0,
	1965	) catch return error.ConcurrencyUnavailable;
1039	1966	},
1040	1967	};
1041		const have_flock = @TypeOf(posix.system.flock) != void;
1042
1043		if (have_flock and !has_flock_open_flags and flags.lock != .none) {
1044		@panic("TODO");
1045		}
1046
1047		if (has_flock_open_flags and flags.lock_nonblocking) {
1048		@panic("TODO");
1049		}
1050
1051		getSqe(iou).* = .{
1052		.opcode = .OPENAT,
1053		.flags = 0,
1054		.ioprio = 0,
1055		.fd = dir.handle,
1056		.off = 0,
1057		.addr = @intFromPtr(&sub_path_c),
1058		.len = @intCast(flags.mode),
1059		.rw_flags = @bitCast(os_flags),
1060		.user_data = @intFromPtr(fiber),
1061		.buf_index = 0,
1062		.personality = 0,
1063		.splice_fd_in = 0,
1064		.addr3 = 0,
1065		.resv = 0,
	1968	closure.* = .{
	1969	.ev = ev,
	1970	.group = group,
	1971	.fiber = fiber,
	1972	.start = start,
1066	1973	};
	1974	@memcpy(closure.contextPointer(), context);
	1975	group.addFiber(ev, fiber);
	1976	const thread: *Thread = .current();
	1977	if (ev.schedule(thread, .{ .head = fiber, .tail = fiber })) thread.submit();
	1978	}
1067	1979
1068		el.yield(null, .nothing);
1069		fiber.exitCancelRegion(thread);
1070
1071		const completion = fiber.resultPointer(Completion);
1072		switch (errno(completion.result)) {
1073		.SUCCESS => return .{ .handle = completion.result },
1074		.INTR => unreachable,
1075		.CANCELED => return error.Canceled,
	1980	fn groupAwait(
	1981	userdata: ?*anyopaque,
	1982	type_erased: *Io.Group,
	1983	initial_token: *anyopaque,
	1984	) Io.Cancelable!void {
	1985	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1986	_ = initial_token;
	1987	ev.yield(null, .{ .group_await = .{ .ptr = type_erased } });
	1988	}
1076	1989
1077		.FAULT => unreachable,
1078		.INVAL => return error.BadPathName,
1079		.BADF => unreachable,
1080		.ACCES => return error.AccessDenied,
1081		.FBIG => return error.FileTooBig,
1082		.OVERFLOW => return error.FileTooBig,
1083		.ISDIR => return error.IsDir,
1084		.LOOP => return error.SymLinkLoop,
1085		.MFILE => return error.ProcessFdQuotaExceeded,
1086		.NAMETOOLONG => return error.NameTooLong,
1087		.NFILE => return error.SystemFdQuotaExceeded,
1088		.NODEV => return error.NoDevice,
1089		.NOENT => return error.FileNotFound,
1090		.NOMEM => return error.SystemResources,
1091		.NOSPC => return error.NoSpaceLeft,
1092		.NOTDIR => return error.NotDir,
1093		.PERM => return error.PermissionDenied,
1094		.EXIST => return error.PathAlreadyExists,
1095		.BUSY => return error.DeviceBusy,
1096		.OPNOTSUPP => return error.FileLocksUnsupported,
1097		.AGAIN => return error.WouldBlock,
1098		.TXTBSY => return error.FileBusy,
1099		.NXIO => return error.NoDevice,
1100		else => \|err\| return posix.unexpectedErrno(err),
1101		}
	1990	fn groupCancel(userdata: ?anyopaque, type_erased: Io.Group, initial_token: *anyopaque) void {
	1991	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1992	_ = initial_token;
	1993	ev.yield(null, .{ .group_cancel = .{ .ptr = type_erased } });
1102	1994	}
1103	1995
1104		fn fileOpen(
1105		userdata: ?*anyopaque,
1106		dir: Io.Dir,
1107		sub_path: []const u8,
1108		flags: Io.File.OpenFlags,
1109		) Io.File.OpenError!Io.File {
1110		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1111		const thread: *Thread = .current();
1112		const iou = &thread.io_uring;
1113		const fiber = thread.currentFiber();
1114		try fiber.enterCancelRegion(thread);
	1996	fn recancel(userdata: ?*anyopaque) void {
	1997	const ev: *Evented = @ptrCast(@alignCast(userdata));
	1998	_ = ev;
	1999	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
	2000	assert(cancel_protection.acknowledged);
	2001	cancel_protection.acknowledged = false;
	2002	}
1115	2003
1116		const posix = std.posix;
1117		const sub_path_c = try posix.toPosixPath(sub_path);
	2004	fn swapCancelProtection(userdata: ?*anyopaque, new: Io.CancelProtection) Io.CancelProtection {
	2005	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2006	_ = ev;
	2007	const cancel_protection = &Thread.current().currentFiber().cancel_protection;
	2008	defer cancel_protection.user = new;
	2009	return cancel_protection.user;
	2010	}
1118	2011
1119		var os_flags: posix.O = .{
1120		.ACCMODE = switch (flags.mode) {
1121		.read_only => .RDONLY,
1122		.write_only => .WRONLY,
1123		.read_write => .RDWR,
	2012	fn checkCancel(userdata: ?*anyopaque) Io.Cancelable!void {
	2013	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2014	_ = ev;
	2015	const fiber = Thread.current().currentFiber();
	2016	switch (fiber.cancel_protection.check()) {
	2017	.blocked => {},
	2018	.unblocked => if (@atomicLoad(Fiber.CancelStatus, &fiber.cancel_status, .monotonic).requested) {
	2019	fiber.cancel_protection.acknowledge();
	2020	return error.Canceled;
1124	2021	},
1125		};
1126
1127		if (@hasField(posix.O, "CLOEXEC")) os_flags.CLOEXEC = true;
1128		if (@hasField(posix.O, "LARGEFILE")) os_flags.LARGEFILE = true;
1129		if (@hasField(posix.O, "NOCTTY")) os_flags.NOCTTY = !flags.allow_ctty;
1130
1131		// Use the O locking flags if the os supports them to acquire the lock
1132		// atomically.
1133		const has_flock_open_flags = @hasField(posix.O, "EXLOCK");
1134		if (has_flock_open_flags) {
1135		// Note that the NONBLOCK flag is removed after the openat() call
1136		// is successful.
1137		switch (flags.lock) {
1138		.none => {},
1139		.shared => {
1140		os_flags.SHLOCK = true;
1141		os_flags.NONBLOCK = flags.lock_nonblocking;
1142		},
1143		.exclusive => {
1144		os_flags.EXLOCK = true;
1145		os_flags.NONBLOCK = flags.lock_nonblocking;
1146		},
1147		}
1148	2022	}
1149		const have_flock = @TypeOf(posix.system.flock) != void;
	2023	}
1150	2024
1151		if (have_flock and !has_flock_open_flags and flags.lock != .none) {
1152		@panic("TODO");
	2025	fn select(userdata: ?anyopaque, futures: []const Io.AnyFuture) Io.Cancelable!usize {
	2026	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2027	var cancel_region: CancelRegion = .init();
	2028	defer cancel_region.deinit();
	2029	var await_count: u31, var result = for (futures, 0..) \|future, future_index\| {
	2030	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	2031	if (@atomicRmw(
	2032	?*Fiber,
	2033	&future_fiber.link.awaiter,
	2034	.Xchg,
	2035	cancel_region.fiber,
	2036	.acq_rel,
	2037	)) \|awaiter\| {
	2038	assert(awaiter == Fiber.finished);
	2039	break .{ @intCast(future_index), future_index };
	2040	}
	2041	} else result: {
	2042	const await_count: u31 = @intCast(futures.len);
	2043	cancel_region.await(.select) catch \|err\| switch (err) {
	2044	error.Canceled => \|e\| break :result .{ await_count + 1, e },
	2045	};
	2046	ev.yield(null, .{ .await = 1 });
	2047	cancel_region.await(.nothing) catch \|err\| switch (err) {
	2048	error.Canceled => \|e\| break :result .{ await_count, e },
	2049	};
	2050	break :result .{ await_count - 1, futures.len };
	2051	};
	2052	for (futures[0 .. result catch futures.len], 0..) \|future, future_index\| {
	2053	const future_fiber: *Fiber = @ptrCast(@alignCast(future));
	2054	const awaiter = @atomicRmw(?*Fiber, &future_fiber.link.awaiter, .Xchg, null, .monotonic);
	2055	if (awaiter == Fiber.finished) {
	2056	@atomicStore(?*Fiber, &future_fiber.link.awaiter, Fiber.finished, .monotonic);
	2057	result = if (result) \|finished_index\| @min(future_index, finished_index) else \|e\| e;
	2058	} else {
	2059	assert(awaiter == cancel_region.fiber);
	2060	await_count -= 1;
	2061	}
1153	2062	}
1154
1155		if (has_flock_open_flags and flags.lock_nonblocking) {
1156		@panic("TODO");
	2063	// Equivalent to `ev.yield(null, .{ .await = await_count });`,
	2064	// but avoiding a context switch in the common case.
	2065	switch (std.math.order(
	2066	@atomicRmw(i32, &cancel_region.fiber.await_count, .Sub, await_count, .monotonic),
	2067	await_count,
	2068	)) {
	2069	.lt => ev.yield(null, .{ .await = 0 }),
	2070	.eq => {},
	2071	.gt => unreachable,
1157	2072	}
	2073	return result;
	2074	}
1158	2075
1159		getSqe(iou).* = .{
1160		.opcode = .OPENAT,
1161		.flags = 0,
	2076	fn futexWait(
	2077	userdata: ?*anyopaque,
	2078	ptr: *const u32,
	2079	expected: u32,
	2080	timeout: Io.Timeout,
	2081	) Io.Cancelable!void {
	2082	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2083	if (builtin.single_threaded) unreachable; // Deadlock.
	2084	const timespec: ?linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
	2085	.none => .{
	2086	null,
	2087	.awake,
	2088	linux.IORING_TIMEOUT_ABS,
	2089	},
	2090	.duration => \|duration\| {
	2091	const ns = duration.raw.toNanoseconds();
	2092	break :timespec .{
	2093	.{
	2094	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2095	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2096	},
	2097	duration.clock,
	2098	0,
	2099	};
	2100	},
	2101	.deadline => \|deadline\| {
	2102	const ns = deadline.raw.toNanoseconds();
	2103	break :timespec .{
	2104	.{
	2105	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2106	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2107	},
	2108	deadline.clock,
	2109	linux.IORING_TIMEOUT_ABS,
	2110	};
	2111	},
	2112	};
	2113	var cancel_region: CancelRegion = .init();
	2114	defer cancel_region.deinit();
	2115	const thread = try cancel_region.awaitIoUring();
	2116	thread.enqueue().* = .{
	2117	.opcode = .FUTEX_WAIT,
	2118	.flags = if (timespec) \|_\| linux.IOSQE_IO_LINK else 0,
1162	2119	.ioprio = 0,
1163		.fd = dir.handle,
1164		.off = 0,
1165		.addr = @intFromPtr(&sub_path_c),
	2120	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2121	.off = expected,
	2122	.addr = @intFromPtr(ptr),
1166	2123	.len = 0,
1167		.rw_flags = @bitCast(os_flags),
1168		.user_data = @intFromPtr(fiber),
	2124	.rw_flags = 0,
	2125	.user_data = @intFromPtr(cancel_region.fiber),
1169	2126	.buf_index = 0,
1170	2127	.personality = 0,
1171	2128	.splice_fd_in = 0,
1172		.addr3 = 0,
	2129	.addr3 = std.math.maxInt(u32),
1173	2130	.resv = 0,
1174	2131	};
1175
1176		el.yield(null, .nothing);
1177		fiber.exitCancelRegion(thread);
1178
1179		const completion = fiber.resultPointer(Completion);
1180		switch (errno(completion.result)) {
1181		.SUCCESS => return .{ .handle = completion.result },
1182		.INTR => unreachable,
1183		.CANCELED => return error.Canceled,
1184
1185		.FAULT => unreachable,
1186		.INVAL => return error.BadPathName,
1187		.BADF => unreachable,
1188		.ACCES => return error.AccessDenied,
1189		.FBIG => return error.FileTooBig,
1190		.OVERFLOW => return error.FileTooBig,
1191		.ISDIR => return error.IsDir,
1192		.LOOP => return error.SymLinkLoop,
1193		.MFILE => return error.ProcessFdQuotaExceeded,
1194		.NAMETOOLONG => return error.NameTooLong,
1195		.NFILE => return error.SystemFdQuotaExceeded,
1196		.NODEV => return error.NoDevice,
1197		.NOENT => return error.FileNotFound,
1198		.NOMEM => return error.SystemResources,
1199		.NOSPC => return error.NoSpaceLeft,
1200		.NOTDIR => return error.NotDir,
1201		.PERM => return error.PermissionDenied,
1202		.EXIST => return error.PathAlreadyExists,
1203		.BUSY => return error.DeviceBusy,
1204		.OPNOTSUPP => return error.FileLocksUnsupported,
1205		.AGAIN => return error.WouldBlock,
1206		.TXTBSY => return error.FileBusy,
1207		.NXIO => return error.NoDevice,
1208		else => \|err\| return posix.unexpectedErrno(err),
1209		}
1210		}
1211
1212		fn fileClose(userdata: ?*anyopaque, file: Io.File) void {
1213		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1214		const thread: *Thread = .current();
1215		const iou = &thread.io_uring;
1216		const fiber = thread.currentFiber();
1217
1218		getSqe(iou).* = .{
1219		.opcode = .CLOSE,
1220		.flags = 0,
	2132	if (timespec) \|timespec_ptr\| thread.enqueue(). = .{
	2133	.opcode = .LINK_TIMEOUT,
	2134	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1221	2135	.ioprio = 0,
1222		.fd = file.handle,
	2136	.fd = 0,
1223	2137	.off = 0,
1224		.addr = 0,
1225		.len = 0,
1226		.rw_flags = 0,
1227		.user_data = @intFromPtr(fiber),
	2138	.addr = @intFromPtr(timespec_ptr),
	2139	.len = 1,
	2140	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	2141	.real => linux.IORING_TIMEOUT_REALTIME,
	2142	else => 0,
	2143	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	2144	}),
	2145	.user_data = @intFromEnum(Completion.UserData.wakeup),
1228	2146	.buf_index = 0,
1229	2147	.personality = 0,
1230	2148	.splice_fd_in = 0,
1231	2149	.addr3 = 0,
1232	2150	.resv = 0,
1233	2151	};
1234
1235		el.yield(null, .nothing);
1236
1237		const completion = fiber.resultPointer(Completion);
1238		switch (errno(completion.result)) {
1239		.SUCCESS => return,
1240		.INTR => unreachable,
1241		.CANCELED => return,
1242
1243		.BADF => unreachable, // Always a race condition.
1244		else => return,
	2152	ev.yield(null, .nothing);
	2153	switch (cancel_region.errno()) {
	2154	.SUCCESS => {}, // notified by `wake()`
	2155	.INTR, .CANCELED => {}, // caller's responsibility to retry
	2156	.AGAIN => {}, // ptr.* != expect
	2157	.INVAL => {}, // possibly timeout overflow
	2158	.TIMEDOUT => unreachable,
	2159	.FAULT => recoverableOsBugDetected(), // ptr was invalid
	2160	else => recoverableOsBugDetected(),
1245	2161	}
1246	2162	}
1247	2163
1248		fn pread(userdata: ?*anyopaque, file: Io.File, buffer: []u8, offset: std.posix.off_t) Io.File.PReadError!usize {
1249		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1250		const thread: *Thread = .current();
1251		const iou = &thread.io_uring;
1252		const fiber = thread.currentFiber();
1253		try fiber.enterCancelRegion(thread);
1254
1255		getSqe(iou).* = .{
1256		.opcode = .READ,
	2164	fn futexWaitUncancelable(userdata: ?anyopaque, ptr: const u32, expected: u32) void {
	2165	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2166	if (builtin.single_threaded) unreachable; // Deadlock.
	2167	var cancel_region: CancelRegion = .initBlocked();
	2168	defer cancel_region.deinit();
	2169	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	2170	error.Canceled => unreachable, // blocked
	2171	};
	2172	thread.enqueue().* = .{
	2173	.opcode = .FUTEX_WAIT,
1257	2174	.flags = 0,
1258	2175	.ioprio = 0,
1259		.fd = file.handle,
1260		.off = @bitCast(offset),
1261		.addr = @intFromPtr(buffer.ptr),
1262		.len = @min(buffer.len, 0x7ffff000),
	2176	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2177	.off = expected,
	2178	.addr = @intFromPtr(ptr),
	2179	.len = 0,
1263	2180	.rw_flags = 0,
1264		.user_data = @intFromPtr(fiber),
	2181	.user_data = @intFromPtr(cancel_region.fiber),
1265	2182	.buf_index = 0,
1266	2183	.personality = 0,
1267	2184	.splice_fd_in = 0,
1268		.addr3 = 0,
	2185	.addr3 = std.math.maxInt(u32),
1269	2186	.resv = 0,
1270	2187	};
1271
1272		el.yield(null, .nothing);
1273		fiber.exitCancelRegion(thread);
1274
1275		const completion = fiber.resultPointer(Completion);
1276		switch (errno(completion.result)) {
1277		.SUCCESS => return @as(u32, @bitCast(completion.result)),
1278		.INTR => unreachable,
1279		.CANCELED => return error.Canceled,
1280
1281		.INVAL => unreachable,
1282		.FAULT => unreachable,
1283		.NOENT => return error.ProcessNotFound,
1284		.AGAIN => return error.WouldBlock,
1285		.BADF => return error.NotOpenForReading, // Can be a race condition.
1286		.IO => return error.InputOutput,
1287		.ISDIR => return error.IsDir,
1288		.NOBUFS => return error.SystemResources,
1289		.NOMEM => return error.SystemResources,
1290		.NOTCONN => return error.SocketUnconnected,
1291		.CONNRESET => return error.ConnectionResetByPeer,
1292		.TIMEDOUT => return error.Timeout,
1293		.NXIO => return error.Unseekable,
1294		.SPIPE => return error.Unseekable,
1295		.OVERFLOW => return error.Unseekable,
1296		else => \|err\| return std.posix.unexpectedErrno(err),
	2188	ev.yield(null, .nothing);
	2189	switch (cancel_region.errno()) {
	2190	.SUCCESS => {}, // notified by `wake()`
	2191	.INTR, .CANCELED => {}, // caller's responsibility to retry
	2192	.AGAIN => {}, // ptr.* != expect
	2193	.INVAL => {}, // possibly timeout overflow
	2194	.FAULT => recoverableOsBugDetected(), // ptr was invalid
	2195	else => recoverableOsBugDetected(),
1297	2196	}
1298	2197	}
1299	2198
1300		fn pwrite(userdata: ?*anyopaque, file: Io.File, buffer: []const u8, offset: std.posix.off_t) Io.File.PWriteError!usize {
1301		const el: *EventLoop = @ptrCast(@alignCast(userdata));
	2199	fn futexWake(userdata: ?anyopaque, ptr: const u32, max_waiters: u32) void {
	2200	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2201	_ = ev;
	2202	if (builtin.single_threaded) unreachable; // Nothing to wake up.
1302	2203	const thread: *Thread = .current();
1303		const iou = &thread.io_uring;
1304		const fiber = thread.currentFiber();
1305		try fiber.enterCancelRegion(thread);
1306
1307		getSqe(iou).* = .{
1308		.opcode = .WRITE,
1309		.flags = 0,
	2204	thread.enqueue().* = .{
	2205	.opcode = .FUTEX_WAKE,
	2206	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
1310	2207	.ioprio = 0,
1311		.fd = file.handle,
1312		.off = @bitCast(offset),
1313		.addr = @intFromPtr(buffer.ptr),
1314		.len = @min(buffer.len, 0x7ffff000),
	2208	.fd = @bitCast(linux.FUTEX2_FLAGS{ .size = .U32, .private = true }),
	2209	.off = max_waiters,
	2210	.addr = @intFromPtr(ptr),
	2211	.len = 0,
1315	2212	.rw_flags = 0,
1316		.user_data = @intFromPtr(fiber),
	2213	.user_data = @intFromEnum(Completion.UserData.futex_wake),
1317	2214	.buf_index = 0,
1318	2215	.personality = 0,
1319	2216	.splice_fd_in = 0,
1320		.addr3 = 0,
	2217	.addr3 = std.math.maxInt(u32),
1321	2218	.resv = 0,
1322	2219	};
	2220	thread.submit();
	2221	}
1323	2222
1324		el.yield(null, .nothing);
1325		fiber.exitCancelRegion(thread);
1326
1327		const completion = fiber.resultPointer(Completion);
1328		switch (errno(completion.result)) {
1329		.SUCCESS => return @as(u32, @bitCast(completion.result)),
1330		.INTR => unreachable,
1331		.CANCELED => return error.Canceled,
1332
1333		.INVAL => return error.InvalidArgument,
1334		.FAULT => unreachable,
1335		.NOENT => return error.ProcessNotFound,
1336		.AGAIN => return error.WouldBlock,
1337		.BADF => return error.NotOpenForWriting, // can be a race condition.
1338		.DESTADDRREQ => unreachable, // `connect` was never called.
1339		.DQUOT => return error.DiskQuota,
1340		.FBIG => return error.FileTooBig,
1341		.IO => return error.InputOutput,
1342		.NOSPC => return error.NoSpaceLeft,
1343		.ACCES => return error.AccessDenied,
1344		.PERM => return error.PermissionDenied,
1345		.PIPE => return error.BrokenPipe,
1346		.NXIO => return error.Unseekable,
1347		.SPIPE => return error.Unseekable,
1348		.OVERFLOW => return error.Unseekable,
1349		.BUSY => return error.DeviceBusy,
1350		.CONNRESET => return error.ConnectionResetByPeer,
1351		.MSGSIZE => return error.MessageOversize,
1352		else => \|err\| return std.posix.unexpectedErrno(err),
	2223	fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Operation.Result {
	2224	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2225	switch (operation) {
	2226	.file_read_streaming => \|o\| return .{
	2227	.file_read_streaming = ev.fileReadStreaming(o.file, o.data) catch \|err\| switch (err) {
	2228	error.Canceled => \|e\| return e,
	2229	else => \|e\| e,
	2230	},
	2231	},
	2232	.file_write_streaming => \|o\| return .{
	2233	.file_write_streaming = ev.fileWriteStreaming(o.file, o.header, o.data, o.splat) catch \|err\| switch (err) {
	2234	error.Canceled => \|e\| return e,
	2235	else => \|e\| e,
	2236	},
	2237	},
	2238	.device_io_control => \|*o\| return .{ .device_io_control = try deviceIoControl(o) },
1353	2239	}
1354	2240	}
1355	2241
1356		fn now(userdata: ?*anyopaque, clockid: std.posix.clockid_t) Io.ClockGetTimeError!Io.Timestamp {
1357		_ = userdata;
1358		const timespec = try std.posix.clock_gettime(clockid);
1359		return @enumFromInt(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);
1360		}
	2242	fn fileReadStreaming(ev: *Evented, file: File, data: []const []u8) File.Reader.Error!usize {
	2243	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
	2244	var i: usize = 0;
	2245	for (data) \|buf\| {
	2246	if (iovecs_buffer.len - i == 0) break;
	2247	if (buf.len != 0) {
	2248	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
	2249	i += 1;
	2250	}
	2251	}
	2252	const dest = iovecs_buffer[0..i];
	2253	assert(dest[0].len > 0);
1361	2254
1362		fn sleep(userdata: ?*anyopaque, clockid: std.posix.clockid_t, deadline: Io.Deadline) Io.SleepError!void {
1363		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1364		const thread: *Thread = .current();
1365		const iou = &thread.io_uring;
1366		const fiber = thread.currentFiber();
1367		try fiber.enterCancelRegion(thread);
	2255	var cancel_region: CancelRegion = .init();
	2256	defer cancel_region.deinit();
	2257	return ev.preadv(&cancel_region, file.handle, dest, null);
	2258	}
1368	2259
1369		const deadline_nanoseconds: i96 = switch (deadline) {
1370		.duration => \|duration\| duration.nanoseconds,
1371		.timestamp => \|timestamp\| @intFromEnum(timestamp),
1372		};
1373		const timespec: std.os.linux.kernel_timespec = .{
1374		.sec = @intCast(@divFloor(deadline_nanoseconds, std.time.ns_per_s)),
1375		.nsec = @intCast(@mod(deadline_nanoseconds, std.time.ns_per_s)),
	2260	fn fileWriteStreaming(
	2261	ev: *Evented,
	2262	file: File,
	2263	header: []const u8,
	2264	data: []const []const u8,
	2265	splat: usize,
	2266	) File.Writer.Error!usize {
	2267	var iovecs: [max_iovecs_len]iovec_const = undefined;
	2268	var iovlen: iovlen_t = 0;
	2269	addBuf(&iovecs, &iovlen, header);
	2270	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
	2271	const pattern = data[data.len - 1];
	2272	if (iovecs.len - iovlen != 0) switch (splat) {
	2273	0 => {},
	2274	1 => addBuf(&iovecs, &iovlen, pattern),
	2275	else => switch (pattern.len) {
	2276	0 => {},
	2277	1 => {
	2278	var backup_buffer: [splat_buffer_size]u8 = undefined;
	2279	const splat_buffer = &backup_buffer;
	2280	const memset_len = @min(splat_buffer.len, splat);
	2281	const buf = splat_buffer[0..memset_len];
	2282	@memset(buf, pattern[0]);
	2283	addBuf(&iovecs, &iovlen, buf);
	2284	var remaining_splat = splat - buf.len;
	2285	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
	2286	assert(buf.len == splat_buffer.len);
	2287	addBuf(&iovecs, &iovlen, splat_buffer);
	2288	remaining_splat -= splat_buffer.len;
	2289	}
	2290	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	2291	},
	2292	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	2293	addBuf(&iovecs, &iovlen, pattern);
	2294	},
	2295	},
1376	2296	};
1377		getSqe(iou).* = .{
1378		.opcode = .TIMEOUT,
1379		.flags = 0,
1380		.ioprio = 0,
1381		.fd = 0,
	2297
	2298	var cancel_region: CancelRegion = .init();
	2299	defer cancel_region.deinit();
	2300	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], null);
	2301	}
	2302
	2303	fn deviceIoControl(o: *const Io.Operation.DeviceIoControl) Io.Cancelable!i32 {
	2304	var cancel_region: CancelRegion = .init();
	2305	defer cancel_region.deinit();
	2306	while (true) {
	2307	try cancel_region.await(.nothing);
	2308	const rc = linux.ioctl(o.file.handle, @bitCast(o.code), @intFromPtr(o.arg));
	2309	switch (linux.errno(rc)) {
	2310	.SUCCESS => return @bitCast(@as(u32, @truncate(rc))),
	2311	.INTR => continue,
	2312	else => \|err\| return -@as(i32, @intFromEnum(err)),
	2313	}
	2314	}
	2315	}
	2316
	2317	fn batchAwaitAsync(userdata: ?anyopaque, batch: Io.Batch) Io.Cancelable!void {
	2318	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2319	var cancel_region: CancelRegion = .init();
	2320	defer cancel_region.deinit();
	2321	batchDrainSubmitted(batch, &cancel_region, false) catch \|err\| switch (err) {
	2322	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
	2323	else => \|e\| return e,
	2324	};
	2325	while (true) {
	2326	batchDrainReady(batch) catch \|err\| switch (err) {
	2327	error.Timeout => unreachable, // no timeout
	2328	};
	2329	if (batch.completed.head != .none) return;
	2330	ev.yield(null, .{ .batch_await = batch });
	2331	}
	2332	}
	2333
	2334	fn batchAwaitConcurrent(
	2335	userdata: ?*anyopaque,
	2336	batch: *Io.Batch,
	2337	timeout: Io.Timeout,
	2338	) Io.Batch.AwaitConcurrentError!void {
	2339	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2340	var cancel_region: CancelRegion = .init();
	2341	defer cancel_region.deinit();
	2342	try batchDrainSubmitted(batch, &cancel_region, true);
	2343	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = while (true) {
	2344	batchDrainReady(batch) catch \|err\| switch (err) {
	2345	error.Timeout => unreachable, // no timeout
	2346	};
	2347	if (batch.completed.head != .none) return;
	2348	switch (timeout) {
	2349	.none => ev.yield(null, .{ .batch_await = batch }),
	2350	.duration => \|duration\| {
	2351	const ns = duration.raw.toNanoseconds();
	2352	break .{
	2353	.{
	2354	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2355	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2356	},
	2357	duration.clock,
	2358	0,
	2359	};
	2360	},
	2361	.deadline => \|deadline\| {
	2362	const ns = deadline.raw.toNanoseconds();
	2363	break .{
	2364	.{
	2365	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	2366	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	2367	},
	2368	deadline.clock,
	2369	linux.IORING_TIMEOUT_ABS,
	2370	};
	2371	},
	2372	}
	2373	};
	2374	{
	2375	const thread = try cancel_region.awaitIoUring();
	2376	thread.enqueue().* = .{
	2377	.opcode = .TIMEOUT,
	2378	.flags = 0,
	2379	.ioprio = 0,
	2380	.fd = 0,
	2381	.off = 0,
	2382	.addr = @intFromPtr(&timespec),
	2383	.len = 1,
	2384	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	2385	.real => linux.IORING_TIMEOUT_REALTIME,
	2386	else => 0,
	2387	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	2388	}),
	2389	.user_data = @intFromPtr(&batch.context) \| 0b11,
	2390	.buf_index = 0,
	2391	.personality = 0,
	2392	.splice_fd_in = 0,
	2393	.addr3 = 0,
	2394	.resv = 0,
	2395	};
	2396	}
	2397	while (batch.completed.head == .none) {
	2398	ev.yield(null, .{ .batch_await = batch });
	2399	batchDrainReady(batch) catch \|err\| switch (err) {
	2400	error.Timeout => \|e\| return if (batch.completed.head == .none) e,
	2401	};
	2402	if (batch.completed.head == .none) continue;
	2403	}
	2404	const thread = try cancel_region.awaitIoUring();
	2405	thread.enqueue().* = .{
	2406	.opcode = .TIMEOUT_REMOVE,
	2407	.flags = 0,
	2408	.ioprio = 0,
	2409	.fd = 0,
1382	2410	.off = 0,
1383		.addr = @intFromPtr(&timespec),
1384		.len = 1,
1385		.rw_flags = @as(u32, switch (deadline) {
1386		.duration => 0,
1387		.timestamp => std.os.linux.IORING_TIMEOUT_ABS,
1388		}) \| @as(u32, switch (clockid) {
1389		.REALTIME => std.os.linux.IORING_TIMEOUT_REALTIME,
1390		.MONOTONIC => 0,
1391		.BOOTTIME => std.os.linux.IORING_TIMEOUT_BOOTTIME,
1392		else => return error.UnsupportedClock,
1393		}),
1394		.user_data = @intFromPtr(fiber),
	2411	.addr = @intFromPtr(&batch.context) \| 0b11,
	2412	.len = 0,
	2413	.rw_flags = 0,
	2414	.user_data = @intFromPtr(cancel_region.fiber),
1395	2415	.buf_index = 0,
1396	2416	.personality = 0,
1397	2417	.splice_fd_in = 0,
1398	2418	.addr3 = 0,
1399	2419	.resv = 0,
1400	2420	};
	2421	ev.yield(null, .nothing);
	2422	switch (cancel_region.errno()) {
	2423	.SUCCESS => return,
	2424	.BUSY, .NOENT => {},
	2425	else => \|err\| unexpectedErrno(err) catch {},
	2426	}
	2427	while (true) {
	2428	batchDrainReady(batch) catch \|err\| switch (err) {
	2429	error.Timeout => return,
	2430	};
	2431	ev.yield(null, .{ .batch_await = batch });
	2432	}
	2433	}
1401	2434
1402		el.yield(null, .nothing);
1403		fiber.exitCancelRegion(thread);
1404
1405		const completion = fiber.resultPointer(Completion);
1406		switch (errno(completion.result)) {
1407		.SUCCESS, .TIME => return,
1408		.INTR => unreachable,
1409		.CANCELED => return error.Canceled,
	2435	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
	2436	fn batchDrainSubmitted(
	2437	batch: *Io.Batch,
	2438	cancel_region: *CancelRegion,
	2439	concurrency: bool,
	2440	) (Io.ConcurrentError \|\| Io.Cancelable)!void {
	2441	var index = batch.submitted.head;
	2442	if (index == .none) return;
	2443	errdefer batch.submitted.head = index;
	2444	const thread = try cancel_region.awaitIoUring();
	2445	while (index != .none) {
	2446	const storage = &batch.storage[index.toIndex()];
	2447	const next_index = storage.submission.node.next;
	2448	if (@as(?Io.Operation.Result, operation: switch (storage.submission.operation) {
	2449	.file_read_streaming => \|o\| {
	2450	const buffer = for (o.data) \|buffer\| {
	2451	if (buffer.len != 0) break buffer;
	2452	} else break :operation .{ .file_read_streaming = 0 };
	2453	const fd = o.file.handle;
	2454	storage.* = .{ .pending = .{
	2455	.node = .{ .prev = batch.pending.tail, .next = .none },
	2456	.tag = .file_read_streaming,
	2457	.context = undefined,
	2458	} };
	2459	thread.enqueue().* = .{
	2460	.opcode = .READ,
	2461	.flags = 0,
	2462	.ioprio = 0,
	2463	.fd = fd,
	2464	.off = std.math.maxInt(u64),
	2465	.addr = @intFromPtr(buffer.ptr),
	2466	.len = @min(buffer.len, 0xfffff000),
	2467	.rw_flags = 0,
	2468	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
	2469	.buf_index = 0,
	2470	.personality = 0,
	2471	.splice_fd_in = 0,
	2472	.addr3 = 0,
	2473	.resv = 0,
	2474	};
	2475	break :operation null;
	2476	},
	2477	.file_write_streaming => \|o\| {
	2478	const buffer = buffer: {
	2479	if (o.header.len != 0) break :buffer o.header;
	2480	for (o.data[0 .. o.data.len - 1]) \|buffer\| {
	2481	if (buffer.len != 0) break :buffer buffer;
	2482	}
	2483	if (o.splat > 0) break :buffer o.data[o.data.len - 1];
	2484	break :operation .{ .file_write_streaming = 0 };
	2485	};
	2486	const fd = o.file.handle;
	2487	storage.* = .{ .pending = .{
	2488	.node = .{ .prev = batch.pending.tail, .next = .none },
	2489	.tag = .file_write_streaming,
	2490	.context = undefined,
	2491	} };
	2492	thread.enqueue().* = .{
	2493	.opcode = .WRITE,
	2494	.flags = 0,
	2495	.ioprio = 0,
	2496	.fd = fd,
	2497	.off = std.math.maxInt(u64),
	2498	.addr = @intFromPtr(buffer.ptr),
	2499	.len = @min(buffer.len, 0xfffff000),
	2500	.rw_flags = 0,
	2501	.user_data = @intFromPtr(&storage.pending.context) \| 0b10,
	2502	.buf_index = 0,
	2503	.personality = 0,
	2504	.splice_fd_in = 0,
	2505	.addr3 = 0,
	2506	.resv = 0,
	2507	};
	2508	break :operation null;
	2509	},
	2510	.device_io_control => \|o\| if (concurrency)
	2511	return error.ConcurrencyUnavailable
	2512	else
	2513	.{ .device_io_control = try deviceIoControl(&o) },
	2514	})) \|result\| {
	2515	switch (batch.completed.tail) {
	2516	.none => batch.completed.head = index,
	2517	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next = index,
	2518	}
	2519	batch.completed.tail = index;
	2520	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2521	} else {
	2522	switch (batch.pending.tail) {
	2523	.none => batch.pending.head = index,
	2524	else => \|tail_index\| batch.storage[tail_index.toIndex()].pending.node.next = index,
	2525	}
	2526	batch.pending.tail = index;
	2527	storage.pending.context[0] = @intFromPtr(batch);
	2528	}
	2529	index = next_index;
	2530	}
	2531	batch.submitted = .{ .head = .none, .tail = .none };
	2532	}
1410	2533
1411		else => \|err\| return std.posix.unexpectedErrno(err),
	2534	fn batchDrainReady(batch: *Io.Batch) Io.Timeout.Error!void {
	2535	while (@atomicRmw(?*anyopaque, &batch.context, .Xchg, null, .acquire)) \|head\| {
	2536	var next: usize = @intFromPtr(head);
	2537	var timeout = false;
	2538	while (cond: switch (@as(u2, @truncate(next))) {
	2539	0b00 => if (timeout) return error.Timeout else false,
	2540	0b01 => {
	2541	assert(!timeout);
	2542	return error.Timeout;
	2543	},
	2544	0b10 => true,
	2545	0b11 => {
	2546	assert(!timeout);
	2547	timeout = true;
	2548	break :cond true;
	2549	},
	2550	}) {
	2551	var context: *Io.Operation.Storage.Pending.Context = @ptrFromInt(next & ~@as(usize, 0b11));
	2552	next = context[0];
	2553	const completion: Completion = .{
	2554	.result = @bitCast(@as(u32, @intCast(context[1]))),
	2555	.flags = @intCast(context[2]),
	2556	};
	2557	const pending: *Io.Operation.Storage.Pending = @fieldParentPtr("context", context);
	2558	const storage: *Io.Operation.Storage = @fieldParentPtr("pending", pending);
	2559	const index: Io.Operation.OptionalIndex = .fromIndex(storage - batch.storage.ptr);
	2560	assert(completion.flags & linux.IORING_CQE_F_SKIP == 0);
	2561	switch (pending.node.prev) {
	2562	.none => batch.pending.head = pending.node.next,
	2563	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.next =
	2564	pending.node.next,
	2565	}
	2566	switch (pending.node.next) {
	2567	.none => batch.pending.tail = pending.node.prev,
	2568	else => \|prev_index\| batch.storage[prev_index.toIndex()].pending.node.prev =
	2569	pending.node.prev,
	2570	}
	2571	if (@as(?Io.Operation.Result, result: switch (pending.tag) {
	2572	.file_read_streaming => .{
	2573	.file_read_streaming = switch (completion.errno()) {
	2574	.SUCCESS => @as(u32, @bitCast(completion.result)),
	2575	.INTR => 0,
	2576	.CANCELED => break :result null,
	2577	.INVAL => \|err\| errnoBug(err),
	2578	.FAULT => \|err\| errnoBug(err),
	2579	.AGAIN => error.WouldBlock,
	2580	.BADF => \|err\| errnoBug(err), // File descriptor used after closed
	2581	.IO => error.InputOutput,
	2582	.ISDIR => error.IsDir,
	2583	.NOBUFS => error.SystemResources,
	2584	.NOMEM => error.SystemResources,
	2585	.NOTCONN => error.SocketUnconnected,
	2586	.CONNRESET => error.ConnectionResetByPeer,
	2587	else => \|err\| unexpectedErrno(err),
	2588	},
	2589	},
	2590	.file_write_streaming => .{
	2591	.file_write_streaming = switch (completion.errno()) {
	2592	.SUCCESS => @as(u32, @bitCast(completion.result)),
	2593	.INTR => 0,
	2594	.CANCELED => break :result null,
	2595	.INVAL => \|err\| errnoBug(err),
	2596	.FAULT => \|err\| errnoBug(err),
	2597	.AGAIN => error.WouldBlock,
	2598	.BADF => error.NotOpenForWriting, // Can be a race condition.
	2599	.DESTADDRREQ => \|err\| errnoBug(err), // `connect` was never called.
	2600	.DQUOT => error.DiskQuota,
	2601	.FBIG => error.FileTooBig,
	2602	.IO => error.InputOutput,
	2603	.NOSPC => error.NoSpaceLeft,
	2604	.PERM => error.PermissionDenied,
	2605	.PIPE => error.BrokenPipe,
	2606	.CONNRESET => \|err\| errnoBug(err), // Not a socket handle.
	2607	.BUSY => error.DeviceBusy,
	2608	else => \|err\| unexpectedErrno(err),
	2609	},
	2610	},
	2611	.device_io_control => unreachable,
	2612	})) \|result\| {
	2613	switch (batch.completed.tail) {
	2614	.none => batch.completed.head = index,
	2615	else => \|tail_index\| batch.storage[tail_index.toIndex()].completion.node.next =
	2616	index,
	2617	}
	2618	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2619	batch.completed.tail = index;
	2620	} else {
	2621	switch (batch.unused.tail) {
	2622	.none => batch.unused.head = index,
	2623	else => \|tail_index\| batch.storage[tail_index.toIndex()].unused.next = index,
	2624	}
	2625	storage.* = .{ .unused = .{ .prev = batch.unused.tail, .next = .none } };
	2626	batch.unused.tail = index;
	2627	}
	2628	}
1412	2629	}
1413	2630	}
1414	2631
1415		fn mutexLock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) error{Canceled}!void {
1416		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1417		el.yield(null, .{ .mutex_lock = .{ .prev_state = prev_state, .mutex = mutex } });
	2632	fn batchCancel(userdata: ?anyopaque, batch: Io.Batch) void {
	2633	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2634	_ = ev;
	2635	batchDrainReady(batch) catch \|err\| switch (err) {
	2636	error.Timeout => unreachable, // no timeout
	2637	};
	2638	var index = batch.pending.head;
	2639	if (index == .none) return;
	2640	var cancel_region: CancelRegion = .initBlocked();
	2641	defer cancel_region.deinit();
	2642	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	2643	error.Canceled => unreachable, // blocked
	2644	};
	2645	while (index != .none) {
	2646	const pending = &batch.storage[index.toIndex()].pending;
	2647	thread.enqueue().* = .{
	2648	.opcode = .ASYNC_CANCEL,
	2649	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	2650	.ioprio = 0,
	2651	.fd = 0,
	2652	.off = 0,
	2653	.addr = @intFromPtr(&pending.context) \| 0b10,
	2654	.len = 0,
	2655	.rw_flags = 0,
	2656	.user_data = @intFromEnum(Completion.UserData.wakeup),
	2657	.buf_index = 0,
	2658	.personality = 0,
	2659	.splice_fd_in = 0,
	2660	.addr3 = 0,
	2661	.resv = 0,
	2662	};
	2663	index = pending.node.next;
	2664	}
	2665	while (batch.pending.head != .none) batchDrainReady(batch) catch \|err\| switch (err) {
	2666	error.Timeout => unreachable, // no timeout
	2667	};
1418	2668	}
1419		fn mutexUnlock(userdata: ?anyopaque, prev_state: Io.Mutex.State, mutex: Io.Mutex) void {
1420		var maybe_waiting_fiber: ?*Fiber = @ptrFromInt(@intFromEnum(prev_state));
1421		while (if (maybe_waiting_fiber) \|waiting_fiber\| @cmpxchgWeak(
1422		Io.Mutex.State,
1423		&mutex.state,
1424		@enumFromInt(@intFromPtr(waiting_fiber)),
1425		@enumFromInt(@intFromPtr(waiting_fiber.queue_next)),
1426		.release,
1427		.acquire,
1428		) else @cmpxchgWeak(
1429		Io.Mutex.State,
1430		&mutex.state,
1431		.locked_once,
1432		.unlocked,
1433		.release,
1434		.acquire,
1435		) orelse return) \|next_state\| maybe_waiting_fiber = @ptrFromInt(@intFromEnum(next_state));
1436		maybe_waiting_fiber.?.queue_next = null;
1437		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1438		el.yield(maybe_waiting_fiber.?, .reschedule);
	2669
	2670	fn dirCreateDir(
	2671	userdata: ?*anyopaque,
	2672	dir: Dir,
	2673	sub_path: []const u8,
	2674	permissions: Dir.Permissions,
	2675	) Dir.CreateDirError!void {
	2676	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2677
	2678	var path_buffer: [PATH_MAX]u8 = undefined;
	2679	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2680
	2681	var cancel_region: CancelRegion = .init();
	2682	defer cancel_region.deinit();
	2683	while (true) {
	2684	const thread = try cancel_region.awaitIoUring();
	2685	thread.enqueue().* = .{
	2686	.opcode = .MKDIRAT,
	2687	.flags = 0,
	2688	.ioprio = 0,
	2689	.fd = dir.handle,
	2690	.off = 0,
	2691	.addr = @intFromPtr(sub_path_posix.ptr),
	2692	.len = permissions.toMode(),
	2693	.rw_flags = 0,
	2694	.user_data = @intFromPtr(cancel_region.fiber),
	2695	.buf_index = 0,
	2696	.personality = 0,
	2697	.splice_fd_in = 0,
	2698	.addr3 = 0,
	2699	.resv = 0,
	2700	};
	2701	ev.yield(null, .nothing);
	2702	switch (cancel_region.errno()) {
	2703	.SUCCESS => return,
	2704	.INTR, .CANCELED => continue,
	2705	.ACCES => return error.AccessDenied,
	2706	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	2707	.PERM => return error.PermissionDenied,
	2708	.DQUOT => return error.DiskQuota,
	2709	.EXIST => return error.PathAlreadyExists,
	2710	.FAULT => \|err\| return errnoBug(err),
	2711	.LOOP => return error.SymLinkLoop,
	2712	.MLINK => return error.LinkQuotaExceeded,
	2713	.NAMETOOLONG => return error.NameTooLong,
	2714	.NOENT => return error.FileNotFound,
	2715	.NOMEM => return error.SystemResources,
	2716	.NOSPC => return error.NoSpaceLeft,
	2717	.NOTDIR => return error.NotDir,
	2718	.ROFS => return error.ReadOnlyFileSystem,
	2719	// dragonfly: when dir_fd is unlinked from filesystem
	2720	.NOTCONN => return error.FileNotFound,
	2721	.ILSEQ => return error.BadPathName,
	2722	else => \|err\| return unexpectedErrno(err),
	2723	}
	2724	}
1439	2725	}
1440	2726
1441		const ConditionImpl = struct {
1442		tail: *Fiber,
1443		event: union(enum) {
1444		queued,
1445		wake: Io.Condition.Wake,
1446		},
1447		};
	2727	fn dirCreateDirPath(
	2728	userdata: ?*anyopaque,
	2729	dir: Dir,
	2730	sub_path: []const u8,
	2731	permissions: Dir.Permissions,
	2732	) Dir.CreateDirPathError!Dir.CreatePathStatus {
	2733	const ev: *Evented = @ptrCast(@alignCast(userdata));
1448	2734
1449		fn conditionWait(userdata: ?anyopaque, cond: Io.Condition, mutex: *Io.Mutex) Io.Cancelable!void {
1450		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1451		el.yield(null, .{ .condition_wait = .{ .cond = cond, .mutex = mutex } });
1452		const thread = Thread.current();
1453		const fiber = thread.currentFiber();
1454		const cond_impl = fiber.resultPointer(ConditionImpl);
1455		try mutex.lock(el.io());
1456		switch (cond_impl.event) {
1457		.queued => {},
1458		.wake => \|wake\| if (fiber.queue_next) \|next_fiber\| switch (wake) {
1459		.one => if (@cmpxchgStrong(
1460		?*Fiber,
1461		@as(?Fiber, @ptrCast(&cond.state)),
1462		null,
1463		next_fiber,
1464		.release,
1465		.acquire,
1466		)) \|old_fiber\| {
1467		const old_cond_impl = old_fiber.?.resultPointer(ConditionImpl);
1468		assert(old_cond_impl.tail.queue_next == null);
1469		old_cond_impl.tail.queue_next = next_fiber;
1470		old_cond_impl.tail = cond_impl.tail;
	2735	var it = Dir.path.componentIterator(sub_path);
	2736	var status: Dir.CreatePathStatus = .existed;
	2737	var component = it.last() orelse return error.BadPathName;
	2738	while (true) {
	2739	if (dirCreateDir(ev, dir, component.path, permissions)) \|_\| {
	2740	status = .created;
	2741	} else \|err\| switch (err) {
	2742	error.PathAlreadyExists => {
	2743	// stat the file and return an error if it's not a directory
	2744	// this is important because otherwise a dangling symlink
	2745	// could cause an infinite loop
	2746	const fstat = try dirStatFile(ev, dir, component.path, .{});
	2747	if (fstat.kind != .directory) return error.NotDir;
1471	2748	},
1472		.all => el.schedule(thread, .{ .head = next_fiber, .tail = cond_impl.tail }),
	2749	error.FileNotFound => \|e\| {
	2750	component = it.previous() orelse return e;
	2751	continue;
	2752	},
	2753	else => \|e\| return e,
	2754	}
	2755	component = it.next() orelse return status;
	2756	}
	2757	}
	2758
	2759	fn dirCreateDirPathOpen(
	2760	userdata: ?*anyopaque,
	2761	dir: Dir,
	2762	sub_path: []const u8,
	2763	permissions: Dir.Permissions,
	2764	options: Dir.OpenOptions,
	2765	) Dir.CreateDirPathOpenError!Dir {
	2766	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2767	return dirOpenDir(ev, dir, sub_path, options) catch \|err\| switch (err) {
	2768	error.FileNotFound => {
	2769	_ = try dirCreateDirPath(ev, dir, sub_path, permissions);
	2770	return dirOpenDir(ev, dir, sub_path, options);
	2771	},
	2772	else => \|e\| return e,
	2773	};
	2774	}
	2775
	2776	fn dirOpenDir(
	2777	userdata: ?*anyopaque,
	2778	dir: Dir,
	2779	sub_path: []const u8,
	2780	options: Dir.OpenOptions,
	2781	) Dir.OpenError!Dir {
	2782	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2783
	2784	var path_buffer: [PATH_MAX]u8 = undefined;
	2785	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2786
	2787	var cancel_region: CancelRegion = .init();
	2788	defer cancel_region.deinit();
	2789	return .{
	2790	.handle = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	2791	.ACCMODE = .RDONLY,
	2792	.DIRECTORY = true,
	2793	.NOFOLLOW = !options.follow_symlinks,
	2794	.CLOEXEC = true,
	2795	.PATH = !options.iterate,
	2796	}, 0) catch \|err\| switch (err) {
	2797	error.IsDir => return errnoBug(.ISDIR),
	2798	error.WouldBlock => return errnoBug(.AGAIN),
	2799	error.FileTooBig => return errnoBug(.FBIG),
	2800	error.NoSpaceLeft => return errnoBug(.NOSPC),
	2801	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
	2802	error.FileBusy => return errnoBug(.TXTBSY),
	2803	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
	2804	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2805	error.AntivirusInterference => unreachable, // Windows-only
	2806	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	2807	else => \|e\| return e,
1473	2808	},
	2809	};
	2810	}
	2811
	2812	fn dirStat(userdata: ?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat {
	2813	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2814	var cancel_region: CancelRegion = .init();
	2815	defer cancel_region.deinit();
	2816	return ev.stat(&cancel_region, dir.handle);
	2817	}
	2818
	2819	fn dirStatFile(
	2820	userdata: ?*anyopaque,
	2821	dir: Dir,
	2822	sub_path: []const u8,
	2823	options: Dir.StatFileOptions,
	2824	) Dir.StatFileError!File.Stat {
	2825	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2826	var path_buffer: [PATH_MAX]u8 = undefined;
	2827	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2828	var cancel_region: CancelRegion = .init();
	2829	defer cancel_region.deinit();
	2830	return ev.statx(&cancel_region, dir.handle, sub_path_posix, linux.AT.NO_AUTOMOUNT \|
	2831	@as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW));
	2832	}
	2833
	2834	fn dirAccess(
	2835	userdata: ?*anyopaque,
	2836	dir: Dir,
	2837	sub_path: []const u8,
	2838	options: Dir.AccessOptions,
	2839	) Dir.AccessError!void {
	2840	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2841	_ = ev;
	2842
	2843	var path_buffer: [PATH_MAX]u8 = undefined;
	2844	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2845
	2846	const mode: u32 =
	2847	@as(u32, if (options.read) linux.R_OK else 0) \|
	2848	@as(u32, if (options.write) linux.W_OK else 0) \|
	2849	@as(u32, if (options.execute) linux.X_OK else 0);
	2850	const flags: u32 = if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW;
	2851
	2852	var cancel_region: CancelRegion = .init();
	2853	defer cancel_region.deinit();
	2854	while (true) {
	2855	try cancel_region.await(.nothing);
	2856	switch (linux.errno(linux.faccessat(dir.handle, sub_path_posix, mode, flags))) {
	2857	.SUCCESS => return,
	2858	.INTR => continue,
	2859	.ACCES => return error.AccessDenied,
	2860	.PERM => return error.PermissionDenied,
	2861	.ROFS => return error.ReadOnlyFileSystem,
	2862	.LOOP => return error.SymLinkLoop,
	2863	.TXTBSY => return error.FileBusy,
	2864	.NOTDIR => return error.FileNotFound,
	2865	.NOENT => return error.FileNotFound,
	2866	.NAMETOOLONG => return error.NameTooLong,
	2867	.INVAL => \|err\| return errnoBug(err),
	2868	.FAULT => \|err\| return errnoBug(err),
	2869	.IO => return error.InputOutput,
	2870	.NOMEM => return error.SystemResources,
	2871	.ILSEQ => return error.BadPathName,
	2872	else => \|err\| return unexpectedErrno(err),
	2873	}
1474	2874	}
1475		fiber.queue_next = null;
1476	2875	}
1477	2876
1478		fn conditionWake(userdata: ?anyopaque, cond: Io.Condition, wake: Io.Condition.Wake) void {
1479		const el: *EventLoop = @ptrCast(@alignCast(userdata));
1480		const waiting_fiber = @atomicRmw(?Fiber, @as(?*Fiber, @ptrCast(&cond.state)), .Xchg, null, .acquire) orelse return;
1481		waiting_fiber.resultPointer(ConditionImpl).event = .{ .wake = wake };
1482		el.yield(waiting_fiber, .reschedule);
	2877	fn dirCreateFile(
	2878	userdata: ?*anyopaque,
	2879	dir: Dir,
	2880	sub_path: []const u8,
	2881	flags: File.CreateFlags,
	2882	) File.OpenError!File {
	2883	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2884
	2885	var path_buffer: [PATH_MAX]u8 = undefined;
	2886	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	2887
	2888	var cancel_region: CancelRegion = .init();
	2889	defer cancel_region.deinit();
	2890	const fd = try ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	2891	.ACCMODE = if (flags.read) .RDWR else .WRONLY,
	2892	.CREAT = true,
	2893	.TRUNC = flags.truncate,
	2894	.EXCL = flags.exclusive,
	2895	.CLOEXEC = true,
	2896	}, flags.permissions.toMode());
	2897	errdefer ev.close(fd);
	2898
	2899	switch (flags.lock) {
	2900	.none => {},
	2901	.shared, .exclusive => try ev.flock(
	2902	&cancel_region,
	2903	fd,
	2904	flags.lock,
	2905	if (flags.lock_nonblocking) .nonblocking else .blocking,
	2906	),
	2907	}
	2908
	2909	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
1483	2910	}
1484	2911
1485		fn errno(signed: i32) std.os.linux.E {
1486		return .init(@bitCast(@as(isize, signed)));
	2912	fn dirCreateFileAtomic(
	2913	userdata: ?*anyopaque,
	2914	dir: Dir,
	2915	dest_path: []const u8,
	2916	options: Dir.CreateFileAtomicOptions,
	2917	) Dir.CreateFileAtomicError!File.Atomic {
	2918	const ev: *Evented = @ptrCast(@alignCast(userdata));
	2919	// Linux has O_TMPFILE, but linkat() does not support AT_REPLACE, so it's
	2920	// useless when we have to make up a bogus path name to do the rename()
	2921	// anyway.
	2922	if (!options.replace) tmpfile: {
	2923	const flags: linux.O = if (@hasField(linux.O, "TMPFILE")) .{
	2924	.ACCMODE = .RDWR,
	2925	.TMPFILE = true,
	2926	.DIRECTORY = true,
	2927	.CLOEXEC = true,
	2928	} else if (@hasField(linux.O, "TMPFILE0") and !@hasField(linux.O, "TMPFILE2")) .{
	2929	.ACCMODE = .RDWR,
	2930	.TMPFILE0 = true,
	2931	.TMPFILE1 = true,
	2932	.DIRECTORY = true,
	2933	.CLOEXEC = true,
	2934	} else break :tmpfile;
	2935
	2936	const dest_dirname = Dir.path.dirname(dest_path);
	2937	if (dest_dirname) \|dirname\| {
	2938	// This has a nice side effect of preemptively triggering EISDIR or
	2939	// ENOENT, avoiding the ambiguity below.
	2940	_ = dirCreateDirPath(ev, dir, dirname, .default_dir) catch \|err\| switch (err) {
	2941	// None of these make sense in this context.
	2942	error.IsDir,
	2943	error.Streaming,
	2944	error.DiskQuota,
	2945	error.PathAlreadyExists,
	2946	error.LinkQuotaExceeded,
	2947	error.PipeBusy,
	2948	error.FileTooBig,
	2949	error.DeviceBusy,
	2950	error.FileLocksUnsupported,
	2951	error.FileBusy,
	2952	=> return error.Unexpected,
	2953
	2954	else => \|e\| return e,
	2955	};
	2956	}
	2957
	2958	var path_buffer: [PATH_MAX]u8 = undefined;
	2959	const sub_path_posix = try pathToPosix(dest_dirname orelse ".", &path_buffer);
	2960
	2961	var cancel_region: CancelRegion = .init();
	2962	defer cancel_region.deinit();
	2963	return .{
	2964	.file = .{
	2965	.handle = ev.openat(
	2966	&cancel_region,
	2967	dir.handle,
	2968	sub_path_posix,
	2969	flags,
	2970	options.permissions.toMode(),
	2971	) catch \|err\| switch (err) {
	2972	error.IsDir, error.FileNotFound => {
	2973	// Ambiguous error code. It might mean the file system
	2974	// does not support O_TMPFILE. Therefore, we must fall
	2975	// back to not using O_TMPFILE.
	2976	break :tmpfile;
	2977	},
	2978	error.FileTooBig => return errnoBug(.FBIG),
	2979	error.DeviceBusy => return errnoBug(.BUSY), // O_EXCL not passed
	2980	error.PathAlreadyExists => return errnoBug(.EXIST), // Not creating.
	2981	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	2982	error.AntivirusInterference => unreachable, // Windows-only
	2983	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	2984	else => \|e\| return e,
	2985	},
	2986	.flags = .{ .nonblocking = false },
	2987	},
	2988	.file_basename_hex = 0,
	2989	.dest_sub_path = dest_path,
	2990	.file_open = true,
	2991	.file_exists = false,
	2992	.close_dir_on_deinit = false,
	2993	.dir = dir,
	2994	};
	2995	}
	2996
	2997	if (Dir.path.dirname(dest_path)) \|dirname\| {
	2998	const new_dir = if (options.make_path)
	2999	dirCreateDirPathOpen(ev, dir, dirname, .default_dir, .{}) catch \|err\| switch (err) {
	3000	// None of these make sense in this context.
	3001	error.IsDir,
	3002	error.Streaming,
	3003	error.DiskQuota,
	3004	error.PathAlreadyExists,
	3005	error.LinkQuotaExceeded,
	3006	error.PipeBusy,
	3007	error.FileTooBig,
	3008	error.FileLocksUnsupported,
	3009	error.DeviceBusy,
	3010	=> return error.Unexpected,
	3011
	3012	else => \|e\| return e,
	3013	}
	3014	else
	3015	try dirOpenDir(ev, dir, dirname, .{});
	3016
	3017	return ev.atomicFileInit(Dir.path.basename(dest_path), options.permissions, new_dir, true);
	3018	}
	3019
	3020	return ev.atomicFileInit(dest_path, options.permissions, dir, false);
1487	3021	}
1488	3022
1489		fn getSqe(iou: IoUring) std.os.linux.io_uring_sqe {
1490		while (true) return iou.get_sqe() catch {
1491		_ = iou.submit_and_wait(0) catch \|err\| switch (err) {
1492		error.SignalInterrupt => std.log.warn("submit_and_wait failed with SignalInterrupt", .{}),
1493		else => \|e\| @panic(@errorName(e)),
	3023	fn atomicFileInit(
	3024	ev: *Evented,
	3025	dest_basename: []const u8,
	3026	permissions: File.Permissions,
	3027	dir: Dir,
	3028	close_dir_on_deinit: bool,
	3029	) Dir.CreateFileAtomicError!File.Atomic {
	3030	while (true) {
	3031	var random_integer: u64 = undefined;
	3032	random(ev, @ptrCast(&random_integer));
	3033	const tmp_sub_path = std.fmt.hex(random_integer);
	3034	const file = dirCreateFile(ev, dir, &tmp_sub_path, .{
	3035	.permissions = permissions,
	3036	.exclusive = true,
	3037	}) catch \|err\| switch (err) {
	3038	error.PathAlreadyExists => continue,
	3039	error.DeviceBusy => continue,
	3040	error.FileBusy => continue,
	3041
	3042	error.IsDir => return error.Unexpected, // No path components.
	3043	error.FileTooBig => return error.Unexpected, // Creating, not opening.
	3044	error.FileLocksUnsupported => return error.Unexpected, // Not asking for locks.
	3045	error.PipeBusy => return error.Unexpected, // Not opening a pipe.
	3046
	3047	else => \|e\| return e,
	3048	};
	3049	return .{
	3050	.file = file,
	3051	.file_basename_hex = random_integer,
	3052	.dest_sub_path = dest_basename,
	3053	.file_open = true,
	3054	.file_exists = true,
	3055	.close_dir_on_deinit = close_dir_on_deinit,
	3056	.dir = dir,
	3057	};
	3058	}
	3059	}
	3060
	3061	fn dirOpenFile(
	3062	userdata: ?*anyopaque,
	3063	dir: Dir,
	3064	sub_path: []const u8,
	3065	flags: File.OpenFlags,
	3066	) File.OpenError!File {
	3067	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3068
	3069	var path_buffer: [PATH_MAX]u8 = undefined;
	3070	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3071
	3072	var cancel_region: CancelRegion = .init();
	3073	defer cancel_region.deinit();
	3074	const fd = try ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	3075	.ACCMODE = switch (flags.mode) {
	3076	.read_only => .RDONLY,
	3077	.write_only => .WRONLY,
	3078	.read_write => .RDWR,
	3079	},
	3080	.NOCTTY = !flags.allow_ctty,
	3081	.NOFOLLOW = !flags.follow_symlinks,
	3082	.CLOEXEC = true,
	3083	.PATH = flags.path_only,
	3084	}, 0);
	3085	errdefer ev.close(fd);
	3086
	3087	if (!flags.allow_directory) {
	3088	const is_dir = is_dir: {
	3089	const s = ev.stat(&cancel_region, fd) catch \|err\| switch (err) {
	3090	// The directory-ness is either unknown or unknowable
	3091	error.Streaming => break :is_dir false,
	3092	else => \|e\| return e,
	3093	};
	3094	break :is_dir s.kind == .directory;
	3095	};
	3096	if (is_dir) return error.IsDir;
	3097	}
	3098
	3099	switch (flags.lock) {
	3100	.none => {},
	3101	.shared, .exclusive => try ev.flock(
	3102	&cancel_region,
	3103	fd,
	3104	flags.lock,
	3105	if (flags.lock_nonblocking) .nonblocking else .blocking,
	3106	),
	3107	}
	3108
	3109	return .{ .handle = fd, .flags = .{ .nonblocking = false } };
	3110	}
	3111
	3112	fn dirClose(userdata: ?*anyopaque, dirs: []const Dir) void {
	3113	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3114	for (dirs) \|dir\| ev.close(dir.handle);
	3115	}
	3116
	3117	fn dirRead(userdata: ?anyopaque, dr: Dir.Reader, buffer: []Dir.Entry) Dir.Reader.Error!usize {
	3118	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3119	var buffer_index: usize = 0;
	3120	while (buffer.len - buffer_index != 0) {
	3121	if (dr.end - dr.index == 0) {
	3122	// Refill the buffer, unless we've already created references to
	3123	// buffered data.
	3124	if (buffer_index != 0) break;
	3125	var cancel_region: CancelRegion = .init();
	3126	defer cancel_region.deinit();
	3127	if (dr.state == .reset) {
	3128	ev.lseek(&cancel_region, dr.dir.handle, 0, linux.SEEK.SET) catch \|err\| switch (err) {
	3129	error.Unseekable => return error.Unexpected,
	3130	else => \|e\| return e,
	3131	};
	3132	dr.state = .reading;
	3133	}
	3134	const n = while (true) {
	3135	try cancel_region.await(.nothing);
	3136	const rc = linux.getdents64(dr.dir.handle, dr.buffer.ptr, dr.buffer.len);
	3137	switch (linux.errno(rc)) {
	3138	.SUCCESS => break rc,
	3139	.INTR => continue,
	3140	.BADF => \|err\| return errnoBug(err), // Dir is invalid or was opened without iteration ability.
	3141	.FAULT => \|err\| return errnoBug(err),
	3142	.NOTDIR => \|err\| return errnoBug(err),
	3143	// To be consistent across platforms, iteration
	3144	// ends if the directory being iterated is deleted
	3145	// during iteration. This matches the behavior of
	3146	// non-Linux, non-WASI UNIX platforms.
	3147	.NOENT => {
	3148	dr.state = .finished;
	3149	return 0;
	3150	},
	3151	// This can occur when reading /proc/$PID/net, or
	3152	// if the provided buffer is too small. Neither
	3153	// scenario is intended to be handled by this API.
	3154	.INVAL => return error.Unexpected,
	3155	.ACCES => return error.AccessDenied, // Lacking permission to iterate this directory.
	3156	else => \|err\| return unexpectedErrno(err),
	3157	}
	3158	};
	3159	if (n == 0) {
	3160	dr.state = .finished;
	3161	return 0;
	3162	}
	3163	dr.index = 0;
	3164	dr.end = n;
	3165	}
	3166	// Linux aligns the header by padding after the null byte of the name
	3167	// to align the next entry. This means we can find the end of the name
	3168	// by looking at only the 8 bytes before the next record. However since
	3169	// file names are usually short it's better to keep the machine code
	3170	// simpler.
	3171	//
	3172	// Furthermore, I observed qemu user mode to not align this struct, so
	3173	// this code makes the conservative choice to not assume alignment.
	3174	const linux_entry: *align(1) linux.dirent64 = @ptrCast(&dr.buffer[dr.index]);
	3175	const next_index = dr.index + linux_entry.reclen;
	3176	dr.index = next_index;
	3177	const name_ptr: [*]u8 = &linux_entry.name;
	3178	const padded_name = name_ptr[0 .. linux_entry.reclen - @offsetOf(linux.dirent64, "name")];
	3179	const name_len = std.mem.findScalar(u8, padded_name, 0).?;
	3180	const name = name_ptr[0..name_len :0];
	3181
	3182	if (std.mem.eql(u8, name, ".") or std.mem.eql(u8, name, "..")) continue;
	3183
	3184	const entry_kind: File.Kind = switch (linux_entry.type) {
	3185	linux.DT.BLK => .block_device,
	3186	linux.DT.CHR => .character_device,
	3187	linux.DT.DIR => .directory,
	3188	linux.DT.FIFO => .named_pipe,
	3189	linux.DT.LNK => .sym_link,
	3190	linux.DT.REG => .file,
	3191	linux.DT.SOCK => .unix_domain_socket,
	3192	else => .unknown,
1494	3193	};
1495		continue;
	3194	buffer[buffer_index] = .{
	3195	.name = name,
	3196	.kind = entry_kind,
	3197	.inode = linux_entry.ino,
	3198	};
	3199	buffer_index += 1;
	3200	}
	3201	return buffer_index;
	3202	}
	3203
	3204	fn dirRealPath(userdata: ?*anyopaque, dir: Dir, out_buffer: []u8) Dir.RealPathError!usize {
	3205	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3206	var cancel_region: CancelRegion = .init();
	3207	defer cancel_region.deinit();
	3208	return ev.realPath(&cancel_region, dir.handle, out_buffer);
	3209	}
	3210
	3211	fn dirRealPathFile(
	3212	userdata: ?*anyopaque,
	3213	dir: Dir,
	3214	sub_path: []const u8,
	3215	out_buffer: []u8,
	3216	) Dir.RealPathFileError!usize {
	3217	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3218
	3219	var path_buffer: [PATH_MAX]u8 = undefined;
	3220	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3221
	3222	var cancel_region: CancelRegion = .init();
	3223	defer cancel_region.deinit();
	3224	const fd = ev.openat(&cancel_region, dir.handle, sub_path_posix, .{
	3225	.CLOEXEC = true,
	3226	.PATH = true,
	3227	}, 0) catch \|err\| switch (err) {
	3228	error.WouldBlock => return errnoBug(.AGAIN),
	3229	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Not asking for locks.
	3230	else => \|e\| return e,
1496	3231	};
	3232	defer ev.close(fd);
	3233	return ev.realPath(&cancel_region, fd, out_buffer);
	3234	}
	3235
	3236	fn dirDeleteFile(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteFileError!void {
	3237	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3238
	3239	var path_buffer: [PATH_MAX]u8 = undefined;
	3240	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3241
	3242	var cancel_region: CancelRegion = .init();
	3243	defer cancel_region.deinit();
	3244	while (true) {
	3245	const thread = try cancel_region.awaitIoUring();
	3246	thread.enqueue().* = .{
	3247	.opcode = .UNLINKAT,
	3248	.flags = 0,
	3249	.ioprio = 0,
	3250	.fd = dir.handle,
	3251	.off = 0,
	3252	.addr = @intFromPtr(sub_path_posix.ptr),
	3253	.len = 0,
	3254	.rw_flags = 0,
	3255	.user_data = @intFromPtr(cancel_region.fiber),
	3256	.buf_index = 0,
	3257	.personality = 0,
	3258	.splice_fd_in = 0,
	3259	.addr3 = 0,
	3260	.resv = 0,
	3261	};
	3262	ev.yield(null, .nothing);
	3263	switch (cancel_region.errno()) {
	3264	.SUCCESS => return,
	3265	.INTR, .CANCELED => continue,
	3266	.PERM => return error.PermissionDenied,
	3267	.ACCES => return error.AccessDenied,
	3268	.BUSY => return error.FileBusy,
	3269	.FAULT => \|err\| return errnoBug(err),
	3270	.IO => return error.FileSystem,
	3271	.ISDIR => return error.IsDir,
	3272	.LOOP => return error.SymLinkLoop,
	3273	.NAMETOOLONG => return error.NameTooLong,
	3274	.NOENT => return error.FileNotFound,
	3275	.NOTDIR => return error.NotDir,
	3276	.NOMEM => return error.SystemResources,
	3277	.ROFS => return error.ReadOnlyFileSystem,
	3278	.EXIST => \|err\| return errnoBug(err),
	3279	.NOTEMPTY => \|err\| return errnoBug(err), // Not passing AT.REMOVEDIR
	3280	.ILSEQ => return error.BadPathName,
	3281	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	3282	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3283	else => \|err\| return unexpectedErrno(err),
	3284	}
	3285	}
	3286	}
	3287
	3288	fn dirDeleteDir(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8) Dir.DeleteDirError!void {
	3289	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3290
	3291	var path_buffer: [PATH_MAX]u8 = undefined;
	3292	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3293
	3294	var cancel_region: CancelRegion = .init();
	3295	defer cancel_region.deinit();
	3296	while (true) {
	3297	const thread = try cancel_region.awaitIoUring();
	3298	thread.enqueue().* = .{
	3299	.opcode = .UNLINKAT,
	3300	.flags = 0,
	3301	.ioprio = 0,
	3302	.fd = dir.handle,
	3303	.off = 0,
	3304	.addr = @intFromPtr(sub_path_posix.ptr),
	3305	.len = 0,
	3306	.rw_flags = linux.AT.REMOVEDIR,
	3307	.user_data = @intFromPtr(cancel_region.fiber),
	3308	.buf_index = 0,
	3309	.personality = 0,
	3310	.splice_fd_in = 0,
	3311	.addr3 = 0,
	3312	.resv = 0,
	3313	};
	3314	ev.yield(null, .nothing);
	3315	switch (cancel_region.errno()) {
	3316	.SUCCESS => return,
	3317	.INTR, .CANCELED => continue,
	3318	.ACCES => return error.AccessDenied,
	3319	.PERM => return error.PermissionDenied,
	3320	.BUSY => return error.FileBusy,
	3321	.FAULT => \|err\| return errnoBug(err),
	3322	.IO => return error.FileSystem,
	3323	.ISDIR => \|err\| return errnoBug(err),
	3324	.LOOP => return error.SymLinkLoop,
	3325	.NAMETOOLONG => return error.NameTooLong,
	3326	.NOENT => return error.FileNotFound,
	3327	.NOTDIR => return error.NotDir,
	3328	.NOMEM => return error.SystemResources,
	3329	.ROFS => return error.ReadOnlyFileSystem,
	3330	.EXIST => \|err\| return errnoBug(err),
	3331	.NOTEMPTY => return error.DirNotEmpty,
	3332	.ILSEQ => return error.BadPathName,
	3333	.INVAL => \|err\| return errnoBug(err), // invalid flags, or pathname has . as last component
	3334	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3335	else => \|err\| return unexpectedErrno(err),
	3336	}
	3337	}
	3338	}
	3339
	3340	fn dirRename(
	3341	userdata: ?*anyopaque,
	3342	old_dir: Dir,
	3343	old_sub_path: []const u8,
	3344	new_dir: Dir,
	3345	new_sub_path: []const u8,
	3346	) Dir.RenameError!void {
	3347	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3348
	3349	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3350	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3351
	3352	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3353	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3354
	3355	var cancel_region: CancelRegion = .init();
	3356	defer cancel_region.deinit();
	3357	return ev.renameat(
	3358	&cancel_region,
	3359	old_dir.handle,
	3360	old_sub_path_posix,
	3361	new_dir.handle,
	3362	new_sub_path_posix,
	3363	.{},
	3364	);
	3365	}
	3366
	3367	fn dirRenamePreserve(
	3368	userdata: ?*anyopaque,
	3369	old_dir: Dir,
	3370	old_sub_path: []const u8,
	3371	new_dir: Dir,
	3372	new_sub_path: []const u8,
	3373	) Dir.RenamePreserveError!void {
	3374	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3375
	3376	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3377	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3378
	3379	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3380	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3381
	3382	var cancel_region: CancelRegion = .init();
	3383	defer cancel_region.deinit();
	3384	return ev.renameat(
	3385	&cancel_region,
	3386	old_dir.handle,
	3387	old_sub_path_posix,
	3388	new_dir.handle,
	3389	new_sub_path_posix,
	3390	.{ .NOREPLACE = true },
	3391	);
	3392	}
	3393
	3394	fn dirSymLink(
	3395	userdata: ?*anyopaque,
	3396	dir: Dir,
	3397	target_path: []const u8,
	3398	sym_link_path: []const u8,
	3399	flags: Dir.SymLinkFlags,
	3400	) Dir.SymLinkError!void {
	3401	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3402	_ = flags;
	3403
	3404	var target_path_buffer: [PATH_MAX]u8 = undefined;
	3405	var sym_link_path_buffer: [PATH_MAX]u8 = undefined;
	3406
	3407	const target_path_posix = try pathToPosix(target_path, &target_path_buffer);
	3408	const sym_link_path_posix = try pathToPosix(sym_link_path, &sym_link_path_buffer);
	3409
	3410	var cancel_region: CancelRegion = .init();
	3411	defer cancel_region.deinit();
	3412	while (true) {
	3413	const thread = try cancel_region.awaitIoUring();
	3414	thread.enqueue().* = .{
	3415	.opcode = .SYMLINKAT,
	3416	.flags = 0,
	3417	.ioprio = 0,
	3418	.fd = dir.handle,
	3419	.off = @intFromPtr(sym_link_path_posix.ptr),
	3420	.addr = @intFromPtr(target_path_posix.ptr),
	3421	.len = 0,
	3422	.rw_flags = 0,
	3423	.user_data = @intFromPtr(cancel_region.fiber),
	3424	.buf_index = 0,
	3425	.personality = 0,
	3426	.splice_fd_in = 0,
	3427	.addr3 = 0,
	3428	.resv = 0,
	3429	};
	3430	ev.yield(null, .nothing);
	3431	switch (cancel_region.errno()) {
	3432	.SUCCESS => return,
	3433	.INTR, .CANCELED => continue,
	3434	.FAULT => \|err\| return errnoBug(err),
	3435	.INVAL => \|err\| return errnoBug(err),
	3436	.ACCES => return error.AccessDenied,
	3437	.PERM => return error.PermissionDenied,
	3438	.DQUOT => return error.DiskQuota,
	3439	.EXIST => return error.PathAlreadyExists,
	3440	.IO => return error.FileSystem,
	3441	.LOOP => return error.SymLinkLoop,
	3442	.NAMETOOLONG => return error.NameTooLong,
	3443	.NOENT => return error.FileNotFound,
	3444	.NOTDIR => return error.NotDir,
	3445	.NOMEM => return error.SystemResources,
	3446	.NOSPC => return error.NoSpaceLeft,
	3447	.ROFS => return error.ReadOnlyFileSystem,
	3448	.ILSEQ => return error.BadPathName,
	3449	else => \|err\| return unexpectedErrno(err),
	3450	}
	3451	}
	3452	}
	3453
	3454	fn dirReadLink(
	3455	userdata: ?*anyopaque,
	3456	dir: Dir,
	3457	sub_path: []const u8,
	3458	buffer: []u8,
	3459	) Dir.ReadLinkError!usize {
	3460	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3461	_ = ev;
	3462
	3463	var sub_path_buffer: [PATH_MAX]u8 = undefined;
	3464	const sub_path_posix = try pathToPosix(sub_path, &sub_path_buffer);
	3465
	3466	var cancel_region: CancelRegion = .init();
	3467	defer cancel_region.deinit();
	3468	while (true) {
	3469	try cancel_region.await(.nothing);
	3470	const rc = linux.readlinkat(dir.handle, sub_path_posix, buffer.ptr, buffer.len);
	3471	switch (linux.errno(rc)) {
	3472	.SUCCESS => {
	3473	const len: usize = @bitCast(rc);
	3474	return len;
	3475	},
	3476	.INTR => continue,
	3477	.ACCES => return error.AccessDenied,
	3478	.FAULT => \|err\| return errnoBug(err),
	3479	.INVAL => return error.NotLink,
	3480	.IO => return error.FileSystem,
	3481	.LOOP => return error.SymLinkLoop,
	3482	.NAMETOOLONG => return error.NameTooLong,
	3483	.NOENT => return error.FileNotFound,
	3484	.NOMEM => return error.SystemResources,
	3485	.NOTDIR => return error.NotDir,
	3486	.ILSEQ => return error.BadPathName,
	3487	else => \|err\| return unexpectedErrno(err),
	3488	}
	3489	}
	3490	}
	3491
	3492	fn dirSetOwner(
	3493	userdata: ?*anyopaque,
	3494	dir: Dir,
	3495	owner: ?File.Uid,
	3496	group: ?File.Gid,
	3497	) Dir.SetOwnerError!void {
	3498	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3499	var cancel_region: CancelRegion = .init();
	3500	defer cancel_region.deinit();
	3501	try ev.fchownat(
	3502	&cancel_region,
	3503	dir.handle,
	3504	"",
	3505	owner orelse std.math.maxInt(linux.uid_t),
	3506	group orelse std.math.maxInt(linux.gid_t),
	3507	linux.AT.EMPTY_PATH,
	3508	);
	3509	}
	3510
	3511	fn dirSetFileOwner(
	3512	userdata: ?*anyopaque,
	3513	dir: Dir,
	3514	sub_path: []const u8,
	3515	owner: ?File.Uid,
	3516	group: ?File.Gid,
	3517	options: Dir.SetFileOwnerOptions,
	3518	) Dir.SetFileOwnerError!void {
	3519	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3520	var path_buffer: [PATH_MAX]u8 = undefined;
	3521	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3522	var cancel_region: CancelRegion = .init();
	3523	defer cancel_region.deinit();
	3524	try ev.fchownat(
	3525	&cancel_region,
	3526	dir.handle,
	3527	sub_path_posix,
	3528	owner orelse std.math.maxInt(linux.uid_t),
	3529	group orelse std.math.maxInt(linux.gid_t),
	3530	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3531	);
	3532	}
	3533
	3534	fn dirSetPermissions(
	3535	userdata: ?*anyopaque,
	3536	dir: Dir,
	3537	permissions: Dir.Permissions,
	3538	) Dir.SetPermissionsError!void {
	3539	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3540	var cancel_region: CancelRegion = .init();
	3541	defer cancel_region.deinit();
	3542	ev.fchmodat(
	3543	&cancel_region,
	3544	dir.handle,
	3545	"",
	3546	permissions.toMode(),
	3547	linux.AT.EMPTY_PATH,
	3548	) catch \|err\| switch (err) {
	3549	error.NameTooLong => return errnoBug(.NAMETOOLONG),
	3550	error.BadPathName => return errnoBug(.ILSEQ),
	3551	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
	3552	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
	3553	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
	3554	else => \|e\| return e,
	3555	};
	3556	}
	3557
	3558	fn dirSetFilePermissions(
	3559	userdata: ?*anyopaque,
	3560	dir: Dir,
	3561	sub_path: []const u8,
	3562	permissions: Dir.Permissions,
	3563	options: Dir.SetFilePermissionsOptions,
	3564	) Dir.SetFilePermissionsError!void {
	3565	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3566	var path_buffer: [PATH_MAX]u8 = undefined;
	3567	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3568	var cancel_region: CancelRegion = .init();
	3569	defer cancel_region.deinit();
	3570	try ev.fchmodat(
	3571	&cancel_region,
	3572	dir.handle,
	3573	sub_path_posix,
	3574	permissions.toMode(),
	3575	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3576	);
	3577	}
	3578
	3579	fn dirSetTimestamps(
	3580	userdata: ?*anyopaque,
	3581	dir: Dir,
	3582	sub_path: []const u8,
	3583	options: Dir.SetTimestampsOptions,
	3584	) Dir.SetTimestampsError!void {
	3585	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3586	var path_buffer: [PATH_MAX]u8 = undefined;
	3587	const sub_path_posix = try pathToPosix(sub_path, &path_buffer);
	3588	var cancel_region: CancelRegion = .init();
	3589	defer cancel_region.deinit();
	3590	try ev.utimensat(
	3591	&cancel_region,
	3592	dir.handle,
	3593	sub_path_posix,
	3594	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
	3595	setTimestampToPosix(options.access_timestamp),
	3596	setTimestampToPosix(options.modify_timestamp),
	3597	} else null,
	3598	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3599	);
	3600	}
	3601
	3602	fn dirHardLink(
	3603	userdata: ?*anyopaque,
	3604	old_dir: Dir,
	3605	old_sub_path: []const u8,
	3606	new_dir: Dir,
	3607	new_sub_path: []const u8,
	3608	options: Dir.HardLinkOptions,
	3609	) Dir.HardLinkError!void {
	3610	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3611
	3612	var old_path_buffer: [PATH_MAX]u8 = undefined;
	3613	var new_path_buffer: [PATH_MAX]u8 = undefined;
	3614
	3615	const old_sub_path_posix = try pathToPosix(old_sub_path, &old_path_buffer);
	3616	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	3617
	3618	var cancel_region: CancelRegion = .init();
	3619	defer cancel_region.deinit();
	3620	return ev.linkat(
	3621	&cancel_region,
	3622	old_dir.handle,
	3623	old_sub_path_posix,
	3624	new_dir.handle,
	3625	new_sub_path_posix,
	3626	if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW,
	3627	);
	3628	}
	3629
	3630	fn fileStat(userdata: ?*anyopaque, file: File) File.StatError!File.Stat {
	3631	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3632	var cancel_region: CancelRegion = .init();
	3633	defer cancel_region.deinit();
	3634	return ev.stat(&cancel_region, file.handle);
	3635	}
	3636
	3637	fn fileLength(userdata: ?*anyopaque, file: File) File.LengthError!u64 {
	3638	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3639	var cancel_region: CancelRegion = .init();
	3640	defer cancel_region.deinit();
	3641	while (true) {
	3642	var statx_buf = std.mem.zeroes(linux.Statx);
	3643	const thread = try cancel_region.awaitIoUring();
	3644	thread.enqueue().* = .{
	3645	.opcode = .STATX,
	3646	.flags = 0,
	3647	.ioprio = 0,
	3648	.fd = file.handle,
	3649	.off = @intFromPtr(&statx_buf),
	3650	.addr = @intFromPtr(""),
	3651	.len = @bitCast(linux.STATX{ .SIZE = true }),
	3652	.rw_flags = linux.AT.EMPTY_PATH,
	3653	.user_data = @intFromPtr(cancel_region.fiber),
	3654	.buf_index = 0,
	3655	.personality = 0,
	3656	.splice_fd_in = 0,
	3657	.addr3 = 0,
	3658	.resv = 0,
	3659	};
	3660	ev.yield(null, .nothing);
	3661	switch (cancel_region.errno()) {
	3662	.SUCCESS => {
	3663	if (!statx_buf.mask.SIZE) return error.Unexpected;
	3664	return statx_buf.size;
	3665	},
	3666	.INTR, .CANCELED => continue,
	3667	.ACCES => \|err\| return errnoBug(err),
	3668	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	3669	.FAULT => \|err\| return errnoBug(err),
	3670	.INVAL => \|err\| return errnoBug(err),
	3671	.LOOP => \|err\| return errnoBug(err),
	3672	.NAMETOOLONG => \|err\| return errnoBug(err),
	3673	.NOENT => \|err\| return errnoBug(err),
	3674	.NOMEM => return error.SystemResources,
	3675	.NOTDIR => \|err\| return errnoBug(err),
	3676	else => \|err\| return unexpectedErrno(err),
	3677	}
	3678	}
	3679	}
	3680
	3681	fn fileClose(userdata: ?*anyopaque, files: []const File) void {
	3682	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3683	for (files) \|file\| ev.close(file.handle);
	3684	}
	3685
	3686	fn fileWritePositional(
	3687	userdata: ?*anyopaque,
	3688	file: File,
	3689	header: []const u8,
	3690	data: []const []const u8,
	3691	splat: usize,
	3692	offset: u64,
	3693	) File.WritePositionalError!usize {
	3694	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3695
	3696	var iovecs: [max_iovecs_len]iovec_const = undefined;
	3697	var iovlen: iovlen_t = 0;
	3698	addBuf(&iovecs, &iovlen, header);
	3699	for (data[0 .. data.len - 1]) \|bytes\| addBuf(&iovecs, &iovlen, bytes);
	3700	const pattern = data[data.len - 1];
	3701	if (iovecs.len - iovlen != 0) switch (splat) {
	3702	0 => {},
	3703	1 => addBuf(&iovecs, &iovlen, pattern),
	3704	else => switch (pattern.len) {
	3705	0 => {},
	3706	1 => {
	3707	var backup_buffer: [splat_buffer_size]u8 = undefined;
	3708	const splat_buffer = &backup_buffer;
	3709	const memset_len = @min(splat_buffer.len, splat);
	3710	const buf = splat_buffer[0..memset_len];
	3711	@memset(buf, pattern[0]);
	3712	addBuf(&iovecs, &iovlen, buf);
	3713	var remaining_splat = splat - buf.len;
	3714	while (remaining_splat > splat_buffer.len and iovecs.len - iovlen != 0) {
	3715	assert(buf.len == splat_buffer.len);
	3716	addBuf(&iovecs, &iovlen, splat_buffer);
	3717	remaining_splat -= splat_buffer.len;
	3718	}
	3719	addBuf(&iovecs, &iovlen, splat_buffer[0..@min(remaining_splat, splat_buffer.len)]);
	3720	},
	3721	else => for (0..@min(splat, iovecs.len - iovlen)) \|_\| {
	3722	addBuf(&iovecs, &iovlen, pattern);
	3723	},
	3724	},
	3725	};
	3726
	3727	var cancel_region: CancelRegion = .init();
	3728	defer cancel_region.deinit();
	3729	return ev.pwritev(&cancel_region, file.handle, iovecs[0..iovlen], offset);
	3730	}
	3731
	3732	/// This is either usize or u32. Since, either is fine, let's use the same
	3733	/// `addBuf` function for both writing to a file and sending network messages.
	3734	const iovlen_t = @FieldType(linux.msghdr_const, "iovlen");
	3735
	3736	fn addBuf(v: []iovec_const, i: *iovlen_t, bytes: []const u8) void {
	3737	// OS checks ptr addr before length so zero length vectors must be omitted.
	3738	if (bytes.len == 0) return;
	3739	if (v.len - i.* == 0) return;
	3740	v[i.*] = .{ .base = bytes.ptr, .len = bytes.len };
	3741	i.* += 1;
	3742	}
	3743
	3744	fn fileWriteFileStreaming(
	3745	userdata: ?*anyopaque,
	3746	file: File,
	3747	header: []const u8,
	3748	file_reader: *File.Reader,
	3749	limit: Io.Limit,
	3750	) File.Writer.WriteFileError!usize {
	3751	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3752	_ = ev;
	3753	_ = file;
	3754	_ = header;
	3755	_ = file_reader;
	3756	_ = limit;
	3757	return error.Unimplemented;
	3758	}
	3759
	3760	fn fileWriteFilePositional(
	3761	userdata: ?*anyopaque,
	3762	file: File,
	3763	header: []const u8,
	3764	file_reader: *File.Reader,
	3765	limit: Io.Limit,
	3766	offset: u64,
	3767	) File.WriteFilePositionalError!usize {
	3768	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3769	_ = ev;
	3770	_ = file;
	3771	_ = header;
	3772	_ = file_reader;
	3773	_ = limit;
	3774	_ = offset;
	3775	return error.Unimplemented;
	3776	}
	3777
	3778	fn fileReadPositional(
	3779	userdata: ?*anyopaque,
	3780	file: File,
	3781	data: []const []u8,
	3782	offset: u64,
	3783	) File.ReadPositionalError!usize {
	3784	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3785
	3786	var iovecs_buffer: [max_iovecs_len]iovec = undefined;
	3787	var i: usize = 0;
	3788	for (data) \|buf\| {
	3789	if (iovecs_buffer.len - i == 0) break;
	3790	if (buf.len != 0) {
	3791	iovecs_buffer[i] = .{ .base = buf.ptr, .len = buf.len };
	3792	i += 1;
	3793	}
	3794	}
	3795	if (i == 0) return 0;
	3796	const dest = iovecs_buffer[0..i];
	3797	assert(dest[0].len > 0);
	3798
	3799	var cancel_region: CancelRegion = .init();
	3800	defer cancel_region.deinit();
	3801	return ev.preadv(&cancel_region, file.handle, dest, offset) catch \|err\| switch (err) {
	3802	error.SocketUnconnected => errnoBug(.NOTCONN), // not a socket
	3803	error.ConnectionResetByPeer => errnoBug(.CONNRESET), // not a socket
	3804	else => \|e\| e,
	3805	};
	3806	}
	3807
	3808	fn fileSeekBy(userdata: ?*anyopaque, file: File, offset: i64) File.SeekError!void {
	3809	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3810	var cancel_region: CancelRegion = .init();
	3811	defer cancel_region.deinit();
	3812	try ev.lseek(&cancel_region, file.handle, @bitCast(offset), linux.SEEK.CUR);
	3813	}
	3814
	3815	fn fileSeekTo(userdata: ?*anyopaque, file: File, offset: u64) File.SeekError!void {
	3816	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3817	var cancel_region: CancelRegion = .init();
	3818	defer cancel_region.deinit();
	3819	try ev.lseek(&cancel_region, file.handle, offset, linux.SEEK.SET);
	3820	}
	3821
	3822	fn fileSync(userdata: ?*anyopaque, file: File) File.SyncError!void {
	3823	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3824	var cancel_region: CancelRegion = .init();
	3825	defer cancel_region.deinit();
	3826	while (true) {
	3827	const thread = try cancel_region.awaitIoUring();
	3828	thread.enqueue().* = .{
	3829	.opcode = .FSYNC,
	3830	.flags = 0,
	3831	.ioprio = 0,
	3832	.fd = file.handle,
	3833	.off = 0,
	3834	.addr = 0,
	3835	.len = 0,
	3836	.rw_flags = 0,
	3837	.user_data = @intFromPtr(cancel_region.fiber),
	3838	.buf_index = 0,
	3839	.personality = 0,
	3840	.splice_fd_in = 0,
	3841	.addr3 = 0,
	3842	.resv = 0,
	3843	};
	3844	ev.yield(null, .nothing);
	3845	switch (cancel_region.errno()) {
	3846	.SUCCESS => return,
	3847	.INTR, .CANCELED => continue,
	3848	.BADF => \|err\| return errnoBug(err),
	3849	.INVAL => \|err\| return errnoBug(err),
	3850	.ROFS => \|err\| return errnoBug(err),
	3851	.IO => return error.InputOutput,
	3852	.NOSPC => return error.NoSpaceLeft,
	3853	.DQUOT => return error.DiskQuota,
	3854	else => \|err\| return unexpectedErrno(err),
	3855	}
	3856	}
	3857	}
	3858
	3859	fn fileIsTty(userdata: ?*anyopaque, file: File) Io.Cancelable!bool {
	3860	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3861	_ = ev;
	3862	var cancel_region: CancelRegion = .init();
	3863	defer cancel_region.deinit();
	3864	while (true) {
	3865	try cancel_region.await(.nothing);
	3866	var wsz: winsize = undefined;
	3867	const fd: usize = @bitCast(@as(isize, file.handle));
	3868	const rc = linux.syscall3(.ioctl, fd, linux.T.IOCGWINSZ, @intFromPtr(&wsz));
	3869	switch (linux.errno(rc)) {
	3870	.SUCCESS => return true,
	3871	.INTR => continue,
	3872	else => return false,
	3873	}
	3874	}
	3875	}
	3876
	3877	fn fileEnableAnsiEscapeCodes(userdata: ?*anyopaque, file: File) File.EnableAnsiEscapeCodesError!void {
	3878	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3879	if (!try fileIsTty(ev, file)) return error.NotTerminalDevice;
	3880	}
	3881
	3882	fn fileSetLength(userdata: ?*anyopaque, file: File, length: u64) File.SetLengthError!void {
	3883	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3884	var cancel_region: CancelRegion = .init();
	3885	defer cancel_region.deinit();
	3886	while (true) {
	3887	const thread = try cancel_region.awaitIoUring();
	3888	thread.enqueue().* = .{
	3889	.opcode = .FTRUNCATE,
	3890	.flags = 0,
	3891	.ioprio = 0,
	3892	.fd = file.handle,
	3893	.off = length,
	3894	.addr = 0,
	3895	.len = 0,
	3896	.rw_flags = 0,
	3897	.user_data = @intFromPtr(cancel_region.fiber),
	3898	.buf_index = 0,
	3899	.personality = 0,
	3900	.splice_fd_in = 0,
	3901	.addr3 = 0,
	3902	.resv = 0,
	3903	};
	3904	ev.yield(null, .nothing);
	3905	switch (cancel_region.errno()) {
	3906	.SUCCESS => return,
	3907	.INTR, .CANCELED => continue,
	3908	.FBIG => return error.FileTooBig,
	3909	.IO => return error.InputOutput,
	3910	.PERM => return error.PermissionDenied,
	3911	.TXTBSY => return error.FileBusy,
	3912	.BADF => \|err\| return errnoBug(err), // Handle not open for writing.
	3913	.INVAL => return error.NonResizable, // This is returned for /dev/null for example.
	3914	else => \|err\| return unexpectedErrno(err),
	3915	}
	3916	}
	3917	}
	3918
	3919	fn fileSetOwner(
	3920	userdata: ?*anyopaque,
	3921	file: File,
	3922	owner: ?File.Uid,
	3923	group: ?File.Gid,
	3924	) File.SetOwnerError!void {
	3925	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3926	var cancel_region: CancelRegion = .init();
	3927	defer cancel_region.deinit();
	3928	try ev.fchownat(
	3929	&cancel_region,
	3930	file.handle,
	3931	"",
	3932	owner orelse std.math.maxInt(linux.uid_t),
	3933	group orelse std.math.maxInt(linux.gid_t),
	3934	linux.AT.EMPTY_PATH,
	3935	);
	3936	}
	3937
	3938	fn fileSetPermissions(
	3939	userdata: ?*anyopaque,
	3940	file: File,
	3941	permissions: File.Permissions,
	3942	) File.SetPermissionsError!void {
	3943	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3944	var cancel_region: CancelRegion = .init();
	3945	defer cancel_region.deinit();
	3946	ev.fchmodat(
	3947	&cancel_region,
	3948	file.handle,
	3949	"",
	3950	permissions.toMode(),
	3951	linux.AT.EMPTY_PATH,
	3952	) catch \|err\| switch (err) {
	3953	error.NameTooLong => return errnoBug(.NAMETOOLONG),
	3954	error.BadPathName => return errnoBug(.ILSEQ),
	3955	error.ProcessFdQuotaExceeded => return errnoBug(.MFILE),
	3956	error.SystemFdQuotaExceeded => return errnoBug(.NFILE),
	3957	error.OperationUnsupported => return errnoBug(.OPNOTSUPP),
	3958	else => \|e\| return e,
	3959	};
	3960	}
	3961
	3962	fn fileSetTimestamps(
	3963	userdata: ?*anyopaque,
	3964	file: File,
	3965	options: File.SetTimestampsOptions,
	3966	) File.SetTimestampsError!void {
	3967	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3968	var cancel_region: CancelRegion = .init();
	3969	defer cancel_region.deinit();
	3970	try ev.utimensat(
	3971	&cancel_region,
	3972	file.handle,
	3973	"",
	3974	if (options.modify_timestamp != .now or options.access_timestamp != .now) &.{
	3975	setTimestampToPosix(options.access_timestamp),
	3976	setTimestampToPosix(options.modify_timestamp),
	3977	} else null,
	3978	linux.AT.EMPTY_PATH,
	3979	);
	3980	}
	3981
	3982	fn fileLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!void {
	3983	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3984	var cancel_region: CancelRegion = .init();
	3985	defer cancel_region.deinit();
	3986	ev.flock(&cancel_region, file.handle, lock, .blocking) catch \|err\| switch (err) {
	3987	error.WouldBlock => unreachable, // blocking
	3988	else => \|e\| return e,
	3989	};
	3990	}
	3991
	3992	fn fileTryLock(userdata: ?*anyopaque, file: File, lock: File.Lock) File.LockError!bool {
	3993	const ev: *Evented = @ptrCast(@alignCast(userdata));
	3994	var cancel_region: CancelRegion = .init();
	3995	defer cancel_region.deinit();
	3996	ev.flock(&cancel_region, file.handle, lock, switch (lock) {
	3997	.none => .blocking,
	3998	.shared, .exclusive => .nonblocking,
	3999	}) catch \|err\| switch (err) {
	4000	error.WouldBlock => return false,
	4001	else => \|e\| return e,
	4002	};
	4003	return true;
	4004	}
	4005
	4006	fn fileUnlock(userdata: ?*anyopaque, file: File) void {
	4007	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4008	var cancel_region: CancelRegion = .initBlocked();
	4009	defer cancel_region.deinit();
	4010	ev.flock(&cancel_region, file.handle, .none, .blocking) catch \|err\| switch (err) {
	4011	error.Canceled => unreachable, // blocked
	4012	error.WouldBlock => unreachable, // blocking
	4013	error.SystemResources => return recoverableOsBugDetected(), // Resource deallocation.
	4014	error.FileLocksUnsupported => return recoverableOsBugDetected(), // We already got the lock.
	4015	error.Unexpected => return recoverableOsBugDetected(), // Resource deallocation must succeed.
	4016	};
	4017	}
	4018
	4019	fn fileDowngradeLock(userdata: ?*anyopaque, file: File) File.DowngradeLockError!void {
	4020	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4021	var cancel_region: CancelRegion = .init();
	4022	defer cancel_region.deinit();
	4023	ev.flock(&cancel_region, file.handle, .shared, .nonblocking) catch \|err\| switch (err) {
	4024	error.WouldBlock => return errnoBug(.AGAIN), // File was not locked in exclusive mode.
	4025	error.SystemResources => return errnoBug(.NOLCK), // Lock already obtained.
	4026	error.FileLocksUnsupported => return errnoBug(.OPNOTSUPP), // Lock already obtained.
	4027	else => \|e\| return e,
	4028	};
	4029	}
	4030
	4031	fn fileRealPath(userdata: ?*anyopaque, file: File, out_buffer: []u8) File.RealPathError!usize {
	4032	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4033	var cancel_region: CancelRegion = .init();
	4034	defer cancel_region.deinit();
	4035	return ev.realPath(&cancel_region, file.handle, out_buffer);
	4036	}
	4037
	4038	fn fileHardLink(
	4039	userdata: ?*anyopaque,
	4040	file: File,
	4041	new_dir: Dir,
	4042	new_sub_path: []const u8,
	4043	options: File.HardLinkOptions,
	4044	) File.HardLinkError!void {
	4045	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4046
	4047	var new_path_buffer: [PATH_MAX]u8 = undefined;
	4048	const new_sub_path_posix = try pathToPosix(new_sub_path, &new_path_buffer);
	4049
	4050	var cancel_region: CancelRegion = .init();
	4051	defer cancel_region.deinit();
	4052	return ev.linkat(
	4053	&cancel_region,
	4054	file.handle,
	4055	"",
	4056	new_dir.handle,
	4057	new_sub_path_posix,
	4058	linux.AT.EMPTY_PATH \| @as(u32, if (options.follow_symlinks) 0 else linux.AT.SYMLINK_NOFOLLOW),
	4059	);
	4060	}
	4061
	4062	fn fileMemoryMapCreate(
	4063	userdata: ?*anyopaque,
	4064	file: File,
	4065	options: File.MemoryMap.CreateOptions,
	4066	) File.MemoryMap.CreateError!File.MemoryMap {
	4067	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4068	_ = ev;
	4069	const prot: linux.PROT = .{
	4070	.READ = options.protection.read,
	4071	.WRITE = options.protection.write,
	4072	.EXEC = options.protection.execute,
	4073	};
	4074	const flags: linux.MAP = .{
	4075	.TYPE = .SHARED_VALIDATE,
	4076	.POPULATE = options.populate,
	4077	};
	4078
	4079	const page_align = std.heap.page_size_min;
	4080
	4081	var cancel_region: CancelRegion = .init();
	4082	defer cancel_region.deinit();
	4083	const contents = while (true) {
	4084	try cancel_region.await(.nothing);
	4085	const casted_offset = std.math.cast(i64, options.offset) orelse return error.Unseekable;
	4086	const rc = linux.mmap(null, options.len, prot, flags, file.handle, casted_offset);
	4087	switch (linux.errno(rc)) {
	4088	.SUCCESS => break @as([*]align(page_align) u8, @ptrFromInt(rc))[0..options.len],
	4089	.INTR => continue,
	4090	.ACCES => return error.AccessDenied,
	4091	.AGAIN => return error.LockedMemoryLimitExceeded,
	4092	.MFILE => return error.ProcessFdQuotaExceeded,
	4093	.NFILE => return error.SystemFdQuotaExceeded,
	4094	.NOMEM => return error.OutOfMemory,
	4095	.PERM => return error.PermissionDenied,
	4096	.OVERFLOW => return error.Unseekable,
	4097	.BADF => \|err\| return errnoBug(err), // Always a race condition.
	4098	.INVAL => \|err\| return errnoBug(err), // Invalid parameters to mmap()
	4099	.OPNOTSUPP => \|err\| return errnoBug(err), // Bad flags with MAP.SHARED_VALIDATE on Linux.
	4100	else => \|err\| return unexpectedErrno(err),
	4101	}
	4102	};
	4103	return .{
	4104	.file = file,
	4105	.offset = options.offset,
	4106	.memory = contents,
	4107	.section = {},
	4108	};
	4109	}
	4110
	4111	fn fileMemoryMapDestroy(userdata: ?anyopaque, mm: File.MemoryMap) void {
	4112	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4113	_ = ev;
	4114	const memory = mm.memory;
	4115	if (memory.len == 0) return;
	4116	switch (linux.errno(linux.munmap(memory.ptr, memory.len))) {
	4117	.SUCCESS => {},
	4118	else => \|err\| if (builtin.mode == .Debug)
	4119	std.log.err("failed to unmap {d} bytes at {*}: {t}", .{ memory.len, memory.ptr, err }),
	4120	}
	4121	mm.* = undefined;
	4122	}
	4123
	4124	fn processExecutableOpen(
	4125	userdata: ?*anyopaque,
	4126	flags: File.OpenFlags,
	4127	) process.OpenExecutableError!File {
	4128	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4129	return dirOpenFile(ev, .{ .handle = linux.AT.FDCWD }, "/proc/self/exe", flags);
	4130	}
	4131
	4132	fn processExecutablePath(userdata: ?*anyopaque, out_buffer: []u8) process.ExecutablePathError!usize {
	4133	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4134	return dirReadLink(ev, .cwd(), "/proc/self/exe", out_buffer) catch \|err\| switch (err) {
	4135	error.UnsupportedReparsePointType => unreachable, // Windows-only
	4136	error.NetworkNotFound => unreachable, // Windows-only
	4137	error.FileBusy => unreachable, // Windows-only
	4138	else => \|e\| return e,
	4139	};
	4140	}
	4141
	4142	fn lockStderr(userdata: ?*anyopaque, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4143	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4144	const ev_io = ev.io();
	4145	ev.stderr_mutex.lockUncancelable(ev_io);
	4146	errdefer ev.stderr_mutex.unlock(ev_io);
	4147	return ev.initLockedStderr(terminal_mode);
	4148	}
	4149
	4150	fn tryLockStderr(
	4151	userdata: ?*anyopaque,
	4152	terminal_mode: ?Io.Terminal.Mode,
	4153	) Io.Cancelable!?Io.LockedStderr {
	4154	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4155	const ev_io = ev.io();
	4156	if (!ev.stderr_mutex.tryLock()) return null;
	4157	errdefer ev.stderr_mutex.unlock(ev_io);
	4158	return try ev.initLockedStderr(terminal_mode);
	4159	}
	4160
	4161	fn initLockedStderr(ev: *Evented, terminal_mode: ?Io.Terminal.Mode) Io.Cancelable!Io.LockedStderr {
	4162	if (!ev.stderr_writer_initialized) {
	4163	const ev_io = ev.io();
	4164	try ev.scanEnviron();
	4165	const NO_COLOR = ev.environ.exist.NO_COLOR;
	4166	const CLICOLOR_FORCE = ev.environ.exist.CLICOLOR_FORCE;
	4167	ev.stderr_mode = terminal_mode orelse
	4168	try .detect(ev_io, ev.stderr_writer.file, NO_COLOR, CLICOLOR_FORCE);
	4169	ev.stderr_writer_initialized = true;
	4170	}
	4171	return .{
	4172	.file_writer = &ev.stderr_writer,
	4173	.terminal_mode = terminal_mode orelse ev.stderr_mode,
	4174	};
	4175	}
	4176
	4177	fn unlockStderr(userdata: ?*anyopaque) void {
	4178	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4179	ev.stderr_writer.interface.flush() catch \|err\| switch (err) {
	4180	error.WriteFailed => switch (ev.stderr_writer.err.?) {
	4181	error.Canceled => recancel(ev),
	4182	else => {},
	4183	},
	4184	};
	4185	ev.stderr_writer.interface.end = 0;
	4186	ev.stderr_writer.interface.buffer = &.{};
	4187	ev.stderr_mutex.unlock(ev.io());
	4188	}
	4189
	4190	fn processCurrentPath(userdata: ?*anyopaque, buffer: []u8) process.CurrentPathError!usize {
	4191	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4192	_ = ev;
	4193	var cancel_region: CancelRegion = .init();
	4194	defer cancel_region.deinit();
	4195	while (true) {
	4196	try cancel_region.await(.nothing);
	4197	switch (linux.errno(linux.getcwd(buffer.ptr, buffer.len))) {
	4198	.SUCCESS => return std.mem.findScalar(u8, buffer, 0).?,
	4199	.INTR => continue,
	4200	.NOENT => return error.CurrentDirUnlinked,
	4201	.RANGE => return error.NameTooLong,
	4202	.FAULT => \|err\| return errnoBug(err),
	4203	.INVAL => \|err\| return errnoBug(err),
	4204	else => \|err\| return unexpectedErrno(err),
	4205	}
	4206	}
	4207	}
	4208
	4209	fn processSetCurrentDir(userdata: ?*anyopaque, dir: Dir) process.SetCurrentDirError!void {
	4210	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4211	_ = ev;
	4212	if (dir.handle == linux.AT.FDCWD) return;
	4213	var cancel_region: CancelRegion = .init();
	4214	defer cancel_region.deinit();
	4215	while (true) {
	4216	try cancel_region.await(.nothing);
	4217	switch (linux.errno(linux.fchdir(dir.handle))) {
	4218	.SUCCESS => return,
	4219	.INTR => continue,
	4220	.ACCES => return error.AccessDenied,
	4221	.NOTDIR => return error.NotDir,
	4222	.IO => return error.FileSystem,
	4223	.BADF => \|err\| return errnoBug(err),
	4224	else => \|err\| return unexpectedErrno(err),
	4225	}
	4226	}
	4227	}
	4228
	4229	fn processSetCurrentPath(userdata: ?*anyopaque, dir_path: []const u8) ChdirError!void {
	4230	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4231	_ = ev;
	4232	var path_buffer: [PATH_MAX]u8 = undefined;
	4233	const dir_path_posix = try pathToPosix(dir_path, &path_buffer);
	4234	var cancel_region: CancelRegion = .init();
	4235	defer cancel_region.deinit();
	4236	while (true) {
	4237	try cancel_region.await(.nothing);
	4238	switch (linux.errno(linux.chdir(dir_path_posix))) {
	4239	.SUCCESS => return,
	4240	.INTR => continue,
	4241	.ACCES => return error.AccessDenied,
	4242	.IO => return error.FileSystem,
	4243	.LOOP => return error.SymLinkLoop,
	4244	.NAMETOOLONG => return error.NameTooLong,
	4245	.NOENT => return error.FileNotFound,
	4246	.NOMEM => return error.SystemResources,
	4247	.NOTDIR => return error.NotDir,
	4248	.ILSEQ => return error.BadPathName,
	4249	.FAULT => \|err\| return errnoBug(err),
	4250	else => \|err\| return unexpectedErrno(err),
	4251	}
	4252	}
	4253	}
	4254
	4255	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
	4256	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4257
	4258	try ev.scanEnviron(); // for PATH
	4259	const PATH = ev.environ.string.PATH orelse default_PATH;
	4260
	4261	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4262	defer arena_allocator.deinit();
	4263	const arena = arena_allocator.allocator();
	4264
	4265	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4266	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4267
	4268	const env_block = env_block: {
	4269	const prog_fd: i32 = -1;
	4270	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4271	.zig_progress_fd = prog_fd,
	4272	});
	4273	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4274	.zig_progress_fd = prog_fd,
	4275	});
	4276	};
	4277
	4278	return execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
	4279	}
	4280
	4281	fn processReplacePath(
	4282	userdata: ?*anyopaque,
	4283	dir: Dir,
	4284	options: process.ReplaceOptions,
	4285	) process.ReplaceError {
	4286	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4287	_ = ev;
	4288	_ = dir;
	4289	_ = options;
	4290	@panic("TODO processReplacePath");
	4291	}
	4292
	4293	fn processSpawn(userdata: ?*anyopaque, options: process.SpawnOptions) process.SpawnError!process.Child {
	4294	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4295	const spawned = try ev.spawn(options);
	4296	defer ev.close(spawned.err_fd);
	4297
	4298	// Wait for the child to report any errors in or before `execvpe`.
	4299	var child_err: ForkBailError = undefined;
	4300	var cancel_region: CancelRegion = .initBlocked();
	4301	defer cancel_region.deinit();
	4302	ev.readAll(&cancel_region, spawned.err_fd, @ptrCast(&child_err)) catch \|read_err\| {
	4303	switch (read_err) {
	4304	error.Canceled => unreachable, // blocked
	4305	error.EndOfStream => {
	4306	// Write end closed by CLOEXEC at the time of the `execvpe` call,
	4307	// indicating success.
	4308	},
	4309	else => {
	4310	// Problem reading the error from the error reporting pipe. We
	4311	// don't know if the child is alive or dead. Better to assume it is
	4312	// alive so the resource does not risk being leaked.
	4313	},
	4314	}
	4315	return .{
	4316	.id = spawned.pid,
	4317	.thread_handle = {},
	4318	.stdin = spawned.stdin,
	4319	.stdout = spawned.stdout,
	4320	.stderr = spawned.stderr,
	4321	.request_resource_usage_statistics = options.request_resource_usage_statistics,
	4322	};
	4323	};
	4324	return child_err;
	4325	}
	4326
	4327	fn processSpawnPath(
	4328	userdata: ?*anyopaque,
	4329	dir: Dir,
	4330	options: process.SpawnOptions,
	4331	) process.SpawnError!process.Child {
	4332	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4333	_ = ev;
	4334	_ = dir;
	4335	_ = options;
	4336	@panic("TODO processSpawnPath");
	4337	}
	4338
	4339	const Spawned = struct {
	4340	pid: pid_t,
	4341	err_fd: fd_t,
	4342	stdin: ?File,
	4343	stdout: ?File,
	4344	stderr: ?File,
	4345	};
	4346	fn spawn(ev: *Evented, options: process.SpawnOptions) process.SpawnError!Spawned {
	4347	// The child process does need to access (one end of) these pipes. However,
	4348	// we must initially set CLOEXEC to avoid a race condition. If another thread
	4349	// is racing to spawn a different child process, we don't want it to inherit
	4350	// these FDs in any scenario; that would mean that, for instance, calls to
	4351	// `poll` from the parent would not report the child's stdout as closing when
	4352	// expected, since the other child may retain a reference to the write end of
	4353	// the pipe. So, we create the pipes with CLOEXEC initially. After fork, we
	4354	// need to do something in the new child to make sure we preserve the reference
	4355	// we want. We could use `fcntl` to remove CLOEXEC from the FD, but as it
	4356	// turns out, we `dup2` everything anyway, so there's no need!
	4357	const pipe_flags: linux.O = .{ .CLOEXEC = true };
	4358
	4359	const stdin_pipe = if (options.stdin == .pipe) try pipe2(pipe_flags) else undefined;
	4360	errdefer if (options.stdin == .pipe) {
	4361	ev.destroyPipe(stdin_pipe);
	4362	};
	4363
	4364	const stdout_pipe = if (options.stdout == .pipe) try pipe2(pipe_flags) else undefined;
	4365	errdefer if (options.stdout == .pipe) {
	4366	ev.destroyPipe(stdout_pipe);
	4367	};
	4368
	4369	const stderr_pipe = if (options.stderr == .pipe) try pipe2(pipe_flags) else undefined;
	4370	errdefer if (options.stderr == .pipe) {
	4371	ev.destroyPipe(stderr_pipe);
	4372	};
	4373
	4374	const any_ignore =
	4375	options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore;
	4376	const dev_null_fd = if (any_ignore) dev_null_fd: {
	4377	var cancel_region: CancelRegion = .init();
	4378	defer cancel_region.deinit();
	4379	break :dev_null_fd try ev.null_fd.open(ev, &cancel_region, "/dev/null", .{
	4380	.ACCMODE = .RDWR,
	4381	});
	4382	} else undefined;
	4383
	4384	const prog_pipe: [2]fd_t = if (options.progress_node.index != .none) pipe: {
	4385	// We use CLOEXEC for the same reason as in `pipe_flags`.
	4386	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
	4387	_ = linux.fcntl(pipe[0], linux.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
	4388	break :pipe pipe;
	4389	} else .{ -1, -1 };
	4390	errdefer ev.destroyPipe(prog_pipe);
	4391
	4392	var arena_allocator = std.heap.ArenaAllocator.init(ev.allocator());
	4393	defer arena_allocator.deinit();
	4394	const arena = arena_allocator.allocator();
	4395
	4396	// The POSIX standard does not allow malloc() between fork() and execve(),
	4397	// and this allocator may be a libc allocator.
	4398	// I have personally observed the child process deadlocking when it tries
	4399	// to call malloc() due to a heap allocation between fork() and execve(),
	4400	// in musl v1.1.24.
	4401	// Additionally, we want to reduce the number of possible ways things
	4402	// can fail between fork() and execve().
	4403	// Therefore, we do all the allocation for the execve() before the fork().
	4404	// This means we must do the null-termination of argv and env vars here.
	4405	const argv_buf = try arena.allocSentinel(?[*:0]const u8, options.argv.len, null);
	4406	for (options.argv, 0..) \|arg, i\| argv_buf[i] = (try arena.dupeZ(u8, arg)).ptr;
	4407
	4408	const prog_fileno = 3;
	4409	comptime assert(@max(linux.STDIN_FILENO, linux.STDOUT_FILENO, linux.STDERR_FILENO) + 1 == prog_fileno);
	4410
	4411	const env_block = env_block: {
	4412	const prog_fd: i32 = if (prog_pipe[1] == -1) -1 else prog_fileno;
	4413	if (options.environ_map) \|environ_map\| break :env_block try environ_map.createPosixBlock(arena, .{
	4414	.zig_progress_fd = prog_fd,
	4415	});
	4416	break :env_block try ev.environ.process_environ.createPosixBlock(arena, .{
	4417	.zig_progress_fd = prog_fd,
	4418	});
	4419	};
	4420
	4421	// This pipe communicates to the parent errors in the child between `fork` and `execvpe`.
	4422	// It is closed by the child (via CLOEXEC) without writing if `execvpe` succeeds.
	4423	const err_pipe: [2]fd_t = try pipe2(.{ .CLOEXEC = true });
	4424	errdefer ev.destroyPipe(err_pipe);
	4425
	4426	try ev.scanEnviron(); // for PATH
	4427	const PATH = ev.environ.string.PATH orelse default_PATH;
	4428
	4429	const pid_result: pid_t = fork: {
	4430	const rc = linux.fork();
	4431	switch (linux.errno(rc)) {
	4432	.SUCCESS => break :fork @intCast(rc),
	4433	.AGAIN => return error.SystemResources,
	4434	.NOMEM => return error.SystemResources,
	4435	.NOSYS => return error.OperationUnsupported,
	4436	else => \|err\| return unexpectedErrno(err),
	4437	}
	4438	};
	4439
	4440	if (pid_result == 0) {
	4441	defer comptime unreachable; // We are the child.
	4442	_ = swapCancelProtection(ev, .blocked);
	4443	const ep1 = err_pipe[1];
	4444
	4445	ev.setUpChildIo(options.stdin, stdin_pipe[0], linux.STDIN_FILENO, dev_null_fd) catch \|err\|
	4446	ev.forkBail(ep1, err);
	4447	ev.setUpChildIo(options.stdout, stdout_pipe[1], linux.STDOUT_FILENO, dev_null_fd) catch \|err\|
	4448	ev.forkBail(ep1, err);
	4449	ev.setUpChildIo(options.stderr, stderr_pipe[1], linux.STDERR_FILENO, dev_null_fd) catch \|err\|
	4450	ev.forkBail(ep1, err);
	4451
	4452	switch (options.cwd) {
	4453	.inherit => {},
	4454	.dir => \|cwd\| processSetCurrentDir(ev, cwd) catch \|err\| ev.forkBail(ep1, err),
	4455	.path => \|cwd\| processSetCurrentPath(ev, cwd) catch \|err\| ev.forkBail(ep1, err),
	4456	}
	4457
	4458	// Must happen after fchdir above, the cwd file descriptor might be
	4459	// equal to prog_fileno and be clobbered by this dup2 call.
	4460	if (prog_pipe[1] != -1) dup2(prog_pipe[1], prog_fileno) catch \|err\| ev.forkBail(ep1, err);
	4461
	4462	if (options.gid) \|gid\| {
	4463	switch (linux.errno(linux.setregid(gid, gid))) {
	4464	.SUCCESS => {},
	4465	.AGAIN => ev.forkBail(ep1, error.ResourceLimitReached),
	4466	.INVAL => ev.forkBail(ep1, error.InvalidUserId),
	4467	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4468	else => ev.forkBail(ep1, error.Unexpected),
	4469	}
	4470	}
	4471
	4472	if (options.uid) \|uid\| {
	4473	switch (linux.errno(linux.setreuid(uid, uid))) {
	4474	.SUCCESS => {},
	4475	.AGAIN => ev.forkBail(ep1, error.ResourceLimitReached),
	4476	.INVAL => ev.forkBail(ep1, error.InvalidUserId),
	4477	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4478	else => ev.forkBail(ep1, error.Unexpected),
	4479	}
	4480	}
	4481
	4482	if (options.pgid) \|pid\| {
	4483	switch (linux.errno(linux.setpgid(0, pid))) {
	4484	.SUCCESS => {},
	4485	.ACCES => ev.forkBail(ep1, error.ProcessAlreadyExec),
	4486	.INVAL => ev.forkBail(ep1, error.InvalidProcessGroupId),
	4487	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4488	else => ev.forkBail(ep1, error.Unexpected),
	4489	}
	4490	}
	4491
	4492	if (options.start_suspended) {
	4493	switch (linux.errno(linux.kill(linux.getpid(), .STOP))) {
	4494	.SUCCESS => {},
	4495	.PERM => ev.forkBail(ep1, error.PermissionDenied),
	4496	else => ev.forkBail(ep1, error.Unexpected),
	4497	}
	4498	}
	4499
	4500	const err = execv(options.expand_arg0, argv_buf.ptr[0].?, argv_buf.ptr, env_block, PATH);
	4501	ev.forkBail(ep1, err);
	4502	}
	4503
	4504	const pid: pid_t = @intCast(pid_result); // We are the parent.
	4505	errdefer comptime unreachable; // The child is forked; we must not error from now on
	4506
	4507	ev.close(err_pipe[1]); // make sure only the child holds the write end open
	4508
	4509	if (options.stdin == .pipe) ev.close(stdin_pipe[0]);
	4510	if (options.stdout == .pipe) ev.close(stdout_pipe[1]);
	4511	if (options.stderr == .pipe) ev.close(stderr_pipe[1]);
	4512
	4513	if (prog_pipe[1] != -1) ev.close(prog_pipe[1]);
	4514
	4515	options.progress_node.setIpcFile(ev, .{ .handle = prog_pipe[0], .flags = .{ .nonblocking = true } });
	4516
	4517	return .{
	4518	.pid = pid,
	4519	.err_fd = err_pipe[0],
	4520	.stdin = switch (options.stdin) {
	4521	.pipe => .{ .handle = stdin_pipe[1], .flags = .{ .nonblocking = false } },
	4522	else => null,
	4523	},
	4524	.stdout = switch (options.stdout) {
	4525	.pipe => .{ .handle = stdout_pipe[0], .flags = .{ .nonblocking = false } },
	4526	else => null,
	4527	},
	4528	.stderr = switch (options.stderr) {
	4529	.pipe => .{ .handle = stderr_pipe[0], .flags = .{ .nonblocking = false } },
	4530	else => null,
	4531	},
	4532	};
	4533	}
	4534
	4535	pub const PipeError = error{
	4536	SystemFdQuotaExceeded,
	4537	ProcessFdQuotaExceeded,
	4538	} \|\| Io.UnexpectedError;
	4539	pub fn pipe2(flags: linux.O) PipeError![2]fd_t {
	4540	var fds: [2]fd_t = undefined;
	4541	switch (linux.errno(linux.pipe2(&fds, flags))) {
	4542	.SUCCESS => return fds,
	4543	.INVAL => \|err\| return errnoBug(err), // Invalid flags
	4544	.NFILE => return error.SystemFdQuotaExceeded,
	4545	.MFILE => return error.ProcessFdQuotaExceeded,
	4546	else => \|err\| return unexpectedErrno(err),
	4547	}
	4548	}
	4549	fn destroyPipe(ev: *Evented, pipe: [2]fd_t) void {
	4550	if (pipe[0] != -1) ev.close(pipe[0]);
	4551	if (pipe[0] != pipe[1]) ev.close(pipe[1]);
	4552	}
	4553
	4554	fn setUpChildIo(
	4555	ev: *Evented,
	4556	stdio: process.SpawnOptions.StdIo,
	4557	pipe_fd: fd_t,
	4558	std_fileno: i32,
	4559	dev_null_fd: fd_t,
	4560	) !void {
	4561	switch (stdio) {
	4562	.pipe => try dup2(pipe_fd, std_fileno),
	4563	.close => ev.close(std_fileno),
	4564	.inherit => {},
	4565	.ignore => try dup2(dev_null_fd, std_fileno),
	4566	.file => \|file\| try dup2(file.handle, std_fileno),
	4567	}
	4568	}
	4569
	4570	pub const DupError = error{
	4571	ProcessFdQuotaExceeded,
	4572	SystemResources,
	4573	} \|\| Io.UnexpectedError \|\| Io.Cancelable;
	4574	pub fn dup2(old_fd: fd_t, new_fd: fd_t) DupError!void {
	4575	var cancel_region: CancelRegion = .init();
	4576	defer cancel_region.deinit();
	4577	while (true) {
	4578	try cancel_region.await(.nothing);
	4579	switch (linux.errno(linux.dup2(old_fd, new_fd))) {
	4580	.SUCCESS => {},
	4581	.BUSY, .INTR => continue,
	4582	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	4583	.BADF => \|err\| return errnoBug(err), // use after free
	4584	.MFILE => return error.ProcessFdQuotaExceeded,
	4585	.NOMEM => return error.SystemResources,
	4586	else => \|err\| return unexpectedErrno(err),
	4587	}
	4588	}
	4589	}
	4590
	4591	/// Errors that can occur between fork() and execv()
	4592	const ForkBailError = process.SetCurrentDirError \|\| ChdirError \|\|
	4593	process.SpawnError \|\| process.ReplaceError;
	4594	/// Child of fork calls this to report an error to the fork parent. Then the
	4595	/// child exits.
	4596	fn forkBail(ev: *Evented, fd: fd_t, err: ForkBailError) noreturn {
	4597	var cancel_region: CancelRegion = .initBlocked();
	4598	defer cancel_region.deinit();
	4599	ev.writeAll(&cancel_region, fd, @ptrCast(&err)) catch {};
	4600	const exit = if (builtin.single_threaded) linux.exit else linux.exit_group;
	4601	exit(1);
	4602	}
	4603
	4604	fn execv(
	4605	arg0_expand: process.ArgExpansion,
	4606	file: [*:0]const u8,
	4607	child_argv: [:null]?[:0]const u8,
	4608	env_block: process.Environ.PosixBlock,
	4609	PATH: []const u8,
	4610	) process.ReplaceError {
	4611	const file_slice = std.mem.sliceTo(file, 0);
	4612	if (std.mem.findScalar(u8, file_slice, '/') != null) return execvPath(file, child_argv, env_block);
	4613
	4614	// Use of PATH_MAX here is valid as the path_buf will be passed
	4615	// directly to the operating system in posixExecvPath.
	4616	var path_buf: [PATH_MAX]u8 = undefined;
	4617	var it = std.mem.tokenizeScalar(u8, PATH, ':');
	4618	var seen_eacces = false;
	4619	var err: process.ReplaceError = error.FileNotFound;
	4620
	4621	// In case of expanding arg0 we must put it back if we return with an error.
	4622	const prev_arg0 = child_argv[0];
	4623	defer switch (arg0_expand) {
	4624	.expand => child_argv[0] = prev_arg0,
	4625	.no_expand => {},
	4626	};
	4627
	4628	while (it.next()) \|search_path\| {
	4629	const path_len = search_path.len + file_slice.len + 1;
	4630	if (path_buf.len < path_len + 1) return error.NameTooLong;
	4631	@memcpy(path_buf[0..search_path.len], search_path);
	4632	path_buf[search_path.len] = '/';
	4633	@memcpy(path_buf[search_path.len + 1 ..][0..file_slice.len], file_slice);
	4634	path_buf[path_len] = 0;
	4635	const full_path = path_buf[0..path_len :0].ptr;
	4636	switch (arg0_expand) {
	4637	.expand => child_argv[0] = full_path,
	4638	.no_expand => {},
	4639	}
	4640	err = execvPath(full_path, child_argv, env_block);
	4641	switch (err) {
	4642	error.AccessDenied => seen_eacces = true,
	4643	error.FileNotFound, error.NotDir => {},
	4644	else => \|e\| return e,
	4645	}
	4646	}
	4647	if (seen_eacces) return error.AccessDenied;
	4648	return err;
	4649	}
	4650	/// This function ignores PATH environment variable.
	4651	pub fn execvPath(
	4652	path: [*:0]const u8,
	4653	child_argv: [:null]const ?[:0]const u8,
	4654	env_block: process.Environ.PosixBlock,
	4655	) process.ReplaceError {
	4656	var cancel_region: CancelRegion = .init();
	4657	defer cancel_region.deinit();
	4658	try cancel_region.await(.nothing);
	4659	switch (linux.errno(linux.execve(path, child_argv, env_block.slice.ptr))) {
	4660	.FAULT => \|err\| return errnoBug(err), // Bad pointer parameter.
	4661	.@"2BIG" => return error.SystemResources,
	4662	.MFILE => return error.ProcessFdQuotaExceeded,
	4663	.NAMETOOLONG => return error.NameTooLong,
	4664	.NFILE => return error.SystemFdQuotaExceeded,
	4665	.NOMEM => return error.SystemResources,
	4666	.ACCES => return error.AccessDenied,
	4667	.PERM => return error.PermissionDenied,
	4668	.INVAL => return error.InvalidExe,
	4669	.NOEXEC => return error.InvalidExe,
	4670	.IO => return error.FileSystem,
	4671	.LOOP => return error.FileSystem,
	4672	.ISDIR => return error.IsDir,
	4673	.NOENT => return error.FileNotFound,
	4674	.NOTDIR => return error.NotDir,
	4675	.TXTBSY => return error.FileBusy,
	4676	.LIBBAD => return error.InvalidExe,
	4677	else => \|err\| return unexpectedErrno(err),
	4678	}
	4679	}
	4680
	4681	fn childWait(userdata: ?anyopaque, child: process.Child) process.Child.WaitError!process.Child.Term {
	4682	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4683	defer ev.childCleanup(child);
	4684
	4685	const pid = child.id.?;
	4686	var info: linux.siginfo_t = undefined;
	4687	var cancel_region: CancelRegion = .init();
	4688	defer cancel_region.deinit();
	4689	while (true) {
	4690	const thread = try cancel_region.awaitIoUring();
	4691	thread.enqueue().* = .{
	4692	.opcode = .WAITID,
	4693	.flags = 0,
	4694	.ioprio = 0,
	4695	.fd = pid,
	4696	.off = @intFromPtr(&info),
	4697	.addr = 0,
	4698	.len = @intFromEnum(linux.P.PID),
	4699	.rw_flags = 0,
	4700	.user_data = @intFromPtr(cancel_region.fiber),
	4701	.buf_index = 0,
	4702	.personality = 0,
	4703	.splice_fd_in = linux.W.EXITED \|
	4704	@as(i32, if (child.request_resource_usage_statistics) linux.W.NOWAIT else 0),
	4705	.addr3 = 0,
	4706	.resv = 0,
	4707	};
	4708	ev.yield(null, .nothing);
	4709	switch (cancel_region.errno()) {
	4710	.SUCCESS => {
	4711	if (child.request_resource_usage_statistics) while (true) {
	4712	try cancel_region.await(.nothing);
	4713	var rusage: linux.rusage = undefined;
	4714	switch (linux.errno(linux.waitid(
	4715	.PID,
	4716	pid,
	4717	&info,
	4718	linux.W.EXITED \| linux.W.NOHANG,
	4719	&rusage,
	4720	))) {
	4721	.SUCCESS => {
	4722	child.resource_usage_statistics.rusage = rusage;
	4723	break;
	4724	},
	4725	.INTR, .CANCELED => continue,
	4726	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4727	else => \|err\| return unexpectedErrno(err),
	4728	}
	4729	};
	4730	const status: u32 = @bitCast(info.fields.common.second.sigchld.status);
	4731	const code: linux.CLD = @enumFromInt(info.code);
	4732	return switch (code) {
	4733	.EXITED => .{ .exited = @truncate(status) },
	4734	.KILLED, .DUMPED => .{ .signal = @enumFromInt(status) },
	4735	.TRAPPED, .STOPPED => .{ .stopped = status },
	4736	_, .CONTINUED => .{ .unknown = status },
	4737	};
	4738	},
	4739	.INTR, .CANCELED => continue,
	4740	.CHILD => \|err\| return errnoBug(err), // Double-free.
	4741	else => \|err\| return unexpectedErrno(err),
	4742	}
	4743	}
	4744	}
	4745
	4746	fn childKill(userdata: ?anyopaque, child: process.Child) void {
	4747	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4748	defer ev.childCleanup(child);
	4749
	4750	const pid = child.id.?;
	4751	var cancel_region: CancelRegion = .initBlocked();
	4752	defer cancel_region.deinit();
	4753	while (true) switch (linux.errno(linux.kill(pid, .TERM))) {
	4754	.SUCCESS => break,
	4755	.INTR => continue,
	4756	.PERM => return,
	4757	.INVAL => \|err\| return errnoBug(err) catch {},
	4758	.SRCH => \|err\| return errnoBug(err) catch {},
	4759	else => \|err\| return unexpectedErrno(err) catch {},
	4760	};
	4761
	4762	var info: linux.siginfo_t = undefined;
	4763	while (true) {
	4764	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	4765	error.Canceled => unreachable, // blocked
	4766	};
	4767	thread.enqueue().* = .{
	4768	.opcode = .WAITID,
	4769	.flags = 0,
	4770	.ioprio = 0,
	4771	.fd = pid,
	4772	.off = @intFromPtr(&info),
	4773	.addr = 0,
	4774	.len = @intFromEnum(linux.P.PID),
	4775	.rw_flags = 0,
	4776	.user_data = @intFromPtr(cancel_region.fiber),
	4777	.buf_index = 0,
	4778	.personality = 0,
	4779	.splice_fd_in = linux.W.EXITED,
	4780	.addr3 = 0,
	4781	.resv = 0,
	4782	};
	4783	ev.yield(null, .nothing);
	4784	switch (cancel_region.errno()) {
	4785	.SUCCESS => return,
	4786	.INTR, .CANCELED => continue,
	4787	.CHILD => \|err\| return errnoBug(err) catch {}, // Double-free.
	4788	else => \|err\| return unexpectedErrno(err) catch {},
	4789	}
	4790	}
	4791	}
	4792
	4793	fn childCleanup(ev: Evented, child: process.Child) void {
	4794	if (child.stdin) \|*stdin\| {
	4795	ev.close(stdin.handle);
	4796	child.stdin = null;
	4797	}
	4798	if (child.stdout) \|*stdout\| {
	4799	ev.close(stdout.handle);
	4800	child.stdout = null;
	4801	}
	4802	if (child.stderr) \|*stderr\| {
	4803	ev.close(stderr.handle);
	4804	child.stderr = null;
	4805	}
	4806	child.id = null;
	4807	}
	4808
	4809	fn progressParentFile(userdata: ?*anyopaque) std.Progress.ParentFileError!File {
	4810	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4811	const cancel_protection = swapCancelProtection(ev, .blocked);
	4812	defer assert(swapCancelProtection(ev, cancel_protection) == .blocked);
	4813	ev.scanEnviron() catch \|err\| switch (err) {
	4814	error.Canceled => unreachable, // blocked
	4815	};
	4816	return ev.environ.zig_progress_file;
	4817	}
	4818
	4819	fn scanEnviron(ev: *Evented) Io.Cancelable!void {
	4820	const ev_io = ev.io();
	4821	try ev.environ_mutex.lock(ev_io);
	4822	defer ev.environ_mutex.unlock(ev_io);
	4823	ev.environ.scan(ev.allocator());
	4824	}
	4825
	4826	fn clockResolution(userdata: ?*anyopaque, clock: Io.Clock) Io.Clock.ResolutionError!Io.Duration {
	4827	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4828	_ = ev;
	4829	const clock_id = clockToPosix(clock);
	4830	var timespec: linux.timespec = undefined;
	4831	return switch (linux.errno(linux.clock_getres(clock_id, &timespec))) {
	4832	.SUCCESS => .fromNanoseconds(nanosecondsFromPosix(&timespec)),
	4833	.INVAL => return error.ClockUnavailable,
	4834	else => \|err\| return unexpectedErrno(err),
	4835	};
	4836	}
	4837
	4838	fn now(userdata: ?*anyopaque, clock: Io.Clock) Io.Timestamp {
	4839	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4840	_ = ev;
	4841	var tp: linux.timespec = undefined;
	4842	switch (linux.errno(linux.clock_gettime(clockToPosix(clock), &tp))) {
	4843	.SUCCESS => return timestampFromPosix(&tp),
	4844	else => return .zero,
	4845	}
	4846	}
	4847
	4848	fn sleep(userdata: ?*anyopaque, timeout: Io.Timeout) Io.Cancelable!void {
	4849	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4850
	4851	const timespec: linux.kernel_timespec, const clock: Io.Clock, const timeout_flags: u32 = timespec: switch (timeout) {
	4852	.none => .{
	4853	.{
	4854	.sec = std.math.maxInt(i64),
	4855	.nsec = std.time.ns_per_s - 1,
	4856	},
	4857	.awake,
	4858	linux.IORING_TIMEOUT_ABS,
	4859	},
	4860	.duration => \|duration\| {
	4861	const ns = duration.raw.toNanoseconds();
	4862	break :timespec .{
	4863	.{
	4864	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	4865	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	4866	},
	4867	duration.clock,
	4868	0,
	4869	};
	4870	},
	4871	.deadline => \|deadline\| {
	4872	const ns = deadline.raw.toNanoseconds();
	4873	break :timespec .{
	4874	.{
	4875	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	4876	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	4877	},
	4878	deadline.clock,
	4879	linux.IORING_TIMEOUT_ABS,
	4880	};
	4881	},
	4882	};
	4883	var cancel_region: CancelRegion = .init();
	4884	defer cancel_region.deinit();
	4885	const thread = try cancel_region.awaitIoUring();
	4886	thread.enqueue().* = .{
	4887	.opcode = .TIMEOUT,
	4888	.flags = 0,
	4889	.ioprio = 0,
	4890	.fd = 0,
	4891	.off = 0,
	4892	.addr = @intFromPtr(&timespec),
	4893	.len = 1,
	4894	.rw_flags = timeout_flags \| @as(u32, switch (clock) {
	4895	.real => linux.IORING_TIMEOUT_REALTIME,
	4896	else => 0,
	4897	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	4898	}),
	4899	.user_data = @intFromPtr(cancel_region.fiber),
	4900	.buf_index = 0,
	4901	.personality = 0,
	4902	.splice_fd_in = 0,
	4903	.addr3 = 0,
	4904	.resv = 0,
	4905	};
	4906	ev.yield(null, .nothing);
	4907	switch (cancel_region.errno()) {
	4908	// Handles SUCCESS as well as clock not available and unexpected
	4909	// errors. The user had a chance to check clock resolution before
	4910	// getting here, which would have reported 0, making this a legal
	4911	// amount of time to sleep.
	4912	else => return,
	4913	.INTR, .CANCELED => return error.Canceled,
	4914	}
	4915	}
	4916
	4917	fn random(userdata: ?*anyopaque, buffer: []u8) void {
	4918	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4919	var thread: *Thread = .current();
	4920	if (!thread.csprng.isInitialized()) {
	4921	@branchHint(.unlikely);
	4922	var seed: [Csprng.seed_len]u8 = undefined;
	4923	{
	4924	const ev_io = ev.io();
	4925	ev.csprng_mutex.lockUncancelable(ev_io);
	4926	defer ev.csprng_mutex.unlock(ev_io);
	4927	if (!ev.csprng.isInitialized()) {
	4928	@branchHint(.unlikely);
	4929	var cancel_region: CancelRegion = .initBlocked();
	4930	defer cancel_region.deinit();
	4931	ev.urandomReadAll(&cancel_region, &seed) catch \|err\| switch (err) {
	4932	error.Canceled => unreachable, // blocked
	4933	else => fallbackSeed(ev, &seed),
	4934	};
	4935	ev.csprng.rng = .init(seed);
	4936	thread = .current();
	4937	}
	4938	ev.csprng.rng.fill(&seed);
	4939	}
	4940	if (!thread.csprng.isInitialized()) {
	4941	@branchHint(.likely);
	4942	thread.csprng.rng = .init(seed);
	4943	} else thread.csprng.rng.addEntropy(&seed);
	4944	}
	4945	thread.csprng.rng.fill(buffer);
	4946	}
	4947
	4948	fn randomSecure(userdata: ?*anyopaque, buffer: []u8) Io.RandomSecureError!void {
	4949	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4950	if (buffer.len == 0) return;
	4951	var cancel_region: CancelRegion = .init();
	4952	defer cancel_region.deinit();
	4953	ev.urandomReadAll(&cancel_region, buffer) catch \|err\| switch (err) {
	4954	error.Canceled => return error.Canceled,
	4955	else => return error.EntropyUnavailable,
	4956	};
	4957	}
	4958
	4959	fn netListenIpUnavailable(
	4960	userdata: ?*anyopaque,
	4961	address: net.IpAddress,
	4962	options: net.IpAddress.ListenOptions,
	4963	) net.IpAddress.ListenError!net.Server {
	4964	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4965	_ = ev;
	4966	_ = address;
	4967	_ = options;
	4968	return error.NetworkDown;
	4969	}
	4970
	4971	fn netAcceptUnavailable(
	4972	userdata: ?*anyopaque,
	4973	listen_handle: net.Socket.Handle,
	4974	) net.Server.AcceptError!net.Stream {
	4975	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4976	_ = ev;
	4977	_ = listen_handle;
	4978	return error.NetworkDown;
	4979	}
	4980
	4981	fn netBindIp(
	4982	userdata: ?*anyopaque,
	4983	address: *const net.IpAddress,
	4984	options: net.IpAddress.BindOptions,
	4985	) net.IpAddress.BindError!net.Socket {
	4986	const ev: *Evented = @ptrCast(@alignCast(userdata));
	4987	const family = posixAddressFamily(address);
	4988	var cancel_region: CancelRegion = .init();
	4989	defer cancel_region.deinit();
	4990	const socket_fd = try ev.socket(&cancel_region, family, options);
	4991	errdefer ev.close(socket_fd);
	4992	var storage: PosixAddress = undefined;
	4993	var addr_len = addressToPosix(address, &storage);
	4994	try ev.bind(&cancel_region, socket_fd, &storage.any, addr_len);
	4995	try ev.getsockname(&cancel_region, socket_fd, &storage.any, &addr_len);
	4996	return .{
	4997	.handle = socket_fd,
	4998	.address = addressFromPosix(&storage),
	4999	};
	5000	}
	5001
	5002	fn netBindIpUnavailable(
	5003	userdata: ?*anyopaque,
	5004	address: *const net.IpAddress,
	5005	options: net.IpAddress.BindOptions,
	5006	) net.IpAddress.BindError!net.Socket {
	5007	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5008	_ = ev;
	5009	_ = address;
	5010	_ = options;
	5011	return error.NetworkDown;
	5012	}
	5013
	5014	fn netConnectIpUnavailable(
	5015	userdata: ?*anyopaque,
	5016	address: *const net.IpAddress,
	5017	options: net.IpAddress.ConnectOptions,
	5018	) net.IpAddress.ConnectError!net.Stream {
	5019	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5020	_ = ev;
	5021	_ = address;
	5022	_ = options;
	5023	return error.NetworkDown;
	5024	}
	5025
	5026	fn netListenUnixUnavailable(
	5027	userdata: ?*anyopaque,
	5028	address: *const net.UnixAddress,
	5029	options: net.UnixAddress.ListenOptions,
	5030	) net.UnixAddress.ListenError!net.Socket.Handle {
	5031	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5032	_ = ev;
	5033	_ = address;
	5034	_ = options;
	5035	return error.AddressFamilyUnsupported;
	5036	}
	5037
	5038	fn netConnectUnixUnavailable(
	5039	userdata: ?*anyopaque,
	5040	address: *const net.UnixAddress,
	5041	) net.UnixAddress.ConnectError!net.Socket.Handle {
	5042	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5043	_ = ev;
	5044	_ = address;
	5045	return error.AddressFamilyUnsupported;
	5046	}
	5047
	5048	fn netSocketCreatePairUnavailable(
	5049	userdata: ?*anyopaque,
	5050	options: net.Socket.CreatePairOptions,
	5051	) net.Socket.CreatePairError![2]net.Socket {
	5052	_ = userdata;
	5053	_ = options;
	5054	return error.OperationUnsupported;
	5055	}
	5056
	5057	fn netSendUnavailable(
	5058	userdata: ?*anyopaque,
	5059	handle: net.Socket.Handle,
	5060	messages: []net.OutgoingMessage,
	5061	flags: net.SendFlags,
	5062	) struct { ?net.Socket.SendError, usize } {
	5063	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5064	_ = ev;
	5065	_ = handle;
	5066	_ = messages;
	5067	_ = flags;
	5068	return .{ error.NetworkDown, 0 };
	5069	}
	5070
	5071	fn netReceive(
	5072	userdata: ?*anyopaque,
	5073	handle: net.Socket.Handle,
	5074	message_buffer: []net.IncomingMessage,
	5075	data_buffer: []u8,
	5076	flags: net.ReceiveFlags,
	5077	timeout: Io.Timeout,
	5078	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
	5079	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5080	const ev_io = ev.io();
	5081
	5082	var message_i: usize = 0;
	5083	var data_i: usize = 0;
	5084
	5085	const deadline: ?struct {
	5086	raw: Io.Timestamp,
	5087	timespec: linux.kernel_timespec,
	5088	clock: Io.Clock,
	5089	} = if (timeout.toTimestamp(ev_io)) \|deadline\| deadline: {
	5090	const ns = deadline.raw.toNanoseconds();
	5091	break :deadline .{
	5092	.raw = deadline.raw,
	5093	.timespec = .{
	5094	.sec = @intCast(@divFloor(ns, std.time.ns_per_s)),
	5095	.nsec = @intCast(@mod(ns, std.time.ns_per_s)),
	5096	},
	5097	.clock = deadline.clock,
	5098	};
	5099	} else null;
	5100
	5101	var cancel_region: CancelRegion = .init();
	5102	defer cancel_region.deinit();
	5103	while (true) {
	5104	if (message_buffer.len - message_i == 0) return .{ null, message_i };
	5105	const message = &message_buffer[message_i];
	5106	const remaining_data_buffer = data_buffer[data_i..];
	5107	var storage: PosixAddress = undefined;
	5108	var iov: iovec = .{ .base = remaining_data_buffer.ptr, .len = remaining_data_buffer.len };
	5109	var msg: linux.msghdr = .{
	5110	.name = &storage.any,
	5111	.namelen = @sizeOf(PosixAddress),
	5112	.iov = (&iov)[0..1],
	5113	.iovlen = 1,
	5114	.control = message.control.ptr,
	5115	.controllen = @intCast(message.control.len),
	5116	.flags = undefined,
	5117	};
	5118
	5119	const thread = cancel_region.awaitIoUring() catch \|err\| return .{ err, message_i };
	5120	thread.enqueue().* = .{
	5121	.opcode = .RECVMSG,
	5122	.flags = if (deadline) \|_\| linux.IOSQE_IO_LINK else 0,
	5123	.ioprio = 0,
	5124	.fd = handle,
	5125	.off = 0,
	5126	.addr = @intFromPtr(&msg),
	5127	.len = 0,
	5128	.rw_flags = linux.MSG.NOSIGNAL \|
	5129	@as(u32, if (flags.oob) linux.MSG.OOB else 0) \|
	5130	@as(u32, if (flags.peek) linux.MSG.PEEK else 0) \|
	5131	@as(u32, if (flags.trunc) linux.MSG.TRUNC else 0),
	5132	.user_data = @intFromPtr(cancel_region.fiber),
	5133	.buf_index = 0,
	5134	.personality = 0,
	5135	.splice_fd_in = 0,
	5136	.addr3 = 0,
	5137	.resv = 0,
	5138	};
	5139	if (deadline) \|deadline_ptr\| thread.enqueue(). = .{
	5140	.opcode = .LINK_TIMEOUT,
	5141	.flags = linux.IOSQE_CQE_SKIP_SUCCESS,
	5142	.ioprio = 0,
	5143	.fd = 0,
	5144	.off = 0,
	5145	.addr = @intFromPtr(&deadline_ptr.timespec),
	5146	.len = 1,
	5147	.rw_flags = linux.IORING_TIMEOUT_ABS \| @as(u32, switch (deadline_ptr.clock) {
	5148	.real => linux.IORING_TIMEOUT_REALTIME,
	5149	else => 0,
	5150	.boot => linux.IORING_TIMEOUT_BOOTTIME,
	5151	}),
	5152	.user_data = @intFromEnum(Completion.UserData.wakeup),
	5153	.buf_index = 0,
	5154	.personality = 0,
	5155	.splice_fd_in = 0,
	5156	.addr3 = 0,
	5157	.resv = 0,
	5158	};
	5159	ev.yield(null, .nothing);
	5160	const completion = cancel_region.completion();
	5161	switch (completion.errno()) {
	5162	.SUCCESS => {
	5163	const data = remaining_data_buffer[0..@intCast(completion.result)];
	5164	data_i += data.len;
	5165	message.* = .{
	5166	.from = addressFromPosix(&storage),
	5167	.data = data,
	5168	.control = if (msg.control) \|ptr\| @as([*]u8, @ptrCast(ptr))[0..msg.controllen] else message.control,
	5169	.flags = .{
	5170	.eor = (msg.flags & linux.MSG.EOR) != 0,
	5171	.trunc = (msg.flags & linux.MSG.TRUNC) != 0,
	5172	.ctrunc = (msg.flags & linux.MSG.CTRUNC) != 0,
	5173	.oob = (msg.flags & linux.MSG.OOB) != 0,
	5174	.errqueue = if (@hasDecl(linux.MSG, "ERRQUEUE")) (msg.flags & linux.MSG.ERRQUEUE) != 0 else false,
	5175	},
	5176	};
	5177	message_i += 1;
	5178	continue;
	5179	},
	5180	.AGAIN => unreachable,
	5181	.INTR, .CANCELED => {
	5182	if (deadline) \|d\| {
	5183	if (now(ev, d.clock).nanoseconds >= d.raw.nanoseconds) return .{ error.Timeout, message_i };
	5184	}
	5185	continue;
	5186	},
	5187
	5188	.BADF => \|err\| return .{ errnoBug(err), message_i },
	5189	.NFILE => return .{ error.SystemFdQuotaExceeded, message_i },
	5190	.MFILE => return .{ error.ProcessFdQuotaExceeded, message_i },
	5191	.FAULT => \|err\| return .{ errnoBug(err), message_i },
	5192	.INVAL => \|err\| return .{ errnoBug(err), message_i },
	5193	.NOBUFS => return .{ error.SystemResources, message_i },
	5194	.NOMEM => return .{ error.SystemResources, message_i },
	5195	.NOTCONN => return .{ error.SocketUnconnected, message_i },
	5196	.NOTSOCK => \|err\| return .{ errnoBug(err), message_i },
	5197	.MSGSIZE => return .{ error.MessageOversize, message_i },
	5198	.PIPE => return .{ error.SocketUnconnected, message_i },
	5199	.OPNOTSUPP => \|err\| return .{ errnoBug(err), message_i },
	5200	.CONNRESET => return .{ error.ConnectionResetByPeer, message_i },
	5201	.NETDOWN => return .{ error.NetworkDown, message_i },
	5202	else => \|err\| return .{ unexpectedErrno(err), message_i },
	5203	}
	5204	}
	5205	}
	5206
	5207	fn netReceiveUnavailable(
	5208	userdata: ?*anyopaque,
	5209	handle: net.Socket.Handle,
	5210	message_buffer: []net.IncomingMessage,
	5211	data_buffer: []u8,
	5212	flags: net.ReceiveFlags,
	5213	timeout: Io.Timeout,
	5214	) struct { ?net.Socket.ReceiveTimeoutError, usize } {
	5215	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5216	_ = ev;
	5217	_ = handle;
	5218	_ = message_buffer;
	5219	_ = data_buffer;
	5220	_ = flags;
	5221	_ = timeout;
	5222	return .{ error.NetworkDown, 0 };
	5223	}
	5224
	5225	fn netReadUnavailable(
	5226	userdata: ?*anyopaque,
	5227	fd: net.Socket.Handle,
	5228	data: [][]u8,
	5229	) net.Stream.Reader.Error!usize {
	5230	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5231	_ = ev;
	5232	_ = fd;
	5233	_ = data;
	5234	return error.NetworkDown;
	5235	}
	5236
	5237	fn netWriteUnavailable(
	5238	userdata: ?*anyopaque,
	5239	handle: net.Socket.Handle,
	5240	header: []const u8,
	5241	data: []const []const u8,
	5242	splat: usize,
	5243	) net.Stream.Writer.Error!usize {
	5244	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5245	_ = ev;
	5246	_ = handle;
	5247	_ = header;
	5248	_ = data;
	5249	_ = splat;
	5250	return error.NetworkDown;
	5251	}
	5252
	5253	fn netWriteFileUnavailable(
	5254	userdata: ?*anyopaque,
	5255	socket_handle: net.Socket.Handle,
	5256	header: []const u8,
	5257	file_reader: *File.Reader,
	5258	limit: Io.Limit,
	5259	) net.Stream.Writer.WriteFileError!usize {
	5260	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5261	_ = ev;
	5262	_ = socket_handle;
	5263	_ = header;
	5264	_ = file_reader;
	5265	_ = limit;
	5266	return error.NetworkDown;
	5267	}
	5268
	5269	fn netClose(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
	5270	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5271	for (handles) \|handle\| ev.close(handle);
	5272	}
	5273
	5274	fn netCloseUnavailable(userdata: ?*anyopaque, handles: []const net.Socket.Handle) void {
	5275	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5276	_ = ev;
	5277	_ = handles;
	5278	unreachable; // How you gonna close something that was impossible to open?
	5279	}
	5280
	5281	fn netShutdown(
	5282	userdata: ?*anyopaque,
	5283	handle: net.Socket.Handle,
	5284	how: net.ShutdownHow,
	5285	) net.ShutdownError!void {
	5286	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5287	var cancel_region: CancelRegion = .init();
	5288	defer cancel_region.deinit();
	5289	while (true) {
	5290	const thread = try cancel_region.awaitIoUring();
	5291	thread.enqueue().* = .{
	5292	.opcode = .SHUTDOWN,
	5293	.flags = 0,
	5294	.ioprio = 0,
	5295	.fd = handle,
	5296	.off = 0,
	5297	.addr = 0,
	5298	.len = switch (how) {
	5299	.recv => linux.SHUT.RD,
	5300	.send => linux.SHUT.WR,
	5301	.both => linux.SHUT.RDWR,
	5302	},
	5303	.rw_flags = 0,
	5304	.user_data = @intFromPtr(cancel_region.fiber),
	5305	.buf_index = 0,
	5306	.personality = 0,
	5307	.splice_fd_in = 0,
	5308	.addr3 = 0,
	5309	.resv = 0,
	5310	};
	5311	ev.yield(null, .nothing);
	5312	switch (cancel_region.errno()) {
	5313	.SUCCESS => return,
	5314	.INTR, .CANCELED => continue,
	5315	.BADF, .NOTSOCK, .INVAL => \|err\| return errnoBug(err),
	5316	.NOTCONN => return error.SocketUnconnected,
	5317	.NOBUFS => return error.SystemResources,
	5318	else => \|err\| return unexpectedErrno(err),
	5319	}
	5320	}
	5321	}
	5322
	5323	fn netShutdownUnavailable(
	5324	userdata: ?*anyopaque,
	5325	handle: net.Socket.Handle,
	5326	how: net.ShutdownHow,
	5327	) net.ShutdownError!void {
	5328	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5329	_ = ev;
	5330	_ = handle;
	5331	_ = how;
	5332	unreachable; // How you gonna shutdown something that was impossible to open?
	5333	}
	5334
	5335	fn netInterfaceNameResolveUnavailable(
	5336	userdata: ?*anyopaque,
	5337	name: *const net.Interface.Name,
	5338	) net.Interface.Name.ResolveError!net.Interface {
	5339	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5340	_ = ev;
	5341	_ = name;
	5342	return error.InterfaceNotFound;
	5343	}
	5344
	5345	fn netInterfaceNameUnavailable(
	5346	userdata: ?*anyopaque,
	5347	interface: net.Interface,
	5348	) net.Interface.NameError!net.Interface.Name {
	5349	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5350	_ = ev;
	5351	_ = interface;
	5352	return error.Unexpected;
	5353	}
	5354
	5355	fn netLookupUnavailable(
	5356	userdata: ?*anyopaque,
	5357	host_name: net.HostName,
	5358	resolved: *Io.Queue(net.HostName.LookupResult),
	5359	options: net.HostName.LookupOptions,
	5360	) net.HostName.LookupError!void {
	5361	const ev: *Evented = @ptrCast(@alignCast(userdata));
	5362	_ = host_name;
	5363	_ = options;
	5364	resolved.close(ev.io());
	5365	return error.NetworkDown;
	5366	}
	5367
	5368	fn bind(
	5369	ev: *Evented,
	5370	cancel_region: *CancelRegion,
	5371	socket_fd: fd_t,
	5372	addr: *const linux.sockaddr,
	5373	addr_len: linux.socklen_t,
	5374	) !void {
	5375	while (true) {
	5376	const thread = try cancel_region.awaitIoUring();
	5377	thread.enqueue().* = .{
	5378	.opcode = .BIND,
	5379	.flags = 0,
	5380	.ioprio = 0,
	5381	.fd = socket_fd,
	5382	.off = addr_len,
	5383	.addr = @intFromPtr(addr),
	5384	.len = 0,
	5385	.rw_flags = 0,
	5386	.user_data = @intFromPtr(cancel_region.fiber),
	5387	.buf_index = 0,
	5388	.personality = 0,
	5389	.splice_fd_in = 0,
	5390	.addr3 = 0,
	5391	.resv = 0,
	5392	};
	5393	ev.yield(null, .nothing);
	5394	switch (cancel_region.errno()) {
	5395	.SUCCESS => return,
	5396	.INTR, .CANCELED => continue,
	5397	.ADDRINUSE => return error.AddressInUse,
	5398	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5399	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5400	.NOTSOCK => \|err\| return errnoBug(err), // invalid `sockfd`
	5401	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
	5402	.ADDRNOTAVAIL => return error.AddressUnavailable,
	5403	.FAULT => \|err\| return errnoBug(err), // invalid `addr` pointer
	5404	.NOMEM => return error.SystemResources,
	5405	else => \|err\| return unexpectedErrno(err),
	5406	}
	5407	}
	5408	}
	5409
	5410	fn close(ev: *Evented, fd: fd_t) void {
	5411	var cancel_region: CancelRegion = .initBlocked();
	5412	defer cancel_region.deinit();
	5413	while (true) {
	5414	const thread = cancel_region.awaitIoUring() catch \|err\| switch (err) {
	5415	error.Canceled => unreachable, // blocked
	5416	};
	5417	thread.enqueue().* = .{
	5418	.opcode = .CLOSE,
	5419	.flags = 0,
	5420	.ioprio = 0,
	5421	.fd = fd,
	5422	.off = 0,
	5423	.addr = 0,
	5424	.len = 0,
	5425	.rw_flags = 0,
	5426	.user_data = @intFromPtr(cancel_region.fiber),
	5427	.buf_index = 0,
	5428	.personality = 0,
	5429	.splice_fd_in = 0,
	5430	.addr3 = 0,
	5431	.resv = 0,
	5432	};
	5433	ev.yield(null, .nothing);
	5434	switch (cancel_region.errno()) {
	5435	.SUCCESS => return,
	5436	.INTR, .CANCELED => continue,
	5437	.BADF => unreachable, // Always a race condition.
	5438	else => break,
	5439	}
	5440	}
	5441	}
	5442
	5443	fn fchmodat(
	5444	ev: *Evented,
	5445	cancel_region: *CancelRegion,
	5446	dir: fd_t,
	5447	path: [*:0]const u8,
	5448	mode: linux.mode_t,
	5449	flags: u32,
	5450	) Dir.SetFilePermissionsError!void {
	5451	_ = ev;
	5452	while (true) {
	5453	try cancel_region.await(.nothing);
	5454	switch (linux.errno(linux.fchmodat2(dir, path, mode, flags))) {
	5455	.SUCCESS => return,
	5456	.INTR => continue,
	5457	.BADF => \|err\| return errnoBug(err),
	5458	.FAULT => \|err\| return errnoBug(err),
	5459	.INVAL => \|err\| return errnoBug(err),
	5460	.ACCES => return error.AccessDenied,
	5461	.IO => return error.InputOutput,
	5462	.LOOP => return error.SymLinkLoop,
	5463	.NOENT => return error.FileNotFound,
	5464	.NOMEM => return error.SystemResources,
	5465	.NOTDIR => return error.FileNotFound,
	5466	.OPNOTSUPP => return error.OperationUnsupported,
	5467	.PERM => return error.PermissionDenied,
	5468	.ROFS => return error.ReadOnlyFileSystem,
	5469	else => \|err\| return unexpectedErrno(err),
	5470	}
	5471	}
	5472	}
	5473
	5474	fn fchownat(
	5475	ev: *Evented,
	5476	cancel_region: *CancelRegion,
	5477	dir: fd_t,
	5478	path: [*:0]const u8,
	5479	owner: linux.uid_t,
	5480	group: linux.gid_t,
	5481	flags: u32,
	5482	) File.SetOwnerError!void {
	5483	_ = ev;
	5484	while (true) {
	5485	try cancel_region.await(.nothing);
	5486	switch (linux.errno(linux.fchownat(dir, path, owner, group, flags))) {
	5487	.SUCCESS => return,
	5488	.INTR => continue,
	5489	.BADF => \|err\| return errnoBug(err), // likely fd refers to directory opened without `Dir.OpenOptions.iterate`
	5490	.FAULT => \|err\| return errnoBug(err),
	5491	.INVAL => \|err\| return errnoBug(err),
	5492	.ACCES => return error.AccessDenied,
	5493	.IO => return error.InputOutput,
	5494	.LOOP => return error.SymLinkLoop,
	5495	.NOENT => return error.FileNotFound,
	5496	.NOMEM => return error.SystemResources,
	5497	.NOTDIR => return error.FileNotFound,
	5498	.PERM => return error.PermissionDenied,
	5499	.ROFS => return error.ReadOnlyFileSystem,
	5500	else => \|err\| return unexpectedErrno(err),
	5501	}
	5502	}
	5503	}
	5504
	5505	fn flock(
	5506	ev: *Evented,
	5507	cancel_region: *CancelRegion,
	5508	fd: fd_t,
	5509	op: File.Lock,
	5510	blocking: enum { blocking, nonblocking },
	5511	) (File.LockError \|\| error{WouldBlock})!void {
	5512	while (true) {
	5513	try cancel_region.await(.nothing);
	5514	switch (linux.errno(linux.flock(fd, LOCK.NB \| @as(i32, switch (op) {
	5515	.none => LOCK.UN,
	5516	.shared => LOCK.SH,
	5517	.exclusive => LOCK.EX,
	5518	})))) {
	5519	.SUCCESS => return,
	5520	.INTR => continue,
	5521	.BADF => \|err\| return errnoBug(err),
	5522	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5523	.NOLCK => return error.SystemResources,
	5524	.AGAIN => {
	5525	const thread = try cancel_region.awaitIoUring();
	5526	thread.enqueue().* = .{
	5527	.opcode = .NOP,
	5528	.flags = 0,
	5529	.ioprio = 0,
	5530	.fd = 0,
	5531	.off = 0,
	5532	.addr = 0,
	5533	.len = 0,
	5534	.rw_flags = 0,
	5535	.user_data = @intFromPtr(cancel_region.fiber),
	5536	.buf_index = 0,
	5537	.personality = 0,
	5538	.splice_fd_in = 0,
	5539	.addr3 = 0,
	5540	.resv = 0,
	5541	};
	5542	ev.yield(null, .nothing);
	5543	switch (cancel_region.errno()) {
	5544	.SUCCESS, .INTR, .CANCELED => {},
	5545	else => unreachable,
	5546	}
	5547	switch (blocking) {
	5548	.blocking => continue,
	5549	.nonblocking => return error.WouldBlock,
	5550	}
	5551	},
	5552	.OPNOTSUPP => return error.FileLocksUnsupported,
	5553	else => \|err\| return unexpectedErrno(err),
	5554	}
	5555	}
	5556	}
	5557
	5558	fn getsockname(
	5559	ev: *Evented,
	5560	cancel_region: *CancelRegion,
	5561	socket_fd: fd_t,
	5562	addr: *linux.sockaddr,
	5563	addr_len: *linux.socklen_t,
	5564	) !void {
	5565	_ = ev;
	5566	while (true) {
	5567	try cancel_region.await(.nothing);
	5568	switch (linux.errno(linux.getsockname(socket_fd, addr, addr_len))) {
	5569	.SUCCESS => return,
	5570	.INTR => continue,
	5571	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5572	.FAULT => \|err\| return errnoBug(err),
	5573	.INVAL => \|err\| return errnoBug(err), // invalid parameters
	5574	.NOTSOCK => \|err\| return errnoBug(err), // always a race condition
	5575	.NOBUFS => return error.SystemResources,
	5576	else => \|err\| return unexpectedErrno(err),
	5577	}
	5578	}
	5579	}
	5580
	5581	fn linkat(
	5582	ev: *Evented,
	5583	cancel_region: *CancelRegion,
	5584	old_dir: fd_t,
	5585	old_path: [*:0]const u8,
	5586	new_dir: fd_t,
	5587	new_path: [*:0]const u8,
	5588	flags: u32,
	5589	) File.HardLinkError!void {
	5590	while (true) {
	5591	const thread = try cancel_region.awaitIoUring();
	5592	thread.enqueue().* = .{
	5593	.opcode = .LINKAT,
	5594	.flags = 0,
	5595	.ioprio = 0,
	5596	.fd = old_dir,
	5597	.off = @intFromPtr(new_path),
	5598	.addr = @intFromPtr(old_path),
	5599	.len = @bitCast(new_dir),
	5600	.rw_flags = flags,
	5601	.user_data = @intFromPtr(cancel_region.fiber),
	5602	.buf_index = 0,
	5603	.personality = 0,
	5604	.splice_fd_in = 0,
	5605	.addr3 = 0,
	5606	.resv = 0,
	5607	};
	5608	ev.yield(null, .nothing);
	5609	switch (cancel_region.errno()) {
	5610	.SUCCESS => return,
	5611	.INTR, .CANCELED => continue,
	5612	.ACCES => return error.AccessDenied,
	5613	.DQUOT => return error.DiskQuota,
	5614	.EXIST => return error.PathAlreadyExists,
	5615	.IO => return error.HardwareFailure,
	5616	.LOOP => return error.SymLinkLoop,
	5617	.MLINK => return error.LinkQuotaExceeded,
	5618	.NAMETOOLONG => return error.NameTooLong,
	5619	.NOENT => return error.FileNotFound,
	5620	.NOMEM => return error.SystemResources,
	5621	.NOSPC => return error.NoSpaceLeft,
	5622	.NOTDIR => return error.NotDir,
	5623	.PERM => return error.PermissionDenied,
	5624	.ROFS => return error.ReadOnlyFileSystem,
	5625	.XDEV => return error.CrossDevice,
	5626	.ILSEQ => return error.BadPathName,
	5627	.FAULT => \|err\| return errnoBug(err),
	5628	.INVAL => \|err\| return errnoBug(err),
	5629	else => \|err\| return unexpectedErrno(err),
	5630	}
	5631	}
	5632	}
	5633
	5634	fn lseek(
	5635	ev: *Evented,
	5636	cancel_region: *CancelRegion,
	5637	fd: fd_t,
	5638	offset: u64,
	5639	whence: u32,
	5640	) File.SeekError!void {
	5641	_ = ev;
	5642	while (true) {
	5643	try cancel_region.await(.nothing);
	5644	var result: u64 = undefined;
	5645	switch (linux.errno(switch (@sizeOf(usize)) {
	5646	else => comptime unreachable,
	5647	4 => linux.llseek(fd, offset, &result, whence),
	5648	8 => linux.lseek(fd, @bitCast(offset), whence),
	5649	})) {
	5650	.SUCCESS => return,
	5651	.INTR => continue,
	5652	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5653	.INVAL => return error.Unseekable,
	5654	.OVERFLOW => return error.Unseekable,
	5655	.SPIPE => return error.Unseekable,
	5656	.NXIO => return error.Unseekable,
	5657	else => \|err\| return unexpectedErrno(err),
	5658	}
	5659	}
	5660	}
	5661
	5662	fn openat(
	5663	ev: *Evented,
	5664	cancel_region: *CancelRegion,
	5665	dir: fd_t,
	5666	path: [*:0]const u8,
	5667	flags: linux.O,
	5668	mode: linux.mode_t,
	5669	) File.OpenError!fd_t {
	5670	var mut_flags = flags;
	5671	if (@hasField(linux.O, "LARGEFILE")) mut_flags.LARGEFILE = true;
	5672	while (true) {
	5673	const thread = try cancel_region.awaitIoUring();
	5674	thread.enqueue().* = .{
	5675	.opcode = .OPENAT,
	5676	.flags = 0,
	5677	.ioprio = 0,
	5678	.fd = dir,
	5679	.off = 0,
	5680	.addr = @intFromPtr(path),
	5681	.len = mode,
	5682	.rw_flags = @bitCast(mut_flags),
	5683	.user_data = @intFromPtr(cancel_region.fiber),
	5684	.buf_index = 0,
	5685	.personality = 0,
	5686	.splice_fd_in = 0,
	5687	.addr3 = 0,
	5688	.resv = 0,
	5689	};
	5690	ev.yield(null, .nothing);
	5691	const completion = cancel_region.completion();
	5692	switch (completion.errno()) {
	5693	.SUCCESS => return completion.result,
	5694	.INTR, .CANCELED => continue,
	5695	.FAULT => \|err\| return errnoBug(err),
	5696	.INVAL => return error.BadPathName,
	5697	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5698	.ACCES => return error.AccessDenied,
	5699	.FBIG => return error.FileTooBig,
	5700	.OVERFLOW => return error.FileTooBig,
	5701	.ISDIR => return error.IsDir,
	5702	.LOOP => return error.SymLinkLoop,
	5703	.MFILE => return error.ProcessFdQuotaExceeded,
	5704	.NAMETOOLONG => return error.NameTooLong,
	5705	.NFILE => return error.SystemFdQuotaExceeded,
	5706	.NODEV => return error.NoDevice,
	5707	.NOENT => return error.FileNotFound,
	5708	.SRCH => return error.FileNotFound, // Linux when opening procfs files.
	5709	.NOMEM => return error.SystemResources,
	5710	.NOSPC => return error.NoSpaceLeft,
	5711	.NOTDIR => return error.NotDir,
	5712	.PERM => return error.PermissionDenied,
	5713	.EXIST => return error.PathAlreadyExists,
	5714	.BUSY => return error.DeviceBusy,
	5715	.OPNOTSUPP => return error.FileLocksUnsupported,
	5716	.AGAIN => return error.WouldBlock,
	5717	.TXTBSY => return error.FileBusy,
	5718	.NXIO => return error.NoDevice,
	5719	.ILSEQ => return error.BadPathName,
	5720	else => \|err\| return unexpectedErrno(err),
	5721	}
	5722	}
	5723	}
	5724
	5725	fn preadv(
	5726	ev: *Evented,
	5727	cancel_region: *CancelRegion,
	5728	fd: fd_t,
	5729	iov: []const iovec,
	5730	offset: ?u64,
	5731	) File.Reader.Error!usize {
	5732	if (iov.len == 0) return 0;
	5733	const gather = iov.len > 1 or iov[0].len > 0xfffff000;
	5734	while (true) {
	5735	const thread = try cancel_region.awaitIoUring();
	5736	thread.enqueue().* = .{
	5737	.opcode = if (gather) .READV else .READ,
	5738	.flags = 0,
	5739	.ioprio = 0,
	5740	.fd = fd,
	5741	.off = offset orelse std.math.maxInt(u64),
	5742	.addr = if (gather) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
	5743	.len = @intCast(if (gather) iov.len else iov[0].len),
	5744	.rw_flags = 0,
	5745	.user_data = @intFromPtr(cancel_region.fiber),
	5746	.buf_index = 0,
	5747	.personality = 0,
	5748	.splice_fd_in = 0,
	5749	.addr3 = 0,
	5750	.resv = 0,
	5751	};
	5752	ev.yield(null, .nothing);
	5753	const completion = cancel_region.completion();
	5754	switch (completion.errno()) {
	5755	.SUCCESS => return @as(u32, @bitCast(completion.result)),
	5756	.INTR, .CANCELED => continue,
	5757	.INVAL => \|err\| return errnoBug(err),
	5758	.FAULT => \|err\| return errnoBug(err),
	5759	.AGAIN => return error.WouldBlock,
	5760	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed
	5761	.IO => return error.InputOutput,
	5762	.ISDIR => return error.IsDir,
	5763	.NOBUFS => return error.SystemResources,
	5764	.NOMEM => return error.SystemResources,
	5765	.NOTCONN => return error.SocketUnconnected,
	5766	.CONNRESET => return error.ConnectionResetByPeer,
	5767	else => \|err\| return unexpectedErrno(err),
	5768	}
	5769	}
	5770	}
	5771
	5772	fn pwritev(
	5773	ev: *Evented,
	5774	cancel_region: *CancelRegion,
	5775	fd: fd_t,
	5776	iov: []const iovec_const,
	5777	offset: ?u64,
	5778	) File.Writer.Error!usize {
	5779	if (iov.len == 0) return 0;
	5780	const scatter = iov.len > 1 or iov[0].len > 0xfffff000;
	5781	while (true) {
	5782	const thread = try cancel_region.awaitIoUring();
	5783	thread.enqueue().* = .{
	5784	.opcode = if (scatter) .WRITEV else .WRITE,
	5785	.flags = 0,
	5786	.ioprio = 0,
	5787	.fd = fd,
	5788	.off = offset orelse std.math.maxInt(u64),
	5789	.addr = if (scatter) @intFromPtr(iov.ptr) else @intFromPtr(iov[0].base),
	5790	.len = @intCast(if (scatter) iov.len else iov[0].len),
	5791	.rw_flags = 0,
	5792	.user_data = @intFromPtr(cancel_region.fiber),
	5793	.buf_index = 0,
	5794	.personality = 0,
	5795	.splice_fd_in = 0,
	5796	.addr3 = 0,
	5797	.resv = 0,
	5798	};
	5799	ev.yield(null, .nothing);
	5800	const completion = cancel_region.completion();
	5801	switch (completion.errno()) {
	5802	.SUCCESS => return @as(u32, @bitCast(completion.result)),
	5803	.INTR, .CANCELED => continue,
	5804	.INVAL => \|err\| return errnoBug(err),
	5805	.FAULT => \|err\| return errnoBug(err),
	5806	.AGAIN => return error.WouldBlock,
	5807	.BADF => return error.NotOpenForWriting, // Can be a race condition.
	5808	.DESTADDRREQ => \|err\| return errnoBug(err), // `connect` was never called.
	5809	.DQUOT => return error.DiskQuota,
	5810	.FBIG => return error.FileTooBig,
	5811	.IO => return error.InputOutput,
	5812	.NOSPC => return error.NoSpaceLeft,
	5813	.PERM => return error.PermissionDenied,
	5814	.PIPE => return error.BrokenPipe,
	5815	.CONNRESET => \|err\| return errnoBug(err), // Not a socket handle.
	5816	.BUSY => return error.DeviceBusy,
	5817	else => \|err\| return unexpectedErrno(err),
	5818	}
	5819	}
	5820	}
	5821
	5822	fn readAll(
	5823	ev: *Evented,
	5824	cancel_region: *CancelRegion,
	5825	fd: fd_t,
	5826	buffer: []u8,
	5827	) (File.Reader.Error \|\| error{EndOfStream})!void {
	5828	var index: usize = 0;
	5829	while (buffer.len - index != 0) {
	5830	const len = try ev.preadv(cancel_region, fd, &.{
	5831	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
	5832	}, null);
	5833	if (len == 0) return error.EndOfStream;
	5834	index += len;
	5835	}
	5836	}
	5837
	5838	fn realPath(
	5839	ev: *Evented,
	5840	cancel_region: *CancelRegion,
	5841	fd: fd_t,
	5842	out_buffer: []u8,
	5843	) File.RealPathError!usize {
	5844	_ = ev;
	5845	var procfs_buf: [std.fmt.count("/proc/self/fd/{d}\x00", .{std.math.minInt(fd_t)})]u8 = undefined;
	5846	const proc_path = std.fmt.bufPrintSentinel(&procfs_buf, "/proc/self/fd/{d}", .{fd}, 0) catch
	5847	unreachable;
	5848	while (true) {
	5849	try cancel_region.await(.nothing);
	5850	const rc = linux.readlink(proc_path, out_buffer.ptr, out_buffer.len);
	5851	switch (linux.errno(rc)) {
	5852	.SUCCESS => return rc,
	5853	.INTR => continue,
	5854	.ACCES => return error.AccessDenied,
	5855	.FAULT => \|err\| return errnoBug(err),
	5856	.IO => return error.FileSystem,
	5857	.LOOP => return error.SymLinkLoop,
	5858	.NAMETOOLONG => return error.NameTooLong,
	5859	.NOENT => return error.FileNotFound,
	5860	.NOMEM => return error.SystemResources,
	5861	.NOTDIR => return error.NotDir,
	5862	.ILSEQ => \|err\| return errnoBug(err),
	5863	else => \|err\| return unexpectedErrno(err),
	5864	}
	5865	}
	5866	}
	5867
	5868	fn renameat(
	5869	ev: *Evented,
	5870	cancel_region: *CancelRegion,
	5871	old_dir: fd_t,
	5872	old_path: [*:0]const u8,
	5873	new_dir: fd_t,
	5874	new_path: [*:0]const u8,
	5875	flags: linux.RENAME,
	5876	) Dir.RenameError!void {
	5877	while (true) {
	5878	const thread = try cancel_region.awaitIoUring();
	5879	thread.enqueue().* = .{
	5880	.opcode = .RENAMEAT,
	5881	.flags = 0,
	5882	.ioprio = 0,
	5883	.fd = old_dir,
	5884	.off = @intFromPtr(new_path),
	5885	.addr = @intFromPtr(old_path),
	5886	.len = @bitCast(new_dir),
	5887	.rw_flags = @bitCast(flags),
	5888	.user_data = @intFromPtr(cancel_region.fiber),
	5889	.buf_index = 0,
	5890	.personality = 0,
	5891	.splice_fd_in = 0,
	5892	.addr3 = 0,
	5893	.resv = 0,
	5894	};
	5895	ev.yield(null, .nothing);
	5896	switch (cancel_region.errno()) {
	5897	.SUCCESS => return,
	5898	.INTR, .CANCELED => continue,
	5899	.ACCES => return error.AccessDenied,
	5900	.PERM => return error.PermissionDenied,
	5901	.BUSY => return error.FileBusy,
	5902	.DQUOT => return error.DiskQuota,
	5903	.ISDIR => return error.IsDir,
	5904	.IO => return error.HardwareFailure,
	5905	.LOOP => return error.SymLinkLoop,
	5906	.MLINK => return error.LinkQuotaExceeded,
	5907	.NAMETOOLONG => return error.NameTooLong,
	5908	.NOENT => return error.FileNotFound,
	5909	.NOTDIR => return error.NotDir,
	5910	.NOMEM => return error.SystemResources,
	5911	.NOSPC => return error.NoSpaceLeft,
	5912	.EXIST => return error.DirNotEmpty,
	5913	.NOTEMPTY => return error.DirNotEmpty,
	5914	.ROFS => return error.ReadOnlyFileSystem,
	5915	.XDEV => return error.CrossDevice,
	5916	.ILSEQ => return error.BadPathName,
	5917	.FAULT => \|err\| return errnoBug(err),
	5918	.INVAL => \|err\| return errnoBug(err),
	5919	else => \|err\| return unexpectedErrno(err),
	5920	}
	5921	}
	5922	}
	5923
	5924	fn setsockopt(
	5925	ev: *Evented,
	5926	cancel_region: *CancelRegion,
	5927	fd: fd_t,
	5928	level: i32,
	5929	opt_name: u32,
	5930	option: u32,
	5931	) !void {
	5932	const o: []const u8 = @ptrCast(&option);
	5933	while (true) {
	5934	const off: extern struct {
	5935	cmd_op: linux.IO_URING_SOCKET_OP,
	5936	pad: u32,
	5937	} align(@alignOf(u64)) = .{
	5938	.cmd_op = .SETSOCKOPT,
	5939	.pad = 0,
	5940	};
	5941	const addr: extern struct { level: i32, opt_name: u32 } align(@alignOf(u64)) = .{
	5942	.level = level,
	5943	.opt_name = opt_name,
	5944	};
	5945	const thread = try cancel_region.awaitIoUring();
	5946	thread.enqueue().* = .{
	5947	.opcode = .URING_CMD,
	5948	.flags = 0,
	5949	.ioprio = 0,
	5950	.fd = fd,
	5951	.off = @as(const u64, @ptrCast(&off)).,
	5952	.addr = @as(const u64, @ptrCast(&addr)).,
	5953	.len = 0,
	5954	.rw_flags = 0,
	5955	.user_data = @intFromPtr(cancel_region.fiber),
	5956	.buf_index = 0,
	5957	.personality = 0,
	5958	.splice_fd_in = @intCast(o.len),
	5959	.addr3 = @intFromPtr(o.ptr),
	5960	.resv = 0,
	5961	};
	5962	ev.yield(null, .nothing);
	5963	switch (cancel_region.errno()) {
	5964	.SUCCESS => return,
	5965	.INTR, .CANCELED => continue,
	5966	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	5967	.NOTSOCK => \|err\| return errnoBug(err),
	5968	.INVAL => \|err\| return errnoBug(err),
	5969	.FAULT => \|err\| return errnoBug(err),
	5970	else => \|err\| return unexpectedErrno(err),
	5971	}
	5972	}
	5973	}
	5974
	5975	fn socket(
	5976	ev: *Evented,
	5977	cancel_region: *CancelRegion,
	5978	family: linux.sa_family_t,
	5979	options: net.IpAddress.BindOptions,
	5980	) error{
	5981	AddressFamilyUnsupported,
	5982	ProtocolUnsupportedBySystem,
	5983	ProcessFdQuotaExceeded,
	5984	SystemFdQuotaExceeded,
	5985	SystemResources,
	5986	ProtocolUnsupportedByAddressFamily,
	5987	SocketModeUnsupported,
	5988	OptionUnsupported,
	5989	Unexpected,
	5990	Canceled,
	5991	}!fd_t {
	5992	const mode = posixSocketMode(options.mode);
	5993	const protocol = posixProtocol(options.protocol);
	5994	const socket_fd = while (true) {
	5995	const thread = try cancel_region.awaitIoUring();
	5996	thread.enqueue().* = .{
	5997	.opcode = .SOCKET,
	5998	.flags = 0,
	5999	.ioprio = 0,
	6000	.fd = family,
	6001	.off = mode \| linux.SOCK.CLOEXEC,
	6002	.addr = 0,
	6003	.len = protocol,
	6004	.rw_flags = 0,
	6005	.user_data = @intFromPtr(cancel_region.fiber),
	6006	.buf_index = 0,
	6007	.personality = 0,
	6008	.splice_fd_in = 0,
	6009	.addr3 = 0,
	6010	.resv = 0,
	6011	};
	6012	ev.yield(null, .nothing);
	6013	const completion = cancel_region.completion();
	6014	switch (completion.errno()) {
	6015	.SUCCESS => break completion.result,
	6016	.INTR, .CANCELED => continue,
	6017	.AFNOSUPPORT => return error.AddressFamilyUnsupported,
	6018	.INVAL => return error.ProtocolUnsupportedBySystem,
	6019	.MFILE => return error.ProcessFdQuotaExceeded,
	6020	.NFILE => return error.SystemFdQuotaExceeded,
	6021	.NOBUFS => return error.SystemResources,
	6022	.NOMEM => return error.SystemResources,
	6023	.PROTONOSUPPORT => return error.ProtocolUnsupportedByAddressFamily,
	6024	.PROTOTYPE => return error.SocketModeUnsupported,
	6025	else => \|err\| return unexpectedErrno(err),
	6026	}
	6027	};
	6028	errdefer ev.close(socket_fd);
	6029
	6030	if (options.ip6_only) {
	6031	if (linux.IPV6 == void) return error.OptionUnsupported;
	6032	try ev.setsockopt(cancel_region, socket_fd, linux.IPPROTO.IPV6, linux.IPV6.V6ONLY, 0);
	6033	}
	6034
	6035	return socket_fd;
	6036	}
	6037
	6038	fn stat(ev: Evented, cancel_region: CancelRegion, fd: fd_t) Dir.StatError!Dir.Stat {
	6039	return ev.statx(cancel_region, fd, "", linux.AT.EMPTY_PATH) catch \|err\| switch (err) {
	6040	error.BadPathName, error.NameTooLong => unreachable, // path is empty
	6041	error.AccessDenied => return errnoBug(.ACCES),
	6042	error.SymLinkLoop => return errnoBug(.LOOP),
	6043	error.FileNotFound => return errnoBug(.NOENT),
	6044	error.NotDir => return errnoBug(.NOTDIR),
	6045	else => \|e\| return e,
	6046	};
	6047	}
	6048
	6049	fn statx(
	6050	ev: *Evented,
	6051	cancel_region: *CancelRegion,
	6052	dir: fd_t,
	6053	path: [*:0]const u8,
	6054	flags: u32,
	6055	) (Dir.StatError \|\| Dir.PathNameError \|\| error{ FileNotFound, NotDir, SymLinkLoop })!Dir.Stat {
	6056	while (true) {
	6057	var statx_buf = std.mem.zeroes(linux.Statx);
	6058	const thread = try cancel_region.awaitIoUring();
	6059	thread.enqueue().* = .{
	6060	.opcode = .STATX,
	6061	.flags = 0,
	6062	.ioprio = 0,
	6063	.fd = dir,
	6064	.off = @intFromPtr(&statx_buf),
	6065	.addr = @intFromPtr(path),
	6066	.len = @bitCast(linux_statx_request),
	6067	.rw_flags = flags,
	6068	.user_data = @intFromPtr(cancel_region.fiber),
	6069	.buf_index = 0,
	6070	.personality = 0,
	6071	.splice_fd_in = 0,
	6072	.addr3 = 0,
	6073	.resv = 0,
	6074	};
	6075	ev.yield(null, .nothing);
	6076	switch (cancel_region.errno()) {
	6077	.SUCCESS => return statFromLinux(&statx_buf),
	6078	.INTR, .CANCELED => continue,
	6079	.ACCES => return error.AccessDenied,
	6080	.BADF => \|err\| return errnoBug(err), // File descriptor used after closed.
	6081	.FAULT => \|err\| return errnoBug(err),
	6082	.INVAL => \|err\| return errnoBug(err),
	6083	.LOOP => return error.SymLinkLoop,
	6084	.NAMETOOLONG => \|err\| return errnoBug(err),
	6085	.NOENT => return error.FileNotFound,
	6086	.NOTDIR => return error.NotDir,
	6087	.NOMEM => return error.SystemResources,
	6088	else => \|err\| return unexpectedErrno(err),
	6089	}
	6090	}
	6091	}
	6092
	6093	fn urandomReadAll(
	6094	ev: *Evented,
	6095	cancel_region: *CancelRegion,
	6096	buffer: []u8,
	6097	) (File.OpenError \|\| File.Reader.Error \|\| error{EndOfStream})!void {
	6098	return ev.readAll(cancel_region, try ev.random_fd.open(ev, cancel_region, "/dev/urandom", .{
	6099	.ACCMODE = .RDONLY,
	6100	.CLOEXEC = true,
	6101	}), buffer);
	6102	}
	6103
	6104	fn utimensat(
	6105	ev: *Evented,
	6106	cancel_region: *CancelRegion,
	6107	dir: fd_t,
	6108	path: [*:0]const u8,
	6109	times: ?*const [2]linux.timespec,
	6110	flags: u32,
	6111	) File.SetTimestampsError!void {
	6112	_ = ev;
	6113	while (true) {
	6114	try cancel_region.await(.nothing);
	6115	switch (linux.errno(linux.utimensat(dir, path, times, flags))) {
	6116	.SUCCESS => return,
	6117	.INTR => continue,
	6118	.BADF => \|err\| return errnoBug(err), // always a race condition
	6119	.FAULT => \|err\| return errnoBug(err),
	6120	.INVAL => \|err\| return errnoBug(err),
	6121	.ACCES => return error.AccessDenied,
	6122	.PERM => return error.PermissionDenied,
	6123	.ROFS => return error.ReadOnlyFileSystem,
	6124	else => \|err\| return unexpectedErrno(err),
	6125	}
	6126	}
	6127	}
	6128
	6129	fn writeAll(
	6130	ev: *Evented,
	6131	cancel_region: *CancelRegion,
	6132	fd: fd_t,
	6133	buffer: []const u8,
	6134	) (File.Writer.Error \|\| error{EndOfStream})!void {
	6135	var index: usize = 0;
	6136	while (buffer.len - index != 0) {
	6137	const len = try ev.pwritev(cancel_region, fd, &.{
	6138	.{ .base = buffer[index..].ptr, .len = buffer.len - index },
	6139	}, null);
	6140	if (len == 0) return error.EndOfStream;
	6141	index += len;
	6142	}
	6143	}
	6144
	6145	test {
	6146	_ = Fiber.CancelProtection;
1497	6147	}

lib/std/Io/Threaded.zig+221-229

...	...	@@ -78,7 +78,7 @@ null_file: NullFile = .{},
78	78	random_file: RandomFile = .{},
79	79	pipe_file: PipeFile = .{},
80	80
81		csprng: Csprng = .{},
	81	csprng: Csprng = .uninitialized,
82	82
83	83	system_basic_information: SystemBasicInformation = .{},
84	84
...	...	@@ -88,10 +88,12 @@ const SystemBasicInformation = if (!is_windows) struct {} else struct {
88	88	};
89	89
90	90	pub const Csprng = struct {
91		rng: std.Random.DefaultCsprng = .{
	91	rng: std.Random.DefaultCsprng,
	92
	93	pub const uninitialized: Csprng = .{ .rng = .{
92	94	.state = undefined,
93	95	.offset = std.math.maxInt(usize),
94		},
	96	} };
95	97
96	98	pub const seed_len = std.Random.DefaultCsprng.secret_seed_length;
97	99
...	...	@@ -120,7 +122,7 @@ pub const Argv0 = switch (native_os) {
120	122	},
121	123	};
122	124
123		const Environ = struct {
	125	pub const Environ = struct {
124	126	/// Unmodified data directly from the OS.
125	127	process_environ: process.Environ,
126	128	/// Protected by `mutex`. Determines whether the other fields have been
...	...	@@ -157,6 +159,127 @@ const Environ = struct {
157	159	HOME: ?[:0]const u8 = null,
158	160	},
159	161	};
	162
	163	pub fn scan(environ: *Environ, allocator: std.mem.Allocator) void {
	164	if (environ.initialized) return;
	165	environ.initialized = true;
	166
	167	if (is_windows) {
	168	// This value expires with any call that modifies the environment,
	169	// which is outside of this Io implementation's control, so references
	170	// must be short-lived.
	171	const peb = windows.peb();
	172	assert(windows.ntdll.RtlEnterCriticalSection(peb.FastPebLock) == .SUCCESS);
	173	defer assert(windows.ntdll.RtlLeaveCriticalSection(peb.FastPebLock) == .SUCCESS);
	174	const ptr = peb.ProcessParameters.Environment;
	175
	176	var i: usize = 0;
	177	while (ptr[i] != 0) {
	178	// There are some special environment variables that start with =,
	179	// so we need a special case to not treat = as a key/value separator
	180	// if it's the first character.
	181	// https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
	182	const key_start = i;
	183	if (ptr[i] == '=') i += 1;
	184	while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
	185	const key_w = ptr[key_start..i];
	186
	187	const value_start = i + 1;
	188	while (ptr[i] != 0) : (i += 1) {} // skip over '=' and value
	189	const value_w = ptr[value_start..i];
	190	i += 1; // skip over null byte
	191
	192	if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'N', 'O', '_', 'C', 'O', 'L', 'O', 'R' })) {
	193	environ.exist.NO_COLOR = true;
	194	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'C', 'L', 'I', 'C', 'O', 'L', 'O', 'R', '_', 'F', 'O', 'R', 'C', 'E' })) {
	195	environ.exist.CLICOLOR_FORCE = true;
	196	} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'Z', 'I', 'G', '_', 'P', 'R', 'O', 'G', 'R', 'E', 'S', 'S' })) {
	197	environ.zig_progress_file = file: {
	198	var value_buf: [std.fmt.count("{d}", .{std.math.maxInt(usize)})]u8 = undefined;
	199	const len = std.unicode.calcWtf8Len(value_w);
	200	if (len > value_buf.len) break :file error.UnrecognizedFormat;
	201	assert(std.unicode.wtf16LeToWtf8(&value_buf, value_w) == len);
	202	break :file .{
	203	.handle = @ptrFromInt(std.fmt.parseInt(usize, value_buf[0..len], 10) catch
	204	break :file error.UnrecognizedFormat),
	205	.flags = .{ .nonblocking = true },
	206	};
	207	};
	208	}
	209	comptime assert(@sizeOf(String) == 0);
	210	}
	211	} else if (native_os == .wasi and !builtin.link_libc) {
	212	var environ_size: usize = undefined;
	213	var environ_buf_size: usize = undefined;
	214
	215	switch (std.os.wasi.environ_sizes_get(&environ_size, &environ_buf_size)) {
	216	.SUCCESS => {},
	217	else => \|err\| {
	218	environ.err = posix.unexpectedErrno(err);
	219	return;
	220	},
	221	}
	222	if (environ_size == 0) return;
	223
	224	const wasi_environ = allocator.alloc([*:0]u8, environ_size) catch \|err\| {
	225	environ.err = err;
	226	return;
	227	};
	228	defer allocator.free(wasi_environ);
	229	const wasi_environ_buf = allocator.alloc(u8, environ_buf_size) catch \|err\| {
	230	environ.err = err;
	231	return;
	232	};
	233	defer allocator.free(wasi_environ_buf);
	234
	235	switch (std.os.wasi.environ_get(wasi_environ.ptr, wasi_environ_buf.ptr)) {
	236	.SUCCESS => {},
	237	else => \|err\| {
	238	environ.err = posix.unexpectedErrno(err);
	239	return;
	240	},
	241	}
	242
	243	for (wasi_environ) \|env\| {
	244	const pair = std.mem.sliceTo(env, 0);
	245	var parts = std.mem.splitScalar(u8, pair, '=');
	246	const key = parts.first();
	247	if (std.mem.eql(u8, key, "NO_COLOR")) {
	248	environ.exist.NO_COLOR = true;
	249	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
	250	environ.exist.CLICOLOR_FORCE = true;
	251	}
	252	comptime assert(@sizeOf(String) == 0);
	253	}
	254	} else {
	255	for (environ.process_environ.block.slice) \|opt_entry\| {
	256	const entry = opt_entry.?;
	257	var entry_i: usize = 0;
	258	while (entry[entry_i] != 0 and entry[entry_i] != '=') : (entry_i += 1) {}
	259	const key = entry[0..entry_i];
	260
	261	var end_i: usize = entry_i;
	262	while (entry[end_i] != 0) : (end_i += 1) {}
	263	const value = entry[entry_i + 1 .. end_i :0];
	264
	265	if (std.mem.eql(u8, key, "NO_COLOR")) {
	266	environ.exist.NO_COLOR = true;
	267	} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
	268	environ.exist.CLICOLOR_FORCE = true;
	269	} else if (std.mem.eql(u8, key, "ZIG_PROGRESS")) {
	270	environ.zig_progress_file = file: {
	271	break :file .{
	272	.handle = std.fmt.parseInt(u31, value, 10) catch
	273	break :file error.UnrecognizedFormat,
	274	.flags = .{ .nonblocking = true },
	275	};
	276	};
	277	} else inline for (@typeInfo(String).@"struct".fields) \|field\| {
	278	if (std.mem.eql(u8, key, field.name)) @field(environ.string, field.name) = value;
	279	}
	280	}
	281	}
	282	}
160	283	};
161	284
162	285	pub const NullFile = switch (native_os) {
...	...	@@ -1397,13 +1520,13 @@ pub fn waitForApcOrAlert() void {
1397	1520	_ = windows.ntdll.NtDelayExecution(windows.TRUE, &infinite_timeout);
1398	1521	}
1399	1522
1400		const max_iovecs_len = 8;
1401		const splat_buffer_size = 64;
	1523	pub const max_iovecs_len = 8;
	1524	pub const splat_buffer_size = 64;
1402	1525	/// Happens to be the same number that matches maximum number of handles that
1403	1526	/// NtWaitForMultipleObjects accepts. We use this value also for poll() on
1404	1527	/// posix systems.
1405	1528	const poll_buffer_len = 64;
1406		const default_PATH = "/usr/local/bin:/bin/:/usr/bin";
	1529	pub const default_PATH = "/usr/local/bin:/bin/:/usr/bin";
1407	1530	/// There are multiple kernel bugs being worked around with retries.
1408	1531	const max_windows_kernel_bug_retries = 13;
1409	1532
...	...	@@ -1588,7 +1711,7 @@ fn worker(t: *Threaded) void {
1588	1711	.cancel_protection = .unblocked,
1589	1712	.futex_waiter = undefined,
1590	1713	.unpark_flag = unpark_flag_init,
1591		.csprng = .{},
	1714	.csprng = .uninitialized,
1592	1715	};
1593	1716	Thread.current = &thread;
1594	1717
...	...	@@ -2563,12 +2686,12 @@ fn operate(userdata: ?*anyopaque, operation: Io.Operation) Io.Cancelable!Io.Oper
2563	2686	fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2564	2687	const t: *Threaded = @ptrCast(@alignCast(userdata));
2565	2688	if (is_windows) {
2566		batchAwaitWindows(b, false) catch \|err\| switch (err) {
	2689	batchDrainSubmittedWindows(b, false) catch \|err\| switch (err) {
2567	2690	error.ConcurrencyUnavailable => unreachable, // passed concurrency=false
2568	2691	else => \|e\| return e,
2569	2692	};
2570	2693	const alertable_syscall = try AlertableSyscall.start();
2571		while (b.pending.head != .none and b.completions.head == .none) waitForApcOrAlert();
	2694	while (b.pending.head != .none and b.completed.head == .none) waitForApcOrAlert();
2572	2695	alertable_syscall.finish();
2573	2696	return;
2574	2697	}
...	...	@@ -2576,7 +2699,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2576	2699	var poll_buffer: [poll_buffer_len]posix.pollfd = undefined;
2577	2700	var poll_len: u32 = 0;
2578	2701	{
2579		var index = b.submissions.head;
	2702	var index = b.submitted.head;
2580	2703	while (index != .none and poll_len < poll_buffer_len) {
2581	2704	const submission = &b.storage[index.toIndex()].submission;
2582	2705	switch (submission.operation) {
...	...	@@ -2605,7 +2728,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2605	2728	1 => {},
2606	2729	else => while (true) {
2607	2730	const timeout_ms: i32 = t: {
2608		if (b.completions.head != .none) {
	2731	if (b.completed.head != .none) {
2609	2732	// It is legal to call batchWait with already completed
2610	2733	// operations in the ring. In such case, we need to avoid
2611	2734	// blocking in the poll syscall, but we can still take this
...	...	@@ -2620,7 +2743,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2620	2743	switch (posix.errno(rc)) {
2621	2744	.SUCCESS => {
2622	2745	if (rc == 0) {
2623		if (b.completions.head != .none) {
	2746	if (b.completed.head != .none) {
2624	2747	// Since there are already completions available in the
2625	2748	// queue, this is neither a timeout nor a case for
2626	2749	// retrying.
...	...	@@ -2629,7 +2752,7 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2629	2752	continue;
2630	2753	}
2631	2754	var prev_index: Io.Operation.OptionalIndex = .none;
2632		var index = b.submissions.head;
	2755	var index = b.submitted.head;
2633	2756	for (poll_buffer[0..poll_len]) \|poll_entry\| {
2634	2757	const storage = &b.storage[index.toIndex()];
2635	2758	const submission = &storage.submission;
...	...	@@ -2638,17 +2761,17 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2638	2761	const result = try operate(t, submission.operation);
2639	2762
2640	2763	switch (prev_index) {
2641		.none => b.submissions.head = next_index,
	2764	.none => b.submitted.head = next_index,
2642	2765	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2643	2766	}
2644		if (next_index == .none) b.submissions.tail = prev_index;
	2767	if (next_index == .none) b.submitted.tail = prev_index;
2645	2768
2646		switch (b.completions.tail) {
2647		.none => b.completions.head = index,
	2769	switch (b.completed.tail) {
	2770	.none => b.completed.head = index,
2648	2771	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2649	2772	}
2650	2773	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2651		b.completions.tail = index;
	2774	b.completed.tail = index;
2652	2775	} else prev_index = index;
2653	2776	index = next_index;
2654	2777	}
...	...	@@ -2662,10 +2785,10 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2662	2785	}
2663	2786	}
2664	2787
2665		var tail_index = b.completions.tail;
2666		defer b.completions.tail = tail_index;
2667		var index = b.submissions.head;
2668		errdefer b.submissions.head = index;
	2788	var tail_index = b.completed.tail;
	2789	defer b.completed.tail = tail_index;
	2790	var index = b.submitted.head;
	2791	errdefer b.submitted.head = index;
2669	2792	while (index != .none) {
2670	2793	const storage = &b.storage[index.toIndex()];
2671	2794	const submission = &storage.submission;
...	...	@@ -2673,22 +2796,22 @@ fn batchAwaitAsync(userdata: ?anyopaque, b: Io.Batch) Io.Cancelable!void {
2673	2796	const result = try operate(t, submission.operation);
2674	2797
2675	2798	switch (tail_index) {
2676		.none => b.completions.head = index,
	2799	.none => b.completed.head = index,
2677	2800	else => b.storage[tail_index.toIndex()].completion.node.next = index,
2678	2801	}
2679	2802	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2680	2803	tail_index = index;
2681	2804	index = next_index;
2682	2805	}
2683		b.submissions = .{ .head = .none, .tail = .none };
	2806	b.submitted = .{ .head = .none, .tail = .none };
2684	2807	}
2685	2808
2686	2809	fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout) Io.Batch.AwaitConcurrentError!void {
2687	2810	const t: *Threaded = @ptrCast(@alignCast(userdata));
2688	2811	if (is_windows) {
2689	2812	const deadline: ?Io.Clock.Timestamp = timeout.toTimestamp(ioBasic(t));
2690		try batchAwaitWindows(b, true);
2691		while (b.pending.head != .none and b.completions.head == .none) {
	2813	try batchDrainSubmittedWindows(b, true);
	2814	while (b.pending.head != .none and b.completed.head == .none) {
2692	2815	var delay_interval: windows.LARGE_INTEGER = interval: {
2693	2816	const d = deadline orelse break :interval std.math.minInt(windows.LARGE_INTEGER);
2694	2817	break :interval timeoutToWindowsInterval(.{ .deadline = d }).?;
...	...	@@ -2701,7 +2824,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2701	2824	// The thread woke due to the timeout. Although spurious
2702	2825	// timeouts are OK, when no deadline is passed we must not
2703	2826	// return `error.Timeout`.
2704		if (timeout != .none and b.completions.head == .none) return error.Timeout;
	2827	if (timeout != .none and b.completed.head == .none) return error.Timeout;
2705	2828	},
2706	2829	else => {},
2707	2830	}
...	...	@@ -2743,7 +2866,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2743	2866	}
2744	2867	} = .{ .gpa = t.allocator, .b = b, .slice = &poll_buffer, .len = 0 };
2745	2868	{
2746		var index = b.submissions.head;
	2869	var index = b.submitted.head;
2747	2870	while (index != .none) {
2748	2871	const submission = &b.storage[index.toIndex()].submission;
2749	2872	switch (submission.operation) {
...	...	@@ -2757,18 +2880,18 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2757	2880	switch (poll_storage.len) {
2758	2881	0 => return,
2759	2882	1 => if (timeout == .none) {
2760		const index = b.submissions.head;
	2883	const index = b.submitted.head;
2761	2884	const storage = &b.storage[index.toIndex()];
2762	2885	const result = try operate(t, storage.submission.operation);
2763	2886
2764		b.submissions = .{ .head = .none, .tail = .none };
	2887	b.submitted = .{ .head = .none, .tail = .none };
2765	2888
2766		switch (b.completions.tail) {
2767		.none => b.completions.head = index,
	2889	switch (b.completed.tail) {
	2890	.none => b.completed.head = index,
2768	2891	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2769	2892	}
2770	2893	storage.* = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
2771		b.completions.tail = index;
	2894	b.completed.tail = index;
2772	2895	return;
2773	2896	},
2774	2897	else => {},
...	...	@@ -2777,7 +2900,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2777	2900	const deadline = timeout.toTimestamp(t_io);
2778	2901	while (true) {
2779	2902	const timeout_ms: i32 = t: {
2780		if (b.completions.head != .none) {
	2903	if (b.completed.head != .none) {
2781	2904	// It is legal to call batchWait with already completed
2782	2905	// operations in the ring. In such case, we need to avoid
2783	2906	// blocking in the poll syscall, but we can still take this
...	...	@@ -2794,7 +2917,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2794	2917	switch (posix.errno(rc)) {
2795	2918	.SUCCESS => {
2796	2919	if (rc == 0) {
2797		if (b.completions.head != .none) {
	2920	if (b.completed.head != .none) {
2798	2921	// Since there are already completions available in the
2799	2922	// queue, this is neither a timeout nor a case for
2800	2923	// retrying.
...	...	@@ -2806,7 +2929,7 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2806	2929	return error.Timeout;
2807	2930	}
2808	2931	var prev_index: Io.Operation.OptionalIndex = .none;
2809		var index = b.submissions.head;
	2932	var index = b.submitted.head;
2810	2933	for (poll_storage.slice[0..poll_storage.len]) \|poll_entry\| {
2811	2934	const submission = &b.storage[index.toIndex()].submission;
2812	2935	const next_index = submission.node.next;
...	...	@@ -2814,17 +2937,20 @@ fn batchAwaitConcurrent(userdata: ?anyopaque, b: Io.Batch, timeout: Io.Timeout
2814	2937	const result = try operate(t, submission.operation);
2815	2938
2816	2939	switch (prev_index) {
2817		.none => b.submissions.head = next_index,
	2940	.none => b.submitted.head = next_index,
2818	2941	else => b.storage[prev_index.toIndex()].submission.node.next = next_index,
2819	2942	}
2820		if (next_index == .none) b.submissions.tail = prev_index;
	2943	if (next_index == .none) b.submitted.tail = prev_index;
2821	2944
2822		switch (b.completions.tail) {
2823		.none => b.completions.head = index,
	2945	switch (b.completed.tail) {
	2946	.none => b.completed.head = index,
2824	2947	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = index,
2825	2948	}
2826		b.completions.tail = index;
2827		b.storage[index.toIndex()] = .{ .completion = .{ .node = .{ .next = .none }, .result = result } };
	2949	b.completed.tail = index;
	2950	b.storage[index.toIndex()] = .{ .completion = .{
	2951	.node = .{ .next = .none },
	2952	.result = result,
	2953	} };
2828	2954	} else prev_index = index;
2829	2955	index = next_index;
2830	2956	}
...	...	@@ -2841,7 +2967,7 @@ const WindowsBatchPendingOperationContext = extern struct {
2841	2967	file: windows.HANDLE,
2842	2968	iosb: windows.IO_STATUS_BLOCK,
2843	2969
2844		const Erased = [3]usize;
	2970	const Erased = Io.Operation.Storage.Pending.Context;
2845	2971
2846	2972	comptime {
2847	2973	assert(@sizeOf(Erased) <= @sizeOf(WindowsBatchPendingOperationContext));
...	...	@@ -2858,24 +2984,9 @@ const WindowsBatchPendingOperationContext = extern struct {
2858	2984
2859	2985	fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2860	2986	const t: *Threaded = @ptrCast(@alignCast(userdata));
2861		{
2862		var tail_index = b.unused.tail;
2863		defer b.unused.tail = tail_index;
2864		var index = b.submissions.head;
2865		errdefer b.submissions.head = index;
2866		while (index != .none) {
2867		const next_index = b.storage[index.toIndex()].submission.node.next;
2868		switch (tail_index) {
2869		.none => b.unused.head = index,
2870		else => b.storage[tail_index.toIndex()].unused.next = index,
2871		}
2872		b.storage[index.toIndex()] = .{ .unused = .{ .prev = tail_index, .next = .none } };
2873		tail_index = index;
2874		index = next_index;
2875		}
2876		b.submissions = .{ .head = .none, .tail = .none };
2877		}
2878	2987	if (is_windows) {
	2988	if (b.pending.head == .none) return;
	2989	waitForApcOrAlert();
2879	2990	var index = b.pending.head;
2880	2991	while (index != .none) {
2881	2992	const pending = &b.storage[index.toIndex()].pending;
...	...	@@ -2889,10 +3000,13 @@ fn batchCancel(userdata: ?anyopaque, b: Io.Batch) void {
2889	3000	t.allocator.free(@as([*]posix.pollfd, @ptrCast(@alignCast(context)))[0..b.storage.len]);
2890	3001	b.context = null;
2891	3002	}
2892		assert(b.pending.head == .none);
2893	3003	}
2894	3004
2895		fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows.ULONG) callconv(.winapi) void {
	3005	fn batchApc(
	3006	apc_context: ?*anyopaque,
	3007	iosb: *windows.IO_STATUS_BLOCK,
	3008	_: windows.ULONG,
	3009	) callconv(.winapi) void {
2896	3010	const b: *Io.Batch = @ptrCast(@alignCast(apc_context));
2897	3011	const context: *WindowsBatchPendingOperationContext = @fieldParentPtr("iosb", iosb);
2898	3012	const erased_context = context.toErased();
...	...	@@ -2918,11 +3032,12 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows
2918	3032	b.unused.tail = .fromIndex(index);
2919	3033	},
2920	3034	else => {
2921		switch (b.completions.tail) {
2922		.none => b.completions.head = .fromIndex(index),
2923		else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next = .fromIndex(index),
	3035	switch (b.completed.tail) {
	3036	.none => b.completed.head = .fromIndex(index),
	3037	else => \|tail_index\| b.storage[tail_index.toIndex()].completion.node.next =
	3038	.fromIndex(index),
2924	3039	}
2925		b.completions.tail = .fromIndex(index);
	3040	b.completed.tail = .fromIndex(index);
2926	3041	const result: Io.Operation.Result = switch (pending.tag) {
2927	3042	.file_read_streaming => .{ .file_read_streaming = ntReadFileResult(iosb) },
2928	3043	.file_write_streaming => .{ .file_write_streaming = ntWriteFileResult(iosb) },
...	...	@@ -2934,9 +3049,9 @@ fn batchApc(apc_context: ?anyopaque, iosb: windows.IO_STATUS_BLOCK, _: windows
2934	3049	}
2935	3050
2936	3051	/// If `concurrency` is false, `error.ConcurrencyUnavailable` is unreachable.
2937		fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, ConcurrencyUnavailable }!void {
2938		var index = b.submissions.head;
2939		errdefer b.submissions.head = index;
	3052	fn batchDrainSubmittedWindows(b: *Io.Batch, concurrency: bool) (Io.ConcurrentError \|\| Io.Cancelable)!void {
	3053	var index = b.submitted.head;
	3054	errdefer b.submitted.head = index;
2940	3055	while (index != .none) {
2941	3056	const storage = &b.storage[index.toIndex()];
2942	3057	const submission = storage.submission;
...	...	@@ -2952,7 +3067,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
2952	3067	b.pending.tail = index;
2953	3068	const context: *WindowsBatchPendingOperationContext = .fromErased(&storage.pending.context);
2954	3069	errdefer {
2955		context.iosb.u.Status = .CANCELLED;
	3070	context.iosb = .{ .u = .{ .Status = .CANCELLED }, .Information = undefined };
2956	3071	batchApc(b, &context.iosb, 0);
2957	3072	}
2958	3073	switch (submission.operation) {
...	...	@@ -2960,10 +3075,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
2960	3075	var data_index: usize = 0;
2961	3076	while (o.data.len - data_index != 0 and o.data[data_index].len == 0) data_index += 1;
2962	3077	if (o.data.len - data_index == 0) {
2963		context.iosb = .{
2964		.u = .{ .Status = .SUCCESS },
2965		.Information = 0,
2966		};
	3078	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
2967	3079	batchApc(b, &context.iosb, 0);
2968	3080	break :o;
2969	3081	}
...	...	@@ -3023,10 +3135,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
3023	3135	.file_write_streaming => \|o\| o: {
3024	3136	const buffer = windowsWriteBuffer(o.header, o.data, o.splat);
3025	3137	if (buffer.len == 0) {
3026		context.iosb = .{
3027		.u = .{ .Status = .SUCCESS },
3028		.Information = 0,
3029		};
	3138	context.iosb = .{ .u = .{ .Status = .SUCCESS }, .Information = 0 };
3030	3139	batchApc(b, &context.iosb, 0);
3031	3140	break :o;
3032	3141	}
...	...	@@ -3140,7 +3249,7 @@ fn batchAwaitWindows(b: *Io.Batch, concurrency: bool) error{ Canceled, Concurren
3140	3249	}
3141	3250	index = submission.node.next;
3142	3251	}
3143		b.submissions = .{ .head = .none, .tail = .none };
	3252	b.submitted = .{ .head = .none, .tail = .none };
3144	3253	}
3145	3254
3146	3255	/// Since Windows only supports writing one contiguous buffer, returns the
...	...	@@ -3155,7 +3264,7 @@ fn windowsWriteBuffer(header: []const u8, data: []const []const u8, splat: usize
3155	3264	if (splat == 0) return &.{};
3156	3265	break :b data[data.len - 1];
3157	3266	};
3158		return buffer[0..@min(buffer.len, std.math.maxInt(u32))];
	3267	return buffer[0..std.math.lossyCast(u32, buffer.len)];
3159	3268	}
3160	3269
3161	3270	fn submitComplete(ring: []u32, complete_tail: *Io.Batch.RingIndex, op: u32) void {
...	...	@@ -4677,8 +4786,8 @@ fn atomicFileInit(
4677	4786	dir: Dir,
4678	4787	close_dir_on_deinit: bool,
4679	4788	) Dir.CreateFileAtomicError!File.Atomic {
4680		var random_integer: u64 = undefined;
4681	4789	while (true) {
	4790	var random_integer: u64 = undefined;
4682	4791	t_io.random(@ptrCast(&random_integer));
4683	4792	const tmp_sub_path = std.fmt.hex(random_integer);
4684	4793	const file = dir.createFile(t_io, &tmp_sub_path, .{
...	...	@@ -14317,11 +14426,11 @@ pub fn posixProtocol(protocol: ?net.Protocol) u32 {
14317	14426	return @intFromEnum(protocol orelse return 0);
14318	14427	}
14319	14428
14320		fn recoverableOsBugDetected() void {
	14429	pub fn recoverableOsBugDetected() void {
14321	14430	if (is_debug) unreachable;
14322	14431	}
14323	14432
14324		fn clockToPosix(clock: Io.Clock) posix.clockid_t {
	14433	pub fn clockToPosix(clock: Io.Clock) posix.clockid_t {
14325	14434	return switch (clock) {
14326	14435	.real => posix.CLOCK.REALTIME,
14327	14436	.awake => switch (native_os) {
...	...	@@ -14355,7 +14464,7 @@ fn clockToWasi(clock: Io.Clock) std.os.wasi.clockid_t {
14355	14464	};
14356	14465	}
14357	14466
14358		const linux_statx_request: std.os.linux.STATX = .{
	14467	pub const linux_statx_request: std.os.linux.STATX = .{
14359	14468	.TYPE = true,
14360	14469	.MODE = true,
14361	14470	.ATIME = true,
...	...	@@ -14367,7 +14476,7 @@ const linux_statx_request: std.os.linux.STATX = .{
14367	14476	.BLOCKS = true,
14368	14477	};
14369	14478
14370		const linux_statx_check: std.os.linux.STATX = .{
	14479	pub const linux_statx_check: std.os.linux.STATX = .{
14371	14480	.TYPE = true,
14372	14481	.MODE = true,
14373	14482	.ATIME = false,
...	...	@@ -14379,7 +14488,7 @@ const linux_statx_check: std.os.linux.STATX = .{
14379	14488	.BLOCKS = false,
14380	14489	};
14381	14490
14382		fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat {
	14491	pub fn statFromLinux(stx: *const std.os.linux.Statx) Io.UnexpectedError!File.Stat {
14383	14492	const actual_mask_int: u32 = @bitCast(stx.mask);
14384	14493	const wanted_mask_int: u32 = @bitCast(linux_statx_check);
14385	14494	if ((actual_mask_int \| wanted_mask_int) != actual_mask_int) return error.Unexpected;
...	...	@@ -14470,11 +14579,11 @@ fn statFromWasi(st: *const std.os.wasi.filestat_t) File.Stat {
14470	14579	};
14471	14580	}
14472	14581
14473		fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {
	14582	pub fn timestampFromPosix(timespec: *const posix.timespec) Io.Timestamp {
14474	14583	return .{ .nanoseconds = nanosecondsFromPosix(timespec) };
14475	14584	}
14476	14585
14477		fn nanosecondsFromPosix(timespec: *const posix.timespec) i96 {
	14586	pub fn nanosecondsFromPosix(timespec: *const posix.timespec) i96 {
14478	14587	return @intCast(@as(i128, timespec.sec) * std.time.ns_per_s + timespec.nsec);
14479	14588	}
14480	14589
...	...	@@ -14492,7 +14601,7 @@ fn timestampToPosix(nanoseconds: i96) posix.timespec {
14492	14601	};
14493	14602	}
14494	14603
14495		fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
	14604	pub fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
14496	14605	return switch (set_ts) {
14497	14606	.unchanged => .OMIT,
14498	14607	.now => .NOW,
...	...	@@ -14500,7 +14609,7 @@ fn setTimestampToPosix(set_ts: File.SetTimestamp) posix.timespec {
14500	14609	};
14501	14610	}
14502	14611
14503		fn pathToPosix(file_path: []const u8, buffer: *[posix.PATH_MAX]u8) Dir.PathNameError![:0]u8 {
	14612	pub fn pathToPosix(file_path: []const u8, buffer: *[posix.PATH_MAX]u8) Dir.PathNameError![:0]u8 {
14504	14613	if (std.mem.containsAtLeastScalar2(u8, file_path, 0, 1)) return error.BadPathName;
14505	14614	// >= rather than > to make room for the null byte
14506	14615	if (file_path.len >= buffer.len) return error.NameTooLong;
...	...	@@ -14996,126 +15105,7 @@ const WindowsEnvironStrings = struct {
14996	15105	fn scanEnviron(t: *Threaded) void {
14997	15106	mutexLock(&t.mutex);
14998	15107	defer mutexUnlock(&t.mutex);
14999
15000		if (t.environ.initialized) return;
15001		t.environ.initialized = true;
15002
15003		if (is_windows) {
15004		// This value expires with any call that modifies the environment,
15005		// which is outside of this Io implementation's control, so references
15006		// must be short-lived.
15007		const peb = windows.peb();
15008		assert(windows.ntdll.RtlEnterCriticalSection(peb.FastPebLock) == .SUCCESS);
15009		defer assert(windows.ntdll.RtlLeaveCriticalSection(peb.FastPebLock) == .SUCCESS);
15010		const ptr = peb.ProcessParameters.Environment;
15011
15012		var i: usize = 0;
15013		while (ptr[i] != 0) {
15014
15015		// There are some special environment variables that start with =,
15016		// so we need a special case to not treat = as a key/value separator
15017		// if it's the first character.
15018		// https://devblogs.microsoft.com/oldnewthing/20100506-00/?p=14133
15019		const key_start = i;
15020		if (ptr[i] == '=') i += 1;
15021		while (ptr[i] != 0 and ptr[i] != '=') : (i += 1) {}
15022		const key_w = ptr[key_start..i];
15023
15024		const value_start = i + 1;
15025		while (ptr[i] != 0) : (i += 1) {} // skip over '=' and value
15026		const value_w = ptr[value_start..i];
15027		i += 1; // skip over null byte
15028
15029		if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'N', 'O', '_', 'C', 'O', 'L', 'O', 'R' })) {
15030		t.environ.exist.NO_COLOR = true;
15031		} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'C', 'L', 'I', 'C', 'O', 'L', 'O', 'R', '_', 'F', 'O', 'R', 'C', 'E' })) {
15032		t.environ.exist.CLICOLOR_FORCE = true;
15033		} else if (windows.eqlIgnoreCaseWtf16(key_w, &.{ 'Z', 'I', 'G', '_', 'P', 'R', 'O', 'G', 'R', 'E', 'S', 'S' })) {
15034		t.environ.zig_progress_file = file: {
15035		var value_buf: [std.fmt.count("{d}", .{std.math.maxInt(usize)})]u8 = undefined;
15036		const len = std.unicode.calcWtf8Len(value_w);
15037		if (len > value_buf.len) break :file error.UnrecognizedFormat;
15038		assert(std.unicode.wtf16LeToWtf8(&value_buf, value_w) == len);
15039		break :file .{
15040		.handle = @ptrFromInt(std.fmt.parseInt(usize, value_buf[0..len], 10) catch
15041		break :file error.UnrecognizedFormat),
15042		.flags = .{ .nonblocking = true },
15043		};
15044		};
15045		}
15046		comptime assert(@sizeOf(Environ.String) == 0);
15047		}
15048		} else if (native_os == .wasi and !builtin.link_libc) {
15049		var environ_count: usize = undefined;
15050		var environ_buf_size: usize = undefined;
15051
15052		switch (std.os.wasi.environ_sizes_get(&environ_count, &environ_buf_size)) {
15053		.SUCCESS => {},
15054		else => \|err\| {
15055		t.environ.err = posix.unexpectedErrno(err);
15056		return;
15057		},
15058		}
15059		if (environ_count == 0) return;
15060
15061		const environ = t.allocator.alloc([*:0]u8, environ_count) catch \|err\| {
15062		t.environ.err = err;
15063		return;
15064		};
15065		defer t.allocator.free(environ);
15066		const environ_buf = t.allocator.alloc(u8, environ_buf_size) catch \|err\| {
15067		t.environ.err = err;
15068		return;
15069		};
15070		defer t.allocator.free(environ_buf);
15071
15072		switch (std.os.wasi.environ_get(environ.ptr, environ_buf.ptr)) {
15073		.SUCCESS => {},
15074		else => \|err\| {
15075		t.environ.err = posix.unexpectedErrno(err);
15076		return;
15077		},
15078		}
15079
15080		for (environ) \|env\| {
15081		const pair = std.mem.sliceTo(env, 0);
15082		var parts = std.mem.splitScalar(u8, pair, '=');
15083		const key = parts.first();
15084		if (std.mem.eql(u8, key, "NO_COLOR")) {
15085		t.environ.exist.NO_COLOR = true;
15086		} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
15087		t.environ.exist.CLICOLOR_FORCE = true;
15088		}
15089		comptime assert(@sizeOf(Environ.String) == 0);
15090		}
15091		} else {
15092		for (t.environ.process_environ.block.slice) \|opt_entry\| {
15093		const entry = opt_entry.?;
15094		var entry_i: usize = 0;
15095		while (entry[entry_i] != 0 and entry[entry_i] != '=') : (entry_i += 1) {}
15096		const key = entry[0..entry_i];
15097
15098		var end_i: usize = entry_i;
15099		while (entry[end_i] != 0) : (end_i += 1) {}
15100		const value = entry[entry_i + 1 .. end_i :0];
15101
15102		if (std.mem.eql(u8, key, "NO_COLOR")) {
15103		t.environ.exist.NO_COLOR = true;
15104		} else if (std.mem.eql(u8, key, "CLICOLOR_FORCE")) {
15105		t.environ.exist.CLICOLOR_FORCE = true;
15106		} else if (std.mem.eql(u8, key, "ZIG_PROGRESS")) {
15107		t.environ.zig_progress_file = file: {
15108		break :file .{
15109		.handle = std.fmt.parseInt(u31, value, 10) catch
15110		break :file error.UnrecognizedFormat,
15111		.flags = .{ .nonblocking = true },
15112		};
15113		};
15114		} else inline for (@typeInfo(Environ.String).@"struct".fields) \|field\| {
15115		if (std.mem.eql(u8, key, field.name)) @field(t.environ.string, field.name) = value;
15116		}
15117		}
15118		}
	15108	t.environ.scan(t.allocator);
15119	15109	}
15120	15110
15121	15111	fn processReplace(userdata: ?*anyopaque, options: process.ReplaceOptions) process.ReplaceError {
...	...	@@ -15213,17 +15203,17 @@ fn spawnPosix(t: *Threaded, options: process.SpawnOptions) process.SpawnError!Sp
15213	15203	const any_ignore = (options.stdin == .ignore or options.stdout == .ignore or options.stderr == .ignore);
15214	15204	const dev_null_fd = if (any_ignore) try getDevNullFd(t) else undefined;
15215	15205
15216		const prog_pipe: [2]posix.fd_t = if (options.progress_node.index != .none)
	15206	const prog_pipe: [2]posix.fd_t = if (options.progress_node.index != .none) pipe: {
15217	15207	// We use CLOEXEC for the same reason as in `pipe_flags`.
15218		try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true })
15219		else
15220		.{ -1, -1 };
	15208	const pipe = try pipe2(.{ .NONBLOCK = true, .CLOEXEC = true });
	15209	switch (native_os) {
	15210	.linux => _ = posix.system.fcntl(pipe[0], posix.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2)),
	15211	else => {},
	15212	}
	15213	break :pipe pipe;
	15214	} else .{ -1, -1 };
15221	15215	errdefer destroyPipe(prog_pipe);
15222	15216
15223		if (native_os == .linux and prog_pipe[0] != -1) {
15224		_ = posix.system.fcntl(prog_pipe[0], posix.F.SETPIPE_SZ, @as(u32, std.Progress.max_packet_len * 2));
15225		}
15226
15227	15217	var arena_allocator = std.heap.ArenaAllocator.init(t.allocator);
15228	15218	defer arena_allocator.deinit();
15229	15219	const arena = arena_allocator.allocator();
...	...	@@ -17241,16 +17231,7 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {
17241	17231
17242	17232	randomSecure(t, &seed) catch \|err\| switch (err) {
17243	17233	error.Canceled => unreachable,
17244		error.EntropyUnavailable => {
17245		@memset(&seed, 0);
17246		const aslr_addr = @intFromPtr(t);
17247		std.mem.writeInt(usize, seed[seed.len - @sizeOf(usize) ..][0..@sizeOf(usize)], aslr_addr, .native);
17248		switch (native_os) {
17249		.windows => fallbackSeedWindows(&seed),
17250		.wasi => if (builtin.link_libc) fallbackSeedPosix(&seed) else fallbackSeedWasi(&seed),
17251		else => fallbackSeedPosix(&seed),
17252		}
17253		},
	17234	error.EntropyUnavailable => fallbackSeed(t, &seed),
17254	17235	};
17255	17236	}
17256	17237	t.csprng.rng = .init(seed);
...	...	@@ -17259,6 +17240,17 @@ fn randomMainThread(t: *Threaded, buffer: []u8) void {
17259	17240	t.csprng.rng.fill(buffer);
17260	17241	}
17261	17242
	17243	pub fn fallbackSeed(aslr_addr: ?anyopaque, seed: [Csprng.seed_len]u8) void {
	17244	@memset(seed, 0);
	17245	std.mem.writeInt(usize, seed[seed.len - @sizeOf(usize) ..][0..@sizeOf(usize)], @intFromPtr(aslr_addr), .native);
	17246	const fallbackSeedImpl = switch (native_os) {
	17247	.windows => fallbackSeedWindows,
	17248	.wasi => if (builtin.link_libc) fallbackSeedPosix else fallbackSeedWasi,
	17249	else => fallbackSeedPosix,
	17250	};
	17251	fallbackSeedImpl(seed);
	17252	}
	17253
17262	17254	fn fallbackSeedPosix(seed: *[Csprng.seed_len]u8) void {
17263	17255	std.mem.writeInt(posix.pid_t, seed[0..@sizeOf(posix.pid_t)], posix.system.getpid(), .native);
17264	17256	const i_1 = @sizeOf(posix.pid_t);

lib/std/os/linux.zig+6-3

...	...	@@ -6717,9 +6717,10 @@ pub const IORING_ACCEPT_MULTISHOT = 1 << 0;
6717	6717	/// IORING_OP_MSG_RING command types, stored in sqe->addr
6718	6718	pub const IORING_MSG_RING_COMMAND = enum(u8) {
6719	6719	/// pass sqe->len as 'res' and off as user_data
6720		DATA,
	6720	DATA = 0,
6721	6721	/// send a registered fd to another ring
6722		SEND_FD,
	6722	SEND_FD = 1,
	6723	_,
6723	6724	};
6724	6725
6725	6726	// io_uring_sqe.msg_ring_flags (rw_flags in the Zig struct)
...	...	@@ -6772,6 +6773,8 @@ pub const IORING_CQE_F_SOCK_NONEMPTY = 1 << 2;
6772	6773	pub const IORING_CQE_F_NOTIF = 1 << 3;
6773	6774	/// If set, the buffer ID set in the completion will get more completions.
6774	6775	pub const IORING_CQE_F_BUF_MORE = 1 << 4;
	6776	pub const IORING_CQE_F_SKIP = 1 << 5;
	6777	pub const IORING_CQE_F_32 = 1 << 15;
6775	6778
6776	6779	pub const IORING_CQE_BUFFER_SHIFT = 16;
6777	6780
...	...	@@ -7068,7 +7071,7 @@ pub const IORING_RESTRICTION = enum(u16) {
7068	7071	_,
7069	7072	};
7070	7073
7071		pub const IO_URING_SOCKET_OP = enum(u16) {
	7074	pub const IO_URING_SOCKET_OP = enum(u32) {
7072	7075	SIOCIN = 0,
7073	7076	SIOCOUTQ = 1,
7074	7077	GETSOCKOPT = 2,

lib/std/process.zig+3-3

...	...	@@ -60,7 +60,7 @@ pub const CurrentPathError = error{
60	60	NameTooLong,
61	61	/// Not possible on Windows. Always returned on WASI.
62	62	CurrentDirUnlinked,
63		} \|\| Io.UnexpectedError;
	63	} \|\| Io.Cancelable \|\| Io.UnexpectedError;
64	64
65	65	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).
66	66	/// On other platforms, the result is an opaque sequence of bytes with no
...	...	@@ -72,7 +72,7 @@ pub fn currentPath(io: Io, buffer: []u8) CurrentPathError!usize {
72	72	pub const CurrentPathAllocError = Allocator.Error \|\| error{
73	73	/// Not possible on Windows. Always returned on WASI.
74	74	CurrentDirUnlinked,
75		} \|\| Io.UnexpectedError;
	75	} \|\| Io.Cancelable \|\| Io.UnexpectedError;
76	76
77	77	/// On Windows, the result is encoded as [WTF-8](https://wtf-8.codeberg.page/).
78	78	/// On other platforms, the result is an opaque sequence of bytes with no
...	...	@@ -355,7 +355,7 @@ pub const SpawnError = error{
355	355	/// On Windows, the volume does not contain a recognized file system. File
356	356	/// system drivers might not be loaded, or the volume may be corrupt.
357	357	UnrecognizedVolume,
358		} \|\| Io.Dir.PathNameError \|\| Io.Cancelable \|\| Io.UnexpectedError;
	358	} \|\| Io.File.OpenError \|\| Io.Dir.PathNameError \|\| Io.Cancelable \|\| Io.UnexpectedError;
359	359
360	360	pub const SpawnOptions = struct {
361	361	argv: []const []const u8,

lib/std/tar.zig+4-4

...	...	@@ -1128,10 +1128,10 @@ fn filePermissions(mode: u32, options: PipeOptions) Io.File.Permissions {
1128	1128
1129	1129	test filePermissions {
1130	1130	if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;
1131		try testing.expectEqual(.default_file, filePermissions(0o744, .{ .mode_mode = .ignore }));
1132		try testing.expectEqual(.executable_file, filePermissions(0o744, .{}));
1133		try testing.expectEqual(.default_file, filePermissions(0o644, .{}));
1134		try testing.expectEqual(.default_file, filePermissions(0o655, .{}));
	1131	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o744, .{ .mode_mode = .ignore }));
	1132	try testing.expectEqual(Io.File.Permissions.executable_file, filePermissions(0o744, .{}));
	1133	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o644, .{}));
	1134	try testing.expectEqual(Io.File.Permissions.default_file, filePermissions(0o655, .{}));
1135	1135	}
1136	1136
1137	1137	test "executable bit" {

src/Compilation.zig+26-45

...	...	@@ -21,6 +21,7 @@ const introspect = @import("introspect.zig");
21	21	const link = @import("link.zig");
22	22	const tracy = @import("tracy.zig");
23	23	const trace = tracy.trace;
	24	const traceNamed = tracy.traceNamed;
24	25	const build_options = @import("build_options");
25	26	const LibCInstallation = std.zig.LibCInstallation;
26	27	const glibc = @import("libs/glibc.zig");
...	...	@@ -4707,8 +4708,8 @@ fn performAllTheWork(
4707	4708	}
4708	4709
4709	4710	if (comp.zcu) \|zcu\| {
4710		const astgen_frame = tracy.namedFrame("astgen");
4711		defer astgen_frame.end();
	4711	const tracy_trace = traceNamed(@src(), "astgen");
	4712	defer tracy_trace.end();
4712	4713
4713	4714	const zir_prog_node = main_progress_node.start("AST Lowering", 0);
4714	4715	defer zir_prog_node.end();
...	...	@@ -4891,11 +4892,7 @@ fn performAllTheWork(
4891	4892
4892	4893	work: while (true) {
4893	4894	for (&comp.work_queues) \|*work_queue\| if (work_queue.popFront()) \|job\| {
4894		try processOneJob(
4895		@intFromEnum(Zcu.PerThread.Id.main),
4896		comp,
4897		job,
4898		);
	4895	try processOneJob(.main, comp, job);
4899	4896	continue :work;
4900	4897	};
4901	4898	if (comp.zcu) \|zcu\| {
...	...	@@ -5160,11 +5157,7 @@ pub fn queueJobs(comp: *Compilation, jobs: []const Job) !void {
5160	5157	for (jobs) \|job\| try comp.queueJob(job);
5161	5158	}
5162	5159
5163		fn processOneJob(
5164		tid: usize,
5165		comp: *Compilation,
5166		job: Job,
5167		) JobError!void {
	5160	fn processOneJob(tid: Zcu.PerThread.Id, comp: *Compilation, job: Job) JobError!void {
5168	5161	switch (job) {
5169	5162	.codegen_func => \|func\| {
5170	5163	const zcu = comp.zcu.?;
...	...	@@ -5229,10 +5222,10 @@ fn processOneJob(
5229	5222	try comp.link_queue.enqueueZcu(comp, tid, .{ .update_line_number = tracked_inst });
5230	5223	},
5231	5224	.analyze_func => \|func\| {
5232		const named_frame = tracy.namedFrame("analyze_func");
5233		defer named_frame.end();
	5225	const tracy_trace = traceNamed(@src(), "analyze_func");
	5226	defer tracy_trace.end();
5234	5227
5235		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5228	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5236	5229	defer pt.deactivate();
5237	5230
5238	5231	pt.ensureFuncBodyUpToDate(func) catch \|err\| switch (err) {
...	...	@@ -5242,10 +5235,10 @@ fn processOneJob(
5242	5235	};
5243	5236	},
5244	5237	.analyze_comptime_unit => \|unit\| {
5245		const named_frame = tracy.namedFrame("analyze_comptime_unit");
5246		defer named_frame.end();
	5238	const tracy_trace = traceNamed(@src(), "analyze_comptime_unit");
	5239	defer tracy_trace.end();
5247	5240
5248		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5241	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5249	5242	defer pt.deactivate();
5250	5243
5251	5244	const maybe_err: Zcu.SemaError!void = switch (unit.unwrap()) {
...	...	@@ -5282,10 +5275,10 @@ fn processOneJob(
5282	5275	}
5283	5276	},
5284	5277	.resolve_type_fully => \|ty\| {
5285		const named_frame = tracy.namedFrame("resolve_type_fully");
5286		defer named_frame.end();
	5278	const tracy_trace = traceNamed(@src(), "resolve_type_fully");
	5279	defer tracy_trace.end();
5287	5280
5288		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5281	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5289	5282	defer pt.deactivate();
5290	5283	Type.fromInterned(ty).resolveFully(pt) catch \|err\| switch (err) {
5291	5284	error.OutOfMemory, error.Canceled => \|e\| return e,
...	...	@@ -5293,10 +5286,10 @@ fn processOneJob(
5293	5286	};
5294	5287	},
5295	5288	.analyze_mod => \|mod\| {
5296		const named_frame = tracy.namedFrame("analyze_mod");
5297		defer named_frame.end();
	5289	const tracy_trace = traceNamed(@src(), "analyze_mod");
	5290	defer tracy_trace.end();
5298	5291
5299		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5292	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5300	5293	defer pt.deactivate();
5301	5294	pt.semaMod(mod) catch \|err\| switch (err) {
5302	5295	error.OutOfMemory, error.Canceled => \|e\| return e,
...	...	@@ -5304,8 +5297,8 @@ fn processOneJob(
5304	5297	};
5305	5298	},
5306	5299	.windows_import_lib => \|index\| {
5307		const named_frame = tracy.namedFrame("windows_import_lib");
5308		defer named_frame.end();
	5300	const tracy_trace = traceNamed(@src(), "windows_import_lib");
	5301	defer tracy_trace.end();
5309	5302
5310	5303	const link_lib = comp.windows_libs.keys()[index];
5311	5304	mingw.buildImportLib(comp, link_lib) catch \|err\| {
...	...	@@ -5642,13 +5635,14 @@ fn workerUpdateFile(
5642	5635	prog_node: std.Progress.Node,
5643	5636	group: *Io.Group,
5644	5637	) void {
5645		const tid = Compilation.getTid();
5646	5638	const io = comp.io;
	5639	const tid: Zcu.PerThread.Id = .acquire(io);
	5640	defer tid.release(io);
5647	5641
5648	5642	const child_prog_node = prog_node.start(fs.path.basename(file.path.sub_path), 0);
5649	5643	defer child_prog_node.end();
5650	5644
5651		const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
	5645	const pt: Zcu.PerThread = .activate(comp.zcu.?, tid);
5652	5646	defer pt.deactivate();
5653	5647	pt.updateFile(file_index, file) catch \|err\| {
5654	5648	pt.reportRetryableFileError(file_index, "unable to load '{s}': {s}", .{ fs.path.basename(file.path.sub_path), @errorName(err) }) catch \|oom\| switch (oom) {
...	...	@@ -5708,9 +5702,10 @@ fn workerUpdateBuiltinFile(comp: Compilation, file: Zcu.File) void {
5708	5702	}
5709	5703
5710	5704	fn workerUpdateEmbedFile(comp: Compilation, ef_index: Zcu.EmbedFile.Index, ef: Zcu.EmbedFile) void {
5711		const tid = Compilation.getTid();
5712	5705	const io = comp.io;
5713		comp.detectEmbedFileUpdate(@enumFromInt(tid), ef_index, ef) catch \|err\| switch (err) {
	5706	const tid: Zcu.PerThread.Id = .acquire(io);
	5707	defer tid.release(io);
	5708	comp.detectEmbedFileUpdate(tid, ef_index, ef) catch \|err\| switch (err) {
5714	5709	error.OutOfMemory => {
5715	5710	comp.mutex.lockUncancelable(io);
5716	5711	defer comp.mutex.unlock(io);
...	...	@@ -5868,7 +5863,7 @@ pub fn translateC(
5868	5863	}
5869	5864
5870	5865	var stdout: []u8 = undefined;
5871		try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, &stdout);
	5866	try @import("main.zig").translateC(gpa, arena, io, argv.items, environ_map, prog_node, comp.thread_limit, &stdout);
5872	5867
5873	5868	if (out_dep_path) \|dep_file_path\| add_deps: {
5874	5869	if (comp.verbose_cimport) log.info("processing dep file at {s}", .{dep_file_path});
...	...	@@ -8394,17 +8389,3 @@ pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
8394	8389	pub fn compilerRtStrip(comp: Compilation) bool {
8395	8390	return comp.root_mod.strip;
8396	8391	}
8397
8398		/// This is a temporary workaround put in place to migrate from `std.Thread.Pool`
8399		/// to `std.Io.Threaded` for asynchronous/concurrent work. The eventual solution
8400		/// will likely involve significant changes to the `InternPool` implementation.
8401		pub fn getTid() usize {
8402		if (my_tid == null) my_tid = next_tid.fetchAdd(1, .monotonic);
8403		return my_tid.?;
8404		}
8405		pub fn setMainThread() void {
8406		my_tid = 0;
8407		}
8408		/// TID 0 is reserved for the main thread.
8409		var next_tid: std.atomic.Value(usize) = .init(1);
8410		threadlocal var my_tid: ?usize = null;

src/InternPool.zig+6-7

...	...	@@ -3,6 +3,7 @@
3	3	const InternPool = @This();
4	4
5	5	const builtin = @import("builtin");
	6	const build_options = @import("build_options");
6	7
7	8	const std = @import("std");
8	9	const Io = std.Io;
...	...	@@ -86,13 +87,11 @@ dep_entries: std.ArrayList(DepEntry),
86	87	/// garbage collection pass.
87	88	free_dep_entries: std.ArrayList(DepEntry.Index),
88	89
89		/// Whether a multi-threaded intern pool is useful.
90		/// Currently `false` until the intern pool is actually accessed
91		/// from multiple threads to reduce the cost of this data structure.
92		const want_multi_threaded = true;
93
94	90	/// Whether a single-threaded intern pool impl is in use.
95		pub const single_threaded = builtin.single_threaded or !want_multi_threaded;
	91	pub const single_threaded = switch (build_options.io_mode) {
	92	.threaded => builtin.single_threaded,
	93	.evented => false, // even without threads, evented can be access from multiple tasks at a time
	94	};
96	95
97	96	pub const empty: InternPool = .{
98	97	.locals = &.{},
...	...	@@ -6915,7 +6914,7 @@ pub fn init(ip: *InternPool, gpa: Allocator, io: Io, available_threads: usize) !
6915	6914	assert(ip.locals.len == 0 and ip.shards.len == 0);
6916	6915	assert(available_threads > 0 and available_threads <= std.math.maxInt(u8));
6917	6916
6918		const used_threads = if (single_threaded) 1 else available_threads;
	6917	const used_threads = if (single_threaded) 1 else @max(available_threads, 2);
6919	6918	ip.locals = try gpa.alloc(Local, used_threads);
6920	6919	@memset(ip.locals, .{
6921	6920	.shared = .{

src/Sema.zig+1

...	...	@@ -23090,6 +23090,7 @@ fn checkAtomicPtrOperand(
23090	23090	) CompileError!Air.Inst.Ref {
23091	23091	const pt = sema.pt;
23092	23092	const zcu = pt.zcu;
	23093	try elem_ty.resolveLayout(pt);
23093	23094	var diag: Zcu.AtomicPtrAlignmentDiagnostics = .{};
23094	23095	const alignment = zcu.atomicPtrAlignment(elem_ty, &diag) catch \|err\| switch (err) {
23095	23096	error.OutOfMemory => return error.OutOfMemory,

src/Zcu.zig+8-4

...	...	@@ -4954,8 +4954,10 @@ pub const CodegenTaskPool = struct {
4954	4954	// We own `air` now, so we are responsbile for freeing it.
4955	4955	var air = orig_air;
4956	4956	defer air.deinit(zcu.comp.gpa);
4957		const tid = Compilation.getTid();
4958		const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
	4957	const io = zcu.comp.io;
	4958	const tid: Zcu.PerThread.Id = .acquire(io);
	4959	defer tid.release(io);
	4960	const pt: Zcu.PerThread = .activate(zcu, tid);
4959	4961	defer pt.deactivate();
4960	4962	return pt.runCodegen(func_index, &air);
4961	4963	}
...	...	@@ -4964,8 +4966,10 @@ pub const CodegenTaskPool = struct {
4964	4966	func_index: InternPool.Index,
4965	4967	air: *Air,
4966	4968	) CodegenResult {
4967		const tid = Compilation.getTid();
4968		const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
	4969	const io = zcu.comp.io;
	4970	const tid: Zcu.PerThread.Id = .acquire(io);
	4971	defer tid.release(io);
	4972	const pt: Zcu.PerThread = .activate(zcu, tid);
4969	4973	defer pt.deactivate();
4970	4974	return pt.runCodegen(func_index, air);
4971	4975	}

src/Zcu/PerThread.zig+75-2

...	...	@@ -41,13 +41,86 @@ zcu: *Zcu,
41	41	tid: Id,
42	42
43	43	pub const IdBacking = u7;
44		pub const Id = if (InternPool.single_threaded) enum { main } else enum(IdBacking) { main, _ };
	44	pub const Id = if (InternPool.single_threaded) enum {
	45	main,
	46
	47	pub fn allocate(arena: Allocator, n: usize) Allocator.Error!void {
	48	_ = arena;
	49	_ = n;
	50	}
	51	pub fn acquire(io: std.Io) Id {
	52	_ = io;
	53	return .main;
	54	}
	55	pub fn release(tid: Id, io: std.Io) void {
	56	_ = io;
	57	_ = tid;
	58	}
	59	} else enum(IdBacking) {
	60	main,
	61	_,
	62
	63	var tid_mutex: std.Io.Mutex = .init;
	64	var tid_cond: std.Io.Condition = .init;
	65	/// This is a temporary workaround put in place to migrate from `std.Thread.Pool`
	66	/// to `std.Io.Threaded` for asynchronous/concurrent work. The eventual solution
	67	/// will likely involve significant changes to the `InternPool` implementation.
	68	var available_tids: std.ArrayList(Id) = .empty;
	69	threadlocal var recursive_depth: usize = 0;
	70	threadlocal var recursive_tid: Id = .main;
	71
	72	pub fn allocate(arena: Allocator, n: usize) Allocator.Error!void {
	73	assert(available_tids.items.len == 0);
	74	try available_tids.ensureTotalCapacityPrecise(arena, n - 1);
	75	for (1..n) \|tid\| available_tids.appendAssumeCapacity(@enumFromInt(tid));
	76	}
	77	pub fn acquire(io: std.Io) Id {
	78	switch (build_options.io_mode) {
	79	.threaded => {
	80	recursive_depth += 1;
	81	if (recursive_depth > 1) {
	82	assert(recursive_tid != .main);
	83	return recursive_tid;
	84	}
	85	},
	86	.evented => {},
	87	}
	88	tid_mutex.lockUncancelable(io);
	89	defer tid_mutex.unlock(io);
	90	while (true) {
	91	if (available_tids.pop()) \|tid\| {
	92	switch (build_options.io_mode) {
	93	.threaded => recursive_tid = tid,
	94	.evented => {},
	95	}
	96	return tid;
	97	}
	98	tid_cond.waitUncancelable(io, &tid_mutex);
	99	}
	100	}
	101	pub fn release(tid: Id, io: std.Io) void {
	102	switch (build_options.io_mode) {
	103	.threaded => {
	104	assert(recursive_tid == tid);
	105	recursive_depth -= 1;
	106	if (recursive_depth > 0) return;
	107	recursive_tid = .main;
	108	},
	109	.evented => {},
	110	}
	111	{
	112	tid_mutex.lockUncancelable(io);
	113	defer tid_mutex.unlock(io);
	114	available_tids.appendAssumeCapacity(tid);
	115	}
	116	tid_cond.signal(io);
	117	}
	118	};
45	119
46	120	pub fn activate(zcu: *Zcu, tid: Id) Zcu.PerThread {
47	121	zcu.intern_pool.activate();
48	122	return .{ .zcu = zcu, .tid = tid };
49	123	}
50
51	124	pub fn deactivate(pt: Zcu.PerThread) void {
52	125	pt.zcu.intern_pool.deactivate();
53	126	}

src/crash_report.zig+9-38

...	...	@@ -1,34 +1,14 @@
1		/// We override the panic implementation to our own one, so we can print our own information before
2		/// calling the default panic handler. This declaration must be re-exposed from `@import("root")`.
3		pub const panic = if (dev.env == .bootstrap)
4		std.debug.simple_panic
5		else
6		std.debug.FullPanic(panicImpl);
7
8		/// We let std install its segfault handler, but we override the target-agnostic handler it calls,
9		/// so we can print our own information before calling the default segfault logic. This declaration
10		/// must be re-exposed from `@import("root")`.
11		pub const debug = struct {
12		pub const handleSegfault = handleSegfaultImpl;
13		};
14
15	1	/// Printed in panic messages when suggesting a command to run, allowing copy-pasting the command.
16	2	/// Set by `main` as soon as arguments are known. The value here is a default in case we somehow
17	3	/// crash earlier than that.
18	4	pub var zig_argv0: []const u8 = "zig";
19	5
20		fn handleSegfaultImpl(addr: ?usize, name: []const u8, opt_ctx: ?std.debug.CpuContextPtr) noreturn {
21		@branchHint(.cold);
22		dumpCrashContext() catch {};
23		std.debug.defaultHandleSegfault(addr, name, opt_ctx);
24		}
25		fn panicImpl(msg: []const u8, first_trace_addr: ?usize) noreturn {
26		@branchHint(.cold);
27		dumpCrashContext() catch {};
28		std.debug.defaultPanic(msg, first_trace_addr orelse @returnAddress());
29		}
	6	const enabled = switch (build_options.io_mode) {
	7	.threaded => build_options.enable_debug_extensions,
	8	.evented => false, // would use threadlocals in a way incompatible with evented
	9	};
30	10
31		pub const AnalyzeBody = if (build_options.enable_debug_extensions) struct {
	11	pub const AnalyzeBody = if (enabled) struct {
32	12	parent: ?*AnalyzeBody,
33	13	sema: *Sema,
34	14	block: *Sema.Block,
...	...	@@ -63,7 +43,7 @@ pub const AnalyzeBody = if (build_options.enable_debug_extensions) struct {
63	43	pub inline fn setBodyIndex(_: @This(), _: usize) void {}
64	44	};
65	45
66		pub const CodegenFunc = if (build_options.enable_debug_extensions) struct {
	46	pub const CodegenFunc = if (enabled) struct {
67	47	zcu: *const Zcu,
68	48	func_index: InternPool.Index,
69	49	threadlocal var current: ?CodegenFunc = null;
...	...	@@ -82,23 +62,14 @@ pub const CodegenFunc = if (build_options.enable_debug_extensions) struct {
82	62	pub fn stop(_: InternPool.Index) void {}
83	63	};
84	64
85		fn dumpCrashContext() Io.Writer.Error!void {
	65	pub fn dumpCrashContext(terminal: Io.Terminal) Io.Writer.Error!void {
86	66	const S = struct {
87		/// In the case of recursive panics or segfaults, don't print the context for a second time.
88		threadlocal var already_dumped = false;
89	67	/// TODO: make this unnecessary. It exists because `print_zir` currently needs an allocator,
90	68	/// but that shouldn't be necessary---it's already only used in one place.
91		threadlocal var crash_heap: [64 * 1024]u8 = undefined;
	69	var crash_heap: [64 * 1024]u8 = undefined;
92	70	};
93		if (S.already_dumped) return;
94		S.already_dumped = true;
95
96		// TODO: this does mean that a different thread could grab the stderr mutex between the context
97		// and the actual panic printing, which would be quite confusing.
98		const stderr = std.debug.lockStderr(&.{});
99		defer std.debug.unlockStderr();
100		const w = &stderr.file_writer.interface;
101	71
	72	const w = terminal.writer;
102	73	try w.writeAll("Compiler crash context:\n");
103	74
104	75	if (CodegenFunc.current) \|*cg\| {

src/introspect.zig+1-5

...	...	@@ -54,11 +54,7 @@ pub fn findZigLibDir(gpa: Allocator, io: Io) !Cache.Directory {
54	54	/// Like `std.process.currentPathAlloc`, but also resolves the path with `Dir.path.resolve`. This
55	55	/// means the path has no repeated separators, no "." or ".." components, and no trailing separator.
56	56	/// On WASI, "" is returned instead of ".".
57		pub fn getResolvedCwd(io: Io, gpa: Allocator) error{
58		OutOfMemory,
59		CurrentDirUnlinked,
60		Unexpected,
61		}![]u8 {
	57	pub fn getResolvedCwd(io: Io, gpa: Allocator) std.process.CurrentPathAllocError![]u8 {
62	58	if (builtin.target.os.tag == .wasi) {
63	59	if (std.debug.runtime_safety) {
64	60	const cwd = try std.process.currentPathAlloc(io, gpa);

src/link.zig+4-4

...	...	@@ -1500,12 +1500,12 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
1500	1500	},
1501	1501	}
1502	1502	}
1503		pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
	1503	pub fn doZcuTask(comp: *Compilation, tid: Zcu.PerThread.Id, task: ZcuTask) void {
1504	1504	const io = comp.io;
1505	1505	const diags = &comp.link_diags;
1506	1506	const zcu = comp.zcu.?;
1507	1507	const ip = &zcu.intern_pool;
1508		const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
	1508	const pt: Zcu.PerThread = .activate(zcu, tid);
1509	1509	defer pt.deactivate();
1510	1510
1511	1511	var timer = comp.startTimer();
...	...	@@ -1610,8 +1610,8 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
1610	1610	}
1611	1611	}
1612	1612	}
1613		pub fn doIdleTask(comp: *Compilation, tid: usize) error{ OutOfMemory, LinkFailure }!bool {
1614		return if (comp.bin_file) \|lf\| lf.idle(@enumFromInt(tid)) else false;
	1613	pub fn doIdleTask(comp: *Compilation, tid: Zcu.PerThread.Id) error{ OutOfMemory, LinkFailure }!bool {
	1614	return if (comp.bin_file) \|lf\| lf.idle(tid) else false;
1615	1615	}
1616	1616	/// After the main pipeline is done, but before flush, the compilation may need to link one final
1617	1617	/// `Nav` into the binary: the `builtin.test_functions` value. Since the link thread isn't running

src/link/Queue.zig+7-5

...	...	@@ -96,12 +96,12 @@ pub fn enqueuePrelink(q: Queue, comp: Compilation, tasks: []const PrelinkTask)
96	96	pub fn enqueueZcu(
97	97	q: *Queue,
98	98	comp: *Compilation,
99		tid: usize,
	99	tid: Zcu.PerThread.Id,
100	100	task: ZcuTask,
101	101	) Io.Cancelable!void {
102	102	const io = comp.io;
103	103
104		assert(tid == 0);
	104	assert(tid == .main);
105	105
106	106	if (q.future != null) {
107	107	if (q.zcu_queue.putOne(io, task)) \|_\| {
...	...	@@ -148,8 +148,9 @@ pub fn finishZcuQueue(q: Queue, comp: Compilation) void {
148	148	}
149	149
150	150	fn runLinkTasks(q: Queue, comp: Compilation) void {
151		const tid = Compilation.getTid();
152	151	const io = comp.io;
	152	const tid: Zcu.PerThread.Id = .acquire(io);
	153	defer tid.release(io);
153	154
154	155	var have_idle_tasks = true;
155	156
...	...	@@ -198,7 +199,7 @@ fn runLinkTasks(q: Queue, comp: Compilation) void {
198	199	}
199	200	}
200	201	}
201		fn runIdleTask(comp: *Compilation, tid: usize) bool {
	202	fn runIdleTask(comp: *Compilation, tid: Zcu.PerThread.Id) bool {
202	203	return link.doIdleTask(comp, tid) catch \|err\| switch (err) {
203	204	error.OutOfMemory => have_more: {
204	205	comp.link_diags.setAllocFailure();
...	...	@@ -217,5 +218,6 @@ const Compilation = @import("../Compilation.zig");
217	218	const InternPool = @import("../InternPool.zig");
218	219	const link = @import("../link.zig");
219	220	const PrelinkTask = link.PrelinkTask;
220		const ZcuTask = link.ZcuTask;
221	221	const Queue = @This();
	222	const Zcu = @import("../Zcu.zig");
	223	const ZcuTask = link.ZcuTask;

src/main.zig+94-50

...	...	@@ -21,7 +21,7 @@ const AstGen = std.zig.AstGen;
21	21	const ZonGen = std.zig.ZonGen;
22	22	const Server = std.zig.Server;
23	23
24		const tracy = @import("tracy.zig");
	24	pub const tracy = @import("tracy.zig");
25	25	const Compilation = @import("Compilation.zig");
26	26	const link = @import("link.zig");
27	27	const Package = @import("Package.zig");
...	...	@@ -52,8 +52,11 @@ pub const std_options: std.Options = .{
52	52	};
53	53	pub const std_options_cwd = if (native_os == .wasi) wasi_cwd else null;
54	54
55		pub const panic = crash_report.panic;
56		pub const debug = crash_report.debug;
	55	pub const debug = struct {
	56	pub fn printCrashContext(terminal: Io.Terminal) void {
	57	crash_report.dumpCrashContext(terminal) catch {};
	58	}
	59	};
57	60
58	61	var preopens: std.process.Preopens = .empty;
59	62	pub fn wasi_cwd() Io.Dir {
...	...	@@ -158,25 +161,55 @@ pub fn log(
158	161	std.log.defaultLog(level, scope, format, args);
159	162	}
160	163
161		var debug_allocator: std.heap.DebugAllocator(.{
162		.stack_trace_frames = build_options.mem_leak_frames,
163		}) = .init;
164
165	164	const use_debug_allocator = build_options.debug_gpa or
166	165	(native_os != .wasi and !builtin.link_libc and switch (builtin.mode) {
167	166	.Debug, .ReleaseSafe => true,
168	167	.ReleaseFast, .ReleaseSmall => false,
169	168	});
170	169
	170	const RootAllocator = if (use_debug_allocator) std.heap.DebugAllocator(.{
	171	.stack_trace_frames = build_options.mem_leak_frames,
	172	.thread_safe = switch (build_options.io_mode) {
	173	.threaded => true,
	174	.evented => false,
	175	},
	176	}) else struct {
	177	pub const init: RootAllocator = .{};
	178	pub fn allocator(_: RootAllocator) Allocator {
	179	if (native_os == .wasi) return std.heap.wasm_allocator;
	180	if (builtin.link_libc) return std.heap.c_allocator;
	181	return std.heap.smp_allocator;
	182	}
	183	pub fn deinit(_: RootAllocator) std.heap.Check {
	184	return .ok;
	185	}
	186	};
	187
171	188	pub fn main(init: std.process.Init.Minimal) anyerror!void {
172		const gpa = gpa: {
173		if (use_debug_allocator) break :gpa debug_allocator.allocator();
174		if (native_os == .wasi) break :gpa std.heap.wasm_allocator;
175		if (builtin.link_libc) break :gpa std.heap.c_allocator;
176		break :gpa std.heap.smp_allocator;
177		};
178		defer if (use_debug_allocator) {
179		_ = debug_allocator.deinit();
	189	var root_allocator: RootAllocator = .init;
	190	defer _ = root_allocator.deinit();
	191	const root_gpa = root_allocator.allocator();
	192	var io_impl: IoImpl = undefined;
	193	switch (build_options.io_mode) {
	194	.threaded => io_impl = .init(root_gpa, .{
	195	.stack_size = thread_stack_size,
	196
	197	.argv0 = .init(init.args),
	198	.environ = init.environ,
	199	}),
	200	.evented => try io_impl.init(root_gpa, .{
	201	.argv0 = .init(init.args),
	202	.environ = init.environ,
	203
	204	.backing_allocator_needs_mutex = use_debug_allocator,
	205	}),
	206	}
	207	defer io_impl.deinit();
	208	io_impl_ptr = &io_impl;
	209	const io = io_impl.io();
	210	const gpa = switch (build_options.io_mode) {
	211	.threaded => root_gpa,
	212	.evented => io_impl.allocator(),
180	213	};
181	214	var arena_instance = std.heap.ArenaAllocator.init(gpa);
182	215	defer arena_instance.deinit();
...	...	@@ -193,17 +226,6 @@ pub fn main(init: std.process.Init.Minimal) anyerror!void {
193	226
194	227	var environ_map = init.environ.createMap(arena) catch \|err\| fatal("failed to parse environment: {t}", .{err});
195	228
196		Compilation.setMainThread();
197
198		var threaded: Io.Threaded = .init(gpa, .{
199		.argv0 = .init(init.args),
200		.environ = init.environ,
201		});
202		defer threaded.deinit();
203		threaded_impl_ptr = &threaded;
204		threaded.stack_size = thread_stack_size;
205		const io = threaded.io();
206
207	229	if (tracy.enable_allocation) {
208	230	var gpa_tracy = tracy.tracyAllocator(gpa);
209	231	return mainArgs(gpa_tracy.allocator(), arena, io, args, &environ_map);
...	...	@@ -3400,7 +3422,7 @@ fn buildOutputType(
3400	3422	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
3401	3423	std.math.maxInt(Zcu.PerThread.IdBacking),
3402	3424	);
3403		setThreadLimit(thread_limit);
	3425	try setThreadLimit(arena, thread_limit);
3404	3426
3405	3427	for (create_module.c_source_files.items) \|*src\| {
3406	3428	dev.check(.c_compiler);
...	...	@@ -4731,13 +4753,13 @@ pub fn translateC(
4731	4753	argv: []const []const u8,
4732	4754	environ_map: *const process.Environ.Map,
4733	4755	prog_node: std.Progress.Node,
	4756	thread_limit: usize,
4734	4757	capture: ?*[]u8,
4735	4758	) !void {
4736		try jitCmd(gpa, arena, io, argv, environ_map, .{
	4759	try jitCmdInner(gpa, arena, io, argv, environ_map, prog_node, thread_limit, .{
4737	4760	.cmd_name = "translate-c",
4738	4761	.root_src_path = "translate-c/main.zig",
4739	4762	.depend_on_aro = true,
4740		.progress_node = prog_node,
4741	4763	.capture = capture,
4742	4764	});
4743	4765	}
...	...	@@ -5187,7 +5209,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5187	5209	@max(n_jobs orelse std.Thread.getCpuCount() catch 1, 1),
5188	5210	std.math.maxInt(Zcu.PerThread.IdBacking),
5189	5211	);
5190		setThreadLimit(thread_limit);
	5212	try setThreadLimit(arena, thread_limit);
5191	5213
5192	5214	// Dummy http client that is not actually used when fetch_command is unsupported.
5193	5215	// Prevents bootstrap from depending on a bunch of unnecessary stuff.
...	...	@@ -5651,7 +5673,7 @@ const JitCmdOptions = struct {
5651	5673	capture: ?*[]u8 = null,
5652	5674	/// Send error bundles via std.zig.Server over stdout
5653	5675	server: bool = false,
5654		progress_node: ?std.Progress.Node = null,
	5676	color: Color = .auto,
5655	5677	};
5656	5678
5657	5679	fn jitCmd(
...	...	@@ -5664,12 +5686,30 @@ fn jitCmd(
5664	5686	) !void {
5665	5687	dev.check(.jit_command);
5666	5688
5667		const color: Color = .auto;
5668		const root_prog_node = if (options.progress_node) \|node\| node else std.Progress.start(io, .{
5669		.disable_printing = (color == .off),
	5689	const root_prog_node = std.Progress.start(io, .{
	5690	.disable_printing = (options.color == .off),
5670	5691	});
5671	5692	defer root_prog_node.end();
5672	5693
	5694	const thread_limit = @min(
	5695	@max(std.Thread.getCpuCount() catch 1, 1),
	5696	std.math.maxInt(Zcu.PerThread.IdBacking),
	5697	);
	5698	try setThreadLimit(arena, thread_limit);
	5699
	5700	return jitCmdInner(gpa, arena, io, args, environ_map, root_prog_node, thread_limit, options);
	5701	}
	5702
	5703	fn jitCmdInner(
	5704	gpa: Allocator,
	5705	arena: Allocator,
	5706	io: Io,
	5707	args: []const []const u8,
	5708	environ_map: *const process.Environ.Map,
	5709	root_prog_node: std.Progress.Node,
	5710	thread_limit: usize,
	5711	options: JitCmdOptions,
	5712	) !void {
5673	5713	const target_query: std.Target.Query = .{};
5674	5714	const resolved_target: Package.Module.ResolvedTarget = .{
5675	5715	.result = std.zig.resolveTargetQueryOrFatal(io, target_query),
...	...	@@ -5702,12 +5742,6 @@ fn jitCmd(
5702	5742	);
5703	5743	defer dirs.deinit(io);
5704	5744
5705		const thread_limit = @min(
5706		@max(std.Thread.getCpuCount() catch 1, 1),
5707		std.math.maxInt(Zcu.PerThread.IdBacking),
5708		);
5709		setThreadLimit(thread_limit);
5710
5711	5745	var child_argv: std.ArrayList([]const u8) = .empty;
5712	5746	try child_argv.ensureUnusedCapacity(arena, args.len + 4);
5713	5747
...	...	@@ -5795,7 +5829,7 @@ fn jitCmd(
5795	5829	process.exit(2);
5796	5830	}
5797	5831	} else {
5798		updateModule(comp, color, root_prog_node) catch \|err\| switch (err) {
	5832	updateModule(comp, options.color, root_prog_node) catch \|err\| switch (err) {
5799	5833	error.CompileErrorsReported => process.exit(2),
5800	5834	else => \|e\| return e,
5801	5835	};
...	...	@@ -7777,15 +7811,25 @@ fn addLibDirectoryWarn2(
7777	7811	});
7778	7812	}
7779	7813
7780		var threaded_impl_ptr: *Io.Threaded = undefined;
7781		fn setThreadLimit(n: usize) void {
7782		// We want a maximum of n total threads to keep the InternPool happy, but
7783		// the main thread doesn't count towards the limits, so use n-1. Also, the
7784		// linker can run concurrently, so we need to set both the async and the
7785		// concurrency limit.
7786		const limit: Io.Limit = .limited(n - 1);
7787		threaded_impl_ptr.setAsyncLimit(limit);
7788		threaded_impl_ptr.concurrent_limit = limit;
	7814	const IoImpl = switch (build_options.io_mode) {
	7815	.threaded => Io.Threaded,
	7816	.evented => Io.Evented,
	7817	};
	7818	var io_impl_ptr: *IoImpl = undefined;
	7819	fn setThreadLimit(arena: std.mem.Allocator, n: usize) Allocator.Error!void {
	7820	switch (build_options.io_mode) {
	7821	.threaded => {
	7822	// We want a maximum of n total threads to keep the InternPool happy, but
	7823	// the main thread doesn't count towards the limits, so use n-1. Also, the
	7824	// linker can run concurrently, so we need to set both the async and the
	7825	// concurrency limit.
	7826	const limit: Io.Limit = .limited(n - 1);
	7827	io_impl_ptr.setAsyncLimit(limit);
	7828	io_impl_ptr.concurrent_limit = limit;
	7829	},
	7830	.evented => {},
	7831	}
	7832	try Zcu.PerThread.Id.allocate(arena, @max(n, 2));
7789	7833	}
7790	7834
7791	7835	fn randInt(io: Io, comptime T: type) T {

src/tracy.zig+40-30

...	...	@@ -9,7 +9,7 @@ pub const callstack_depth = if (enable_callstack and build_options.tracy_callsta
9	9
10	10	const ___tracy_c_zone_context = extern struct {
11	11	id: u32,
12		active: c_int,
	12	active: i32,
13	13
14	14	pub inline fn end(self: @This()) void {
15	15	___tracy_emit_zone_end(self);
...	...	@@ -98,6 +98,16 @@ pub inline fn traceNamed(comptime src: std.builtin.SourceLocation, comptime name
98	98	}
99	99	}
100	100
	101	pub inline fn fiberEnter(fiber: [*:0]const u8) void {
	102	if (!enable) return;
	103	___tracy_fiber_enter(fiber);
	104	}
	105
	106	pub inline fn fiberLeave() void {
	107	if (!enable) return;
	108	___tracy_fiber_leave();
	109	}
	110
101	111	pub fn tracyAllocator(allocator: std.mem.Allocator) TracyAllocator(null) {
102	112	return TracyAllocator(null).init(allocator);
103	113	}
...	...	@@ -197,24 +207,22 @@ pub fn TracyAllocator(comptime name: ?[:0]const u8) type {
197	207
198	208	// This function only accepts comptime-known strings, see `messageCopy` for runtime strings
199	209	pub inline fn message(comptime msg: [:0]const u8) void {
200		if (!enable) return;
201		___tracy_emit_messageL(msg.ptr, if (enable_callstack) callstack_depth else 0);
	210	messageColor(msg, 0);
202	211	}
203	212
204	213	// This function only accepts comptime-known strings, see `messageColorCopy` for runtime strings
205		pub inline fn messageColor(comptime msg: [:0]const u8, color: u32) void {
	214	pub inline fn messageColor(comptime msg: [:0]const u8, color: u24) void {
206	215	if (!enable) return;
207		___tracy_emit_messageLC(msg.ptr, color, if (enable_callstack) callstack_depth else 0);
	216	___tracy_emit_logStringL(.Info, color, if (enable_callstack) callstack_depth else 0, msg.ptr);
208	217	}
209	218
210	219	pub inline fn messageCopy(msg: []const u8) void {
211		if (!enable) return;
212		___tracy_emit_message(msg.ptr, msg.len, if (enable_callstack) callstack_depth else 0);
	220	messageColorCopy(msg, 0);
213	221	}
214	222
215		pub inline fn messageColorCopy(msg: [:0]const u8, color: u32) void {
	223	pub inline fn messageColorCopy(msg: []const u8, color: u24) void {
216	224	if (!enable) return;
217		___tracy_emit_messageC(msg.ptr, msg.len, color, if (enable_callstack) callstack_depth else 0);
	225	___tracy_emit_logString(.Info, color, if (enable_callstack) callstack_depth else 0, msg.len, msg.ptr);
218	226	}
219	227
220	228	pub inline fn frameMark() void {
...	...	@@ -293,33 +301,35 @@ inline fn freeNamed(ptr: [*]u8, comptime name: [:0]const u8) void {
293	301	}
294	302	}
295	303
296		extern fn ___tracy_emit_zone_begin(
297		srcloc: *const ___tracy_source_location_data,
298		active: c_int,
299		) ___tracy_c_zone_context;
300		extern fn ___tracy_emit_zone_begin_callstack(
301		srcloc: *const ___tracy_source_location_data,
302		depth: c_int,
303		active: c_int,
304		) ___tracy_c_zone_context;
	304	pub const MessageSeverity = enum(i8) {
	305	Trace, // Broadly track variable states and events in the software program.
	306	Debug, // Describes variable states and details about specific internal events in the software, that are useful for investigations.
	307	Info, // Describes normal events, which inform on the expected progress and state of your software.
	308	Warning, // Describes potentially dangerous situations caused by unexpected events and states.
	309	Error, // Describes the occurance of unexpected behavior. Does not interrupt the execution of the software.
	310	Fatal, // Describes a critical event that will lead to a software failure/crash.
	311	};
	312
	313	extern fn ___tracy_emit_zone_begin(srcloc: *const ___tracy_source_location_data, active: i32) ___tracy_c_zone_context;
	314	extern fn ___tracy_emit_zone_begin_callstack(srcloc: *const ___tracy_source_location_data, depth: i32, active: i32) ___tracy_c_zone_context;
305	315	extern fn ___tracy_emit_zone_text(ctx: ___tracy_c_zone_context, txt: [*]const u8, size: usize) void;
306	316	extern fn ___tracy_emit_zone_name(ctx: ___tracy_c_zone_context, txt: [*]const u8, size: usize) void;
307	317	extern fn ___tracy_emit_zone_color(ctx: ___tracy_c_zone_context, color: u32) void;
308	318	extern fn ___tracy_emit_zone_value(ctx: ___tracy_c_zone_context, value: u64) void;
309	319	extern fn ___tracy_emit_zone_end(ctx: ___tracy_c_zone_context) void;
310		extern fn ___tracy_emit_memory_alloc(ptr: *const anyopaque, size: usize, secure: c_int) void;
311		extern fn ___tracy_emit_memory_alloc_callstack(ptr: *const anyopaque, size: usize, depth: c_int, secure: c_int) void;
312		extern fn ___tracy_emit_memory_free(ptr: *const anyopaque, secure: c_int) void;
313		extern fn ___tracy_emit_memory_free_callstack(ptr: *const anyopaque, depth: c_int, secure: c_int) void;
314		extern fn ___tracy_emit_memory_alloc_named(ptr: const anyopaque, size: usize, secure: c_int, name: [:0]const u8) void;
315		extern fn ___tracy_emit_memory_alloc_callstack_named(ptr: const anyopaque, size: usize, depth: c_int, secure: c_int, name: [:0]const u8) void;
316		extern fn ___tracy_emit_memory_free_named(ptr: const anyopaque, secure: c_int, name: [:0]const u8) void;
317		extern fn ___tracy_emit_memory_free_callstack_named(ptr: const anyopaque, depth: c_int, secure: c_int, name: [:0]const u8) void;
318		extern fn ___tracy_emit_message(txt: [*]const u8, size: usize, callstack: c_int) void;
319		extern fn ___tracy_emit_messageL(txt: [*:0]const u8, callstack: c_int) void;
320		extern fn ___tracy_emit_messageC(txt: [*]const u8, size: usize, color: u32, callstack: c_int) void;
321		extern fn ___tracy_emit_messageLC(txt: [*:0]const u8, color: u32, callstack: c_int) void;
	320	extern fn ___tracy_emit_memory_alloc(ptr: *const anyopaque, size: usize, secure: i32) void;
	321	extern fn ___tracy_emit_memory_alloc_callstack(ptr: *const anyopaque, size: usize, depth: i32, secure: i32) void;
	322	extern fn ___tracy_emit_memory_free(ptr: *const anyopaque, secure: i32) void;
	323	extern fn ___tracy_emit_memory_free_callstack(ptr: *const anyopaque, depth: i32, secure: i32) void;
	324	extern fn ___tracy_emit_memory_alloc_named(ptr: const anyopaque, size: usize, secure: i32, name: [:0]const u8) void;
	325	extern fn ___tracy_emit_memory_alloc_callstack_named(ptr: const anyopaque, size: usize, depth: i32, secure: i32, name: [:0]const u8) void;
	326	extern fn ___tracy_emit_memory_free_named(ptr: const anyopaque, secure: i32, name: [:0]const u8) void;
	327	extern fn ___tracy_emit_memory_free_callstack_named(ptr: const anyopaque, depth: i32, secure: i32, name: [:0]const u8) void;
	328	extern fn ___tracy_emit_logString(severity: MessageSeverity, color: i32, callstack_depth: i32, size: usize, txt: [*]const u8) void;
	329	extern fn ___tracy_emit_logStringL(severity: MessageSeverity, color: i32, callstack_depth: i32, txt: [*:0]const u8) void;
322	330	extern fn ___tracy_emit_frame_mark(name: ?[*:0]const u8) void;
	331	extern fn ___tracy_fiber_enter(fiber: [*:0]const u8) void;
	332	extern fn ___tracy_fiber_leave() void;
323	333
324	334	const ___tracy_source_location_data = extern struct {
325	335	name: ?[*:0]const u8,

stage1/config.zig.in+1

...	...	@@ -13,4 +13,5 @@ pub const value_tracing = false;
13	13	pub const skip_non_native = false;
14	14	pub const debug_gpa = false;
15	15	pub const dev = .core;
	16	pub const io_mode: enum { threaded, evented } = .threaded;
16	17	pub const value_interpret_mode = .direct;