Commit 841dd0eb874cf0febf9582031f86703f5cde00cb | zig

author	Matthew Lugg <mlugg@mlugg.co.uk> 2026-08-25 08:30:25+02:00
committer	Matthew Lugg <mlugg@mlugg.co.uk> 2026-08-25 08:30:25+02:00
log	841dd0eb874cf0febf9582031f86703f5cde00cb
tree	492bd49abf34c42e0c2ce736d0265651e12fa321
parent	40ad0920b37564a4c119fa6fa385b76a9c2b0c21
parent	edf18aa383cc5553a51e57954cf7e076c4b745db

Merge pull request 'MappedFile: rework node operations' (#36618) from mlugg/mappedfile-stuff into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36618

6 files changed, 2506 insertions(+), 1548 deletions(-)

lib
- compiler
  - test_runner.zig
src

lib/compiler/test_runner.zig+16-7

...	...	@@ -185,7 +185,6 @@ fn mainServer(init: std.process.Init.Minimal) !void {
185	185	.environ = init.environ,
186	186	});
187	187	defer io_instance.deinit();
188		const io = io_instance.io();
189	188
190	189	const mode: fuzz_abi.LimitKind = @fromBackingInt(@intCast(try server.receiveBody_u8()));
191	190	const amount_or_instance = try server.receiveBody_u64();
...	...	@@ -208,7 +207,7 @@ fn mainServer(init: std.process.Init.Minimal) !void {
208	207	.indexes = test_indexes,
209	208	.server = &server,
210	209	.gpa = gpa,
211		.io = io,
	210	.threaded_io = &io_instance,
212	211	.input_poller = undefined,
213	212	};
214	213
...	...	@@ -422,7 +421,7 @@ var fuzz_runner: if (builtin.fuzz) struct {
422	421	indexes: []u32,
423	422	server: *std.zig.Server,
424	423	gpa: std.mem.Allocator,
425		io: Io,
	424	threaded_io: *Io.Threaded,
426	425	input_poller: Io.Future(Io.Cancelable!void),
427	426
428	427	comptime {
...	...	@@ -443,6 +442,12 @@ var fuzz_runner: if (builtin.fuzz) struct {
443	442	defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1);
444	443	is_fuzz_test = false;
445	444
	445	testing.io_instance = .init(testing.allocator, .{
	446	.argv0 = fuzz_runner.threaded_io.argv0,
	447	.environ = fuzz_runner.threaded_io.environ.process_environ,
	448	});
	449	defer testing.io_instance.deinit();
	450
446	451	builtin.test_functions[fuzz_runner.indexes[i]].func() catch \|err\| switch (err) {
447	452	error.SkipZigTest => return,
448	453	else => {
...	...	@@ -473,7 +478,8 @@ var fuzz_runner: if (builtin.fuzz) struct {
473	478
474	479	export fn runner_start_input_poller() void {
475	480	@disableInstrumentation();
476		const future = fuzz_runner.io.concurrent(inputPoller, .{}) catch \|e\| switch (e) {
	481	const io = fuzz_runner.threaded_io.io();
	482	const future = io.concurrent(inputPoller, .{}) catch \|e\| switch (e) {
477	483	error.ConcurrencyUnavailable => @panic("failed to spawn concurrent fuzz input poller"),
478	484	};
479	485	fuzz_runner.input_poller = future;
...	...	@@ -481,17 +487,20 @@ var fuzz_runner: if (builtin.fuzz) struct {
481	487
482	488	export fn runner_stop_input_poller() void {
483	489	@disableInstrumentation();
484		assert(fuzz_runner.input_poller.cancel(fuzz_runner.io) == error.Canceled);
	490	const io = fuzz_runner.threaded_io.io();
	491	assert(fuzz_runner.input_poller.cancel(io) == error.Canceled);
485	492	}
486	493
487	494	export fn runner_futex_wait(ptr: *const u32, expected: u32) bool {
488	495	@disableInstrumentation();
489		return fuzz_runner.io.futexWait(u32, ptr, expected) == error.Canceled;
	496	const io = fuzz_runner.threaded_io.io();
	497	return io.futexWait(u32, ptr, expected) == error.Canceled;
490	498	}
491	499
492	500	export fn runner_futex_wake(ptr: *const u32, waiters: u32) void {
493	501	@disableInstrumentation();
494		fuzz_runner.io.futexWake(u32, ptr, waiters);
	502	const io = fuzz_runner.threaded_io.io();
	503	io.futexWake(u32, ptr, waiters);
495	504	}
496	505
497	506	fn inputPoller() Io.Cancelable!void {

src/InternPool.zig-11

...	...	@@ -5987,17 +5987,6 @@ pub const Alignment = enum(u6) {
5987	5987	return n + 1;
5988	5988	}
5989	5989
5990		pub fn toStdMem(a: Alignment) std.mem.Alignment {
5991		assert(a != .none);
5992		return @fromBackingInt(@intCast(@backingInt(a)));
5993		}
5994
5995		pub fn fromStdMem(a: std.mem.Alignment) Alignment {
5996		const r: Alignment = @fromBackingInt(@intCast(@backingInt(a)));
5997		assert(r != .none);
5998		return r;
5999		}
6000
6001	5990	pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment {
6002	5991	return @fromBackingInt(@intCast(@backingInt(a)));
6003	5992	}

src/link/Coff.zig+291-328

...	...	@@ -21,6 +21,7 @@ const Zcu = @import("../Zcu.zig");
21	21	const ModuleDefinition = @import("../libs/mingw/def.zig").ModuleDefinition;
22	22	const implib = @import("../libs/mingw/implib.zig");
23	23	const Path = std.Build.Cache.Path;
	24	const Alignment = MappedFile.Alignment;
24	25
25	26	base: link.File,
26	27	options: link.File.OpenOptions,
...	...	@@ -532,10 +533,10 @@ pub const Member = struct {
532	533	errdefer _ = coff.export_table.entries.pop();
533	534
534	535	_, const old_size = Node.known.longnames_member.location(&coff.mf).resolve(&coff.mf);
535		const new_size = old_size + name.len + 1;
	536	const new_size = Alignment.@"4".forward(old_size + name.len + 1);
536	537	assert(new_size < comptime try std.math.powi(u64, 10, max_name_len - 1));
537	538
538		try Node.known.longnames_member.resize(&coff.mf, gpa, new_size);
	539	try Node.known.longnames_member.resizeLeaf(&coff.mf, gpa, new_size);
539	540	const name_table_slice = Node.known.longnames_member.slice(&coff.mf);
540	541	const name_slice = name_table_slice[@intCast(old_size)..][0 .. name.len + 1];
541	542	@memcpy(name_slice[0..name.len], name);
...	...	@@ -602,7 +603,7 @@ pub const Member = struct {
602	603	};
603	604
604	605	pub const LongNamesTable = struct {
605		ni: MappedFile.Node.Index = .none,
	606	ni: MappedFile.Node.Index.Optional = .none,
606	607	entries: std.array_hash_map.Auto(void, Entry),
607	608
608	609	pub const Entry = struct {
...	...	@@ -832,7 +833,7 @@ pub const String = enum(u32) {
832	833
833	834	pub const Section = struct {
834	835	si: Symbol.Index,
835		relocation_table_ni: MappedFile.Node.Index,
	836	relocation_table_ni: MappedFile.Node.Index.Optional,
836	837
837	838	pub const RelocationIndex = enum(u16) {
838	839	none,
...	...	@@ -855,7 +856,7 @@ pub const Section = struct {
855	856	sn: Symbol.SectionNumber,
856	857	) ?*align(2) std.coff.Relocation {
857	858	if (sri == .none) return null;
858		const table_slice = sn.section(coff).relocation_table_ni.slice(&coff.mf);
	859	const table_slice = sn.section(coff).relocation_table_ni.unwrap().?.slice(&coff.mf);
859	860	return @ptrCast(@alignCast(&table_slice[@as(u32, sri.unwrap().?) * std.coff.Relocation.sizeOf()]));
860	861	}
861	862	};
...	...	@@ -891,7 +892,7 @@ const SpecialSymbol = enum {
891	892	};
892	893
893	894	pub const Symbol = struct {
894		ni: MappedFile.Node.Index,
	895	ni: MappedFile.Node.Index.Optional,
895	896	rva: u32,
896	897	value: std.meta.BareUnion(Symbol.Value),
897	898	extra: std.meta.BareUnion(Symbol.Extra),
...	...	@@ -986,7 +987,7 @@ pub const Symbol = struct {
986	987	pub fn nodeOffset(sym: const Symbol, coff: Coff) u32 {
987	988	return switch (sym.flags.value_tag) {
988	989	.node_offset => offset: {
989		assert(switch (coff.getNode(sym.ni)) {
	990	assert(switch (coff.getNode(sym.ni.unwrap().?)) {
990	991	// Separate nodes are not created for these entries per-symbol
991	992	.input_section, .import_address_table => true,
992	993	else => false,
...	...	@@ -1052,9 +1053,7 @@ pub const Symbol = struct {
1052	1053	}
1053	1054
1054	1055	pub fn node(si: Symbol.Index, coff: *Coff) MappedFile.Node.Index {
1055		const ni = si.get(coff).ni;
1056		assert(ni != .none);
1057		return ni;
	1056	return si.get(coff).ni.unwrap().?;
1058	1057	}
1059	1058
1060	1059	pub fn sti(si: Symbol.Index, coff: *Coff) SymbolTable.Index {
...	...	@@ -1075,7 +1074,7 @@ pub const Symbol = struct {
1075	1074
1076	1075	pub fn flushMoved(si: Symbol.Index, coff: *Coff) !void {
1077	1076	const sym = si.get(coff);
1078		sym.rva = coff.computeNodeRva(sym.ni) + sym.nodeOffset(coff);
	1077	sym.rva = coff.computeNodeRva(sym.ni.unwrap().?) + sym.nodeOffset(coff);
1079	1078	try si.applyLocationRelocs(coff);
1080	1079	try si.applyTargetRelocs(coff, .none);
1081	1080
...	...	@@ -1199,12 +1198,11 @@ pub const Reloc = extern struct {
1199	1198
1200	1199	pub fn apply(reloc: Reloc, coff: Coff) !void {
1201	1200	const loc_sym = reloc.loc.get(coff);
1202		switch (loc_sym.ni) {
1203		.none => return,
1204		else => \|ni\| if (ni.hasMoved(&coff.mf)) return,
1205		}
1206	1201
1207		const loc_slice = loc_sym.ni.slice(&coff.mf)[@intCast(reloc.offset)..];
	1202	const loc_sym_ni = loc_sym.ni.unwrap() orelse return;
	1203	if (loc_sym_ni.hasMoved(&coff.mf)) return;
	1204
	1205	const loc_slice = loc_sym_ni.slice(&coff.mf)[@intCast(reloc.offset)..];
1208	1206	const target_endian = coff.targetEndian();
1209	1207	const target_machine = coff.targetLoad(&coff.headerPtr().machine);
1210	1208
...	...	@@ -1331,9 +1329,12 @@ pub const Reloc = extern struct {
1331	1329	}
1332	1330
1333	1331	const target_sym = reloc.target.get(coff);
1334		const is_abs = switch (target_sym.ni) {
1335		.none => if (target_sym.section_number == .ABSOLUTE) true else return,
1336		else => \|ni\| if (ni.hasMoved(&coff.mf)) return else false,
	1332	const is_abs = if (target_sym.ni.unwrap()) \|ni\| is_abs: {
	1333	if (ni.hasMoved(&coff.mf)) return;
	1334	break :is_abs false;
	1335	} else is_abs: {
	1336	if (target_sym.section_number != .ABSOLUTE) return;
	1337	break :is_abs true;
1337	1338	};
1338	1339
1339	1340	const target_rva = target_sym.rva +% @as(u64, @bitCast(reloc.addend));
...	...	@@ -1573,7 +1574,7 @@ fn create(
1573	1574	33...64 => .@"PE32+",
1574	1575	else => return error.UnsupportedCOFFArchitecture,
1575	1576	};
1576		const section_align: std.mem.Alignment = switch (machine) {
	1577	const section_align: Alignment = switch (machine) {
1577	1578	.AMD64, .I386 => @fromBackingInt(@intCast(12)),
1578	1579	.SH3, .SH3DSP, .SH4, .SH5 => @fromBackingInt(@intCast(12)),
1579	1580	.MIPS16, .MIPSFPU, .MIPSFPU16, .WCEMIPSV2 => @fromBackingInt(@intCast(12)),
...	...	@@ -1617,22 +1618,22 @@ fn create(
1617	1618	.entries = .empty,
1618	1619	},
1619	1620	.import_table = .{
1620		.ni = .none,
	1621	.ni = undefined,
1621	1622	.entries = .empty,
1622	1623	.iat_symbol_indices = .empty,
1623	1624	},
1624	1625	.export_table = .{
1625		.ni = .none,
1626		.export_directory_table_ni = .none,
	1626	.ni = undefined,
	1627	.export_directory_table_ni = undefined,
1627	1628	.export_address_table_si = .null,
1628		.name_pointer_table_ni = .none,
1629		.ordinal_table_ni = .none,
1630		.name_table_ni = .none,
	1629	.name_pointer_table_ni = undefined,
	1630	.ordinal_table_ni = undefined,
	1631	.name_table_ni = undefined,
1631	1632	.entries = .empty,
1632	1633	},
1633	1634	.symbol_table = .{
1634		.ni = .none,
1635		.strings_ni = .none,
	1635	.ni = undefined,
	1636	.strings_ni = undefined,
1636	1637	.strings = .empty,
1637	1638	.symbols = .empty,
1638	1639	.pending_symbol_index = 0,
...	...	@@ -1794,13 +1795,13 @@ fn initHeaders(
1794	1795	minor_subsystem_version: u16,
1795	1796	magic: std.coff.OptionalHeader.Magic,
1796	1797	subsystem: std.coff.Subsystem,
1797		section_align: std.mem.Alignment,
	1798	section_align: Alignment,
1798	1799	file_name: []const u8,
1799	1800	) !void {
1800	1801	const comp = coff.base.comp;
1801	1802	const gpa = comp.gpa;
1802	1803	const target_endian = coff.targetEndian();
1803		const file_align: std.mem.Alignment = comptime .fromByteUnits(default_file_alignment);
	1804	const file_align: Alignment = comptime .fromByteUnits(default_file_alignment);
1804	1805	const is_image = coff.isImage();
1805	1806	const is_archive = coff.isArchive();
1806	1807	const target = &comp.root_mod.resolved_target.result;
...	...	@@ -1839,34 +1840,20 @@ fn initHeaders(
1839	1840	coff.nodes.appendAssumeCapacity(.file);
1840	1841
1841	1842	const header_ni = Node.known.header;
1842		assert(header_ni == try coff.mf.addOnlyChildNode(gpa, Node.known.file, .{
	1843	assert(header_ni == try Node.known.file.addOnlyHeaderChild(&coff.mf, gpa, .{
1843	1844	.alignment = coff.mf.flags.block_size,
1844		.fixed = true,
1845	1845	}));
1846	1846	coff.nodes.appendAssumeCapacity(.header);
1847	1847
1848		const signature_ni = Node.known.signature;
1849		assert(signature_ni == try coff.mf.addLastChildNode(gpa, if (is_image or !is_archive) header_ni else Node.known.file, .{
1850		.size = if (is_image)
1851		msdos_stub.len + std.coff.pe_signature.len
1852		else if (is_archive)
1853		std.coff.archive_signature.len
1854		else
1855		0,
1856		.alignment = .@"4",
1857		.fixed = true,
1858		}));
1859		coff.nodes.appendAssumeCapacity(.signature);
1860
1861		const signature_slice = signature_ni.slice(&coff.mf);
1862		if (is_image) {
1863		@memcpy(signature_slice[0..msdos_stub.len], &msdos_stub);
1864		@memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature);
1865		} else if (is_archive) {
	1848	const coff_parent_ni: MappedFile.Node.Index = if (is_archive) parent: {
	1849	assert(try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{
	1850	.size = std.coff.archive_signature.len,
	1851	.alignment = .@"4",
	1852	}) == Node.known.signature);
	1853	coff.nodes.appendAssumeCapacity(.signature);
	1854	const signature_slice = Node.known.signature.slice(&coff.mf);
1866	1855	@memcpy(signature_slice, std.coff.archive_signature);
1867		}
1868	1856
1869		const opt_coff_parent_ni = if (is_archive) parent: {
1870	1857	const initial_member_count = Member.Index.known_count + @intFromBool(comp.zcu != null);
1871	1858	try coff.members.ensureTotalCapacity(gpa, initial_member_count);
1872	1859
...	...	@@ -1892,46 +1879,54 @@ fn initHeaders(
1892	1879	const zcu_member = zcu_mi.get(coff);
1893	1880	try zcu_member.initHeader(coff, zcu.main_mod.fully_qualified_name, timestamp);
1894	1881
	1882	assert(try zcu_member.content_ni.addOnlyHeaderChild(&coff.mf, gpa, .{
	1883	.size = @sizeOf(std.coff.Header),
	1884	.alignment = .@"4",
	1885	}) == Node.known.coff_header);
	1886	coff.nodes.appendAssumeCapacity(.coff_header);
	1887
1895	1888	break :parent zcu_member.content_ni;
1896	1889	}
1897	1890
	1891	// If we're not generating any code, no more known nodes are used
	1892
1898	1893	// These placeholder nodes are placed before the first member - if there are
1899	1894	// no other members then the last linker member (longnames) needs to expand
1900	1895	// to fill the padding at the end of the file.
1901		assert(Node.known.zcu_member_header == try coff.mf.addNodeAfter(gpa, Node.known.header, .{}));
1902		assert(Node.known.zcu_member == try coff.mf.addNodeAfter(gpa, Node.known.header, .{}));
1903		coff.nodes.appendAssumeCapacity(.placeholder);
1904		coff.nodes.appendAssumeCapacity(.placeholder);
	1896	while (coff.nodes.len < Node.known_count) {
	1897	_ = try Node.known.header.addHeaderChildAfter(&coff.mf, gpa, .none, .{});
	1898	coff.nodes.appendAssumeCapacity(.placeholder);
	1899	}
1905	1900
1906		break :parent null;
	1901	return;
1907	1902	} else parent: {
	1903	assert(try header_ni.addOnlyHeaderChild(&coff.mf, gpa, .{
	1904	.size = if (is_image) msdos_stub.len + std.coff.pe_signature.len else 0,
	1905	.alignment = .@"4",
	1906	}) == Node.known.signature);
	1907	coff.nodes.appendAssumeCapacity(.signature);
	1908	if (is_image) {
	1909	const signature_slice = Node.known.signature.slice(&coff.mf);
	1910	@memcpy(signature_slice[0..msdos_stub.len], &msdos_stub);
	1911	@memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature);
	1912	}
	1913
1908	1914	// TODO: Not ideal to have this many placeholder nodes - use two distinct `Node.known` types?
1909	1915	while (true) {
1910		const placeholder_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{});
	1916	const placeholder_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .none, .{});
1911	1917	coff.nodes.appendAssumeCapacity(.placeholder);
1912	1918	if (placeholder_ni == Node.known.zcu_member) break;
1913	1919	}
1914	1920
1915		break :parent Node.known.header;
1916		};
	1921	assert(try header_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.signature), .{
	1922	.size = @sizeOf(std.coff.Header),
	1923	.alignment = .@"4",
	1924	}) == Node.known.coff_header);
	1925	coff.nodes.appendAssumeCapacity(.coff_header);
1917	1926
1918		const coff_parent_ni = opt_coff_parent_ni orelse {
1919		// If we're not generating any code, no more known nodes are used
1920		while (coff.nodes.len < Node.known_count) {
1921		_ = try coff.mf.addNodeAfter(gpa, Node.known.header, .{});
1922		coff.nodes.appendAssumeCapacity(.placeholder);
1923		}
1924
1925		return;
	1927	break :parent header_ni;
1926	1928	};
1927	1929
1928		const coff_header_ni = Node.known.coff_header;
1929		assert(coff_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{
1930		.size = @sizeOf(std.coff.Header),
1931		.alignment = .@"4",
1932		.fixed = true,
1933		}));
1934		coff.nodes.appendAssumeCapacity(.coff_header);
1935	1930	{
1936	1931	const coff_header = coff.headerPtr();
1937	1932	coff_header.* = .{
...	...	@@ -1954,10 +1949,9 @@ fn initHeaders(
1954	1949	}
1955	1950
1956	1951	const optional_header_ni = Node.known.optional_header;
1957		assert(optional_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{
	1952	assert(optional_header_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.coff_header), .{
1958	1953	.size = optional_header_size,
1959	1954	.alignment = .@"4",
1960		.fixed = true,
1961	1955	}));
1962	1956	coff.nodes.appendAssumeCapacity(.optional_header);
1963	1957	if (is_image) {
...	...	@@ -2066,10 +2060,9 @@ fn initHeaders(
2066	2060	}
2067	2061
2068	2062	const data_directories_ni = Node.known.data_directories;
2069		assert(data_directories_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{
	2063	assert(data_directories_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(optional_header_ni), .{
2070	2064	.size = data_directories_size,
2071	2065	.alignment = .@"4",
2072		.fixed = true,
2073	2066	}));
2074	2067	coff.nodes.appendAssumeCapacity(.data_directories);
2075	2068	if (is_image) {
...	...	@@ -2082,9 +2075,8 @@ fn initHeaders(
2082	2075	}
2083	2076
2084	2077	const section_table_ni = Node.known.section_table;
2085		assert(section_table_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{
	2078	assert(section_table_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(data_directories_ni), .{
2086	2079	.alignment = .@"4",
2087		.fixed = true,
2088	2080	}));
2089	2081	coff.nodes.appendAssumeCapacity(.section_table);
2090	2082
...	...	@@ -2092,16 +2084,14 @@ fn initHeaders(
2092	2084
2093	2085	if (!is_image) {
2094	2086	// TODO: These two nodes could be inside one movable node?
2095		coff.symbol_table.ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{
	2087	coff.symbol_table.ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(section_table_ni), .{
2096	2088	.alignment = .@"2",
2097		.fixed = true,
2098	2089	.moved = true,
2099	2090	});
2100	2091	coff.nodes.appendAssumeCapacity(.symbol_table);
2101	2092
2102		coff.symbol_table.strings_ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{
	2093	coff.symbol_table.strings_ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(coff.symbol_table.ni), .{
2103	2094	.size = @sizeOf(u32),
2104		.fixed = true,
2105	2095	.resized = true,
2106	2096	});
2107	2097	coff.nodes.appendAssumeCapacity(.string_table);
...	...	@@ -2148,15 +2138,14 @@ fn initHeaders(
2148	2138	}
2149	2139
2150	2140	// TODO: Lazily initialize this instead, avoid the extra logic for this in flushMoved / flushResized
2151		coff.import_table.ni = try coff.mf.addLastChildNode(
2152		gpa,
2153		(try coff.objectSectionMapIndex(
2154		.@".idata",
2155		coff.mf.flags.block_size,
2156		.{ .read = true, .initialized = true },
2157		)).symbol(coff).node(coff),
2158		.{ .alignment = .@"4" },
2159		);
	2141	const import_table_parent_ni = (try coff.objectSectionMapIndex(
	2142	.@".idata",
	2143	coff.mf.flags.block_size,
	2144	.{ .read = true, .initialized = true },
	2145	)).symbol(coff).node(coff);
	2146	coff.import_table.ni = try import_table_parent_ni.addFloatingChild(&coff.mf, gpa, .{
	2147	.alignment = .@"4",
	2148	});
2160	2149	coff.nodes.appendAssumeCapacity(.import_directory_table);
2161	2150
2162	2151	coff.export_table.ni = (try coff.pseudoSectionMapIndex(
...	...	@@ -2165,15 +2154,10 @@ fn initHeaders(
2165	2154	.{ .read = true, .initialized = true },
2166	2155	)).symbol(coff).node(coff);
2167	2156
2168		coff.export_table.export_directory_table_ni = try coff.mf.addLastChildNode(
2169		gpa,
2170		coff.export_table.ni,
2171		.{
2172		.size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1,
2173		.moved = true,
2174		.fixed = true,
2175		},
2176		);
	2157	coff.export_table.export_directory_table_ni = try coff.export_table.ni.addHeaderChildAfter(&coff.mf, gpa, coff.export_table.ni.last(&coff.mf), .{
	2158	.size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1,
	2159	.moved = true,
	2160	});
2177	2161	coff.nodes.appendAssumeCapacity(.export_directory_table);
2178	2162
2179	2163	const name_index = @sizeOf(std.coff.ExportDirectoryTable);
...	...	@@ -2181,7 +2165,7 @@ fn initHeaders(
2181	2165	@memcpy(table_slice[name_index..][0..file_name.len], file_name[0..file_name.len]);
2182	2166	@memset(table_slice[name_index + file_name.len ..], 0);
2183	2167
2184		const export_address_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{
	2168	const export_address_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{
2185	2169	.alignment = .of(std.coff.ExportAddressTableEntry),
2186	2170	.moved = true,
2187	2171	});
...	...	@@ -2191,25 +2175,25 @@ fn initHeaders(
2191	2175	coff.export_table.export_address_table_si = coff.addSymbolAssumeCapacity();
2192	2176
2193	2177	const export_address_table_sym = coff.export_table.export_address_table_si.get(coff);
2194		export_address_table_sym.ni = export_address_table_ni;
	2178	export_address_table_sym.ni = .wrap(export_address_table_ni);
2195	2179	assert(export_address_table_sym.loc_relocs == .none);
2196	2180	export_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len));
2197	2181	export_address_table_sym.section_number =
2198	2182	coff.getNode(coff.export_table.ni).pseudo_section.symbol(coff).get(coff).section_number;
2199	2183
2200		coff.export_table.name_pointer_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{
	2184	coff.export_table.name_pointer_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{
2201	2185	.alignment = .of(std.coff.ExportNamePointerTableEntry),
2202	2186	.moved = true,
2203	2187	});
2204	2188	coff.nodes.appendAssumeCapacity(.export_name_pointer_table);
2205	2189
2206		coff.export_table.ordinal_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{
	2190	coff.export_table.ordinal_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{
2207	2191	.alignment = .of(std.coff.ExportOrdinalTableEntry),
2208	2192	.moved = true,
2209	2193	});
2210	2194	coff.nodes.appendAssumeCapacity(.export_ordinal_table);
2211	2195
2212		coff.export_table.name_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{
	2196	coff.export_table.name_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{
2213	2197	.alignment = .of(u8),
2214	2198	.moved = true,
2215	2199	});
...	...	@@ -2260,7 +2244,7 @@ pub fn initBuiltins(coff: *Coff) !void {
2260	2244	if (coff.isImage()) {
2261	2245	const si = try coff.globalSymbol(.{ .name = "__ImageBase", .type = .data });
2262	2246	const sym = si.get(coff);
2263		sym.ni = Node.known.header;
	2247	sym.ni = .wrap(Node.known.header);
2264	2248	}
2265	2249
2266	2250	defer coff.flushSectionMerges() catch unreachable;
...	...	@@ -2302,14 +2286,13 @@ pub fn initBuiltins(coff: *Coff) !void {
2302	2286	const list_len_si = try coff.globalSymbol(.{ .name = list.global, .type = .data });
2303	2287	const list_len_sym = list_len_si.get(coff);
2304	2288	list_len_sym.setExtra(.{ .size = addr_info.size });
2305		list_len_sym.ni = try coff.mf.addFirstChildNode(gpa, start_sym.ni, .{
	2289	list_len_sym.ni = .wrap(try start_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{
2306	2290	.size = addr_info.size,
2307		.fixed = true,
2308		});
	2291	}));
2309	2292	coff.nodes.appendAssumeCapacity(.{ .builtin = list_len_si });
2310	2293	list_len_sym.section_number = start_sym.section_number;
2311	2294
2312		const start_slice = list_len_sym.ni.slice(&coff.mf);
	2295	const start_slice = list_len_sym.ni.unwrap().?.slice(&coff.mf);
2313	2296	switch (addr_info.magic) {
2314	2297	_ => unreachable,
2315	2298	inline .PE32, .@"PE32+" => \|t\| {
...	...	@@ -2324,14 +2307,13 @@ pub fn initBuiltins(coff: *Coff) !void {
2324	2307	const list_end_si = coff.addSymbolAssumeCapacity();
2325	2308	const list_end_sym = list_end_si.get(coff);
2326	2309	list_end_sym.setExtra(.{ .size = addr_info.size });
2327		list_end_sym.ni = try coff.mf.addFirstChildNode(gpa, end_sym.ni, .{
	2310	list_end_sym.ni = .wrap(try end_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{
2328	2311	.size = addr_info.size,
2329		.fixed = true,
2330		});
	2312	}));
2331	2313	coff.nodes.appendAssumeCapacity(.{ .builtin = list_end_si });
2332	2314	list_end_sym.section_number = start_sym.section_number;
2333	2315
2334		@memset(list_end_sym.ni.slice(&coff.mf), 0);
	2316	@memset(list_end_sym.ni.unwrap().?.slice(&coff.mf), 0);
2335	2317
2336	2318	try list_len_si.flushMoved(coff);
2337	2319	try list_end_si.flushMoved(coff);
...	...	@@ -2387,7 +2369,7 @@ fn getNode(coff: *const Coff, ni: MappedFile.Node.Index) Node {
2387	2369	}
2388	2370	fn computeNodeRva(coff: *Coff, ni: MappedFile.Node.Index) u32 {
2389	2371	const parent_rva = parent_rva: {
2390		const parent_si = switch (coff.getNode(ni.parent(&coff.mf))) {
	2372	const parent_si = switch (coff.getNode(ni.parent(&coff.mf).unwrap().?)) {
2391	2373	.file,
2392	2374	.header,
2393	2375	.signature,
...	...	@@ -2452,11 +2434,11 @@ fn computeSymbolSectionOffset(
2452	2434	relative_to: enum { image, pseudo },
2453	2435	) u32 {
2454	2436	var section_offset: u32 = sym.nodeOffset(coff);
2455		var parent_ni = sym.ni;
	2437	var parent_ni = sym.ni.unwrap().?;
2456	2438	while (true) {
2457	2439	const offset, _ = parent_ni.location(&coff.mf).resolve(&coff.mf);
2458	2440	section_offset += @intCast(offset);
2459		parent_ni = parent_ni.parent(&coff.mf);
	2441	parent_ni = parent_ni.parent(&coff.mf).unwrap().?;
2460	2442	switch (coff.getNode(parent_ni)) {
2461	2443	else => unreachable,
2462	2444	.image_section => break,
...	...	@@ -2475,7 +2457,7 @@ pub inline fn targetEndian(_: *const Coff) std.lang.Endian {
2475	2457
2476	2458	fn targetAddrInfo(coff: *Coff) struct {
2477	2459	size: u8,
2478		alignment: std.mem.Alignment,
	2460	alignment: Alignment,
2479	2461	magic: std.coff.OptionalHeader.Magic,
2480	2462	} {
2481	2463	const magic = coff.targetLoad(&coff.optionalHeaderStandardPtr().magic);
...	...	@@ -2741,7 +2723,7 @@ fn getOrPutSymbolName(coff: *Coff, name: []const u8, opt_string: ?String) !Symbo
2741	2723	const string_index = coff.symbol_table.strings_ni.location(&coff.mf).resolve(&coff.mf)[1];
2742	2724	string_gop.value_ptr.* = @fromBackingInt(@intCast(string_index));
2743	2725
2744		try coff.symbol_table.strings_ni.resize(&coff.mf, gpa, string_index + name.len + 1);
	2726	try coff.symbol_table.strings_ni.resizeLeaf(&coff.mf, gpa, string_index + name.len + 1);
2745	2727	const slice = coff.symbol_table.strings_ni.slice(&coff.mf);
2746	2728	@memcpy(slice[@intCast(string_index)..][0..name.len], name);
2747	2729	slice[@intCast(string_index + name.len)] = 0;
...	...	@@ -2875,9 +2857,9 @@ fn navSection(
2875	2857	switch (nav_resolved.@"linksection") {
2876	2858	.none => coff.mf.flags.block_size,
2877	2859	else => switch (nav_resolved.@"align") {
2878		.none => Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu),
2879		else => \|alignment\| alignment,
2880		}.toStdMem(),
	2860	.none => .fromIp(Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu)),
	2861	else => \|a\| .fromIp(a),
	2862	},
2881	2863	},
2882	2864	attributes,
2883	2865	)).symbol(coff);
...	...	@@ -2966,24 +2948,22 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind,
2966	2948	const comp = coff.base.comp;
2967	2949	const gpa = comp.gpa;
2968	2950
2969		// TODO: These two nodes could to be inside a movable node if kind == .coff\|.import
2970		const header_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{
	2951	const header_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, Node.known.file.last(&coff.mf), .{
2971	2952	.size = @sizeOf(std.coff.ArchiveMemberHeader),
2972	2953	.alignment = .@"2",
2973		.fixed = true,
2974	2954	.moved = true,
2975	2955	});
2976	2956
2977		const content_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{
2978		// The actual alignment required by the spec is 2, but to allow aligned access to
2979		// the various COFF data structures in-place during linking we overalign
2980		.alignment = switch (kind) {
2981		.coff => .@"4",
2982		else => .@"2",
2983		},
2984		.size = size,
	2957	// The actual alignment required by the spec is 2, but to allow aligned access to
	2958	// the various COFF data structures in-place during linking we overalign
	2959	const content_align: Alignment = switch (kind) {
	2960	.first_linker, .second_linker, .longnames, .coff => .@"4",
	2961	else => .@"2",
	2962	};
	2963	const content_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{
	2964	.alignment = content_align,
	2965	.size = content_align.forward(size),
2985	2966	.resized = size > 0,
2986		.fixed = true,
2987	2967	});
2988	2968
2989	2969	const mi: Member.Index = @fromBackingInt(@intCast(coff.members.items.len));
...	...	@@ -3009,7 +2989,7 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind,
3009	2989	const old_size = Node.known.second_linker_member.location(&coff.mf).resolve(&coff.mf)[1];
3010	2990	const old_header_size = new_num_members * @sizeOf(u32);
3011	2991	const trailing_size: usize = @intCast(old_size - old_header_size);
3012		try Node.known.second_linker_member.resize(&coff.mf, gpa, old_size + @sizeOf(u32));
	2992	try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, old_size + @sizeOf(u32));
3013	2993
3014	2994	const slice = Node.known.second_linker_member.slice(&coff.mf);
3015	2995	@memmove(
...	...	@@ -3047,7 +3027,7 @@ fn appendMemberSymbolString(
3047	3027	name: []const u8,
3048	3028	offset: u64,
3049	3029	) !void {
3050		try strings_ni.resize(&coff.mf, coff.base.comp.gpa, new_size);
	3030	try strings_ni.resizeLeaf(&coff.mf, coff.base.comp.gpa, new_size);
3051	3031	const name_slice = strings_ni.slice(&coff.mf)[offset..][0 .. name.len + 1];
3052	3032	@memcpy(name_slice[0..name.len], name);
3053	3033	name_slice[name.len] = 0;
...	...	@@ -3080,7 +3060,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void {
3080	3060	{
3081	3061	const old_header_size: usize = @intCast(@sizeOf(u32) + @backingInt(mfli) * @sizeOf(u32));
3082	3062	const new_header_size: usize = @intCast(old_header_size + @sizeOf(u32));
3083		try Node.known.first_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size);
	3063	try Node.known.first_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size));
3084	3064
3085	3065	const slice = Node.known.first_linker_member.slice(&coff.mf);
3086	3066	@memmove(slice[new_header_size..][0..coff.lib_string_len], slice[old_header_size..][0..coff.lib_string_len]);
...	...	@@ -3094,7 +3074,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void {
3094	3074	const num_members = coff.targetLoad(coff.secondLinkerMemberNumMembersPtr());
3095	3075	const old_header_size = 2 * @sizeOf(u32) + num_members * @sizeOf(u32) + @backingInt(mfli) * @sizeOf(u16);
3096	3076	const new_header_size = old_header_size + @sizeOf(u16);
3097		try Node.known.second_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size);
	3077	try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size));
3098	3078
3099	3079	const old_needs_sort = coff.pending_members.get(Member.Index.second) != null;
3100	3080	const needs_sort = old_needs_sort or (if (coff.lib_string_table.items.len > 0)
...	...	@@ -3151,7 +3131,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void {
3151	3131	else
3152	3132	.NULL,
3153	3133	};
3154		} else blk: switch (coff.getNode(sym.ni)) {
	3134	} else blk: switch (coff.getNode(sym.ni.unwrap().?)) {
3155	3135	.image_section => .{
3156	3136	try coff.getOrPutSymbolName(&sym.section_number.header(coff).name, null),
3157	3137	1,
...	...	@@ -3192,7 +3172,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void {
3192	3172	};
3193	3173	},
3194	3174	else => {
3195		log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni)), si });
	3175	log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni.unwrap().?)), si });
3196	3176	unreachable;
3197	3177	},
3198	3178	};
...	...	@@ -3201,7 +3181,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void {
3201	3181	const new_num_symbols = old_num_symbols + 1 + num_aux_symbols;
3202	3182	coff.targetStore(&coff.headerPtr().number_of_symbols, new_num_symbols);
3203	3183
3204		try coff.symbol_table.ni.resize(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf());
	3184	try coff.symbol_table.ni.resizeLeaf(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf());
3205	3185
3206	3186	sti.* = .wrap(old_num_symbols);
3207	3187	si.flushSymbolTableIndex(coff);
...	...	@@ -3255,13 +3235,13 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void {
3255	3235	std.mem.byteSwapAllFieldsAligned(std.coff.WeakExternalDefinition, .@"2", aux_ptr);
3256	3236
3257	3237	break :aux_init;
3258		} else switch (coff.getNode(sym.ni)) {
	3238	} else switch (coff.getNode(sym.ni.unwrap().?)) {
3259	3239	.image_section => \|sec_si\| {
3260	3240	assert(si == sec_si);
3261	3241	const header = sym.section_number.header(coff);
3262	3242	const aux_ptr = coff.symbolTableSectionAuxEntryPtr(sti.*).?;
3263	3243	aux_ptr.* = .{
3264		.length = @intCast(sym.ni.location(&coff.mf).resolve(&coff.mf)[1]),
	3244	.length = @intCast(sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[1]),
3265	3245	.number_of_relocations = header.number_of_relocations,
3266	3246	.number_of_linenumbers = header.number_of_linenumbers,
3267	3247	.checksum = 0,
...	...	@@ -3288,7 +3268,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void {
3288	3268	.ABSOLUTE,
3289	3269	.DEBUG,
3290	3270	=> unreachable,
3291		else => switch (coff.getNode(sym.ni)) {
	3271	else => switch (coff.getNode(sym.ni.unwrap().?)) {
3292	3272	.image_section => 0,
3293	3273	else => coff.computeSymbolSectionOffset(sym, .image),
3294	3274	},
...	...	@@ -3364,13 +3344,13 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S
3364	3344	const section_index = coff.targetLoad(&coff_header.number_of_sections);
3365	3345	const section_table_len = section_index + 1;
3366	3346	coff.targetStore(&coff_header.number_of_sections, section_table_len);
3367		try Node.known.section_table.resize(
	3347	try Node.known.section_table.resizeLeaf(
3368	3348	&coff.mf,
3369	3349	gpa,
3370	3350	@sizeOf(std.coff.SectionHeader) * section_table_len,
3371	3351	);
3372	3352
3373		const ni = try coff.mf.addLastChildNode(gpa, coff.sectionParent(), .{
	3353	const ni = try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{
3374	3354	.alignment = coff.mf.flags.block_size,
3375	3355	.moved = true,
3376	3356	.bubbles_moved = false,
...	...	@@ -3397,7 +3377,7 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S
3397	3377
3398	3378	{
3399	3379	const sym = si.get(coff);
3400		sym.ni = ni;
	3380	sym.ni = .wrap(ni);
3401	3381	sym.rva = rva;
3402	3382	sym.section_number = @fromBackingInt(@intCast(section_table_len));
3403	3383	}
...	...	@@ -3481,7 +3461,7 @@ const ObjectSectionAttributes = packed struct {
3481	3461	fn pseudoSectionMapIndex(
3482	3462	coff: *Coff,
3483	3463	name: String,
3484		alignment: std.mem.Alignment,
	3464	alignment: Alignment,
3485	3465	attributes: ObjectSectionAttributes,
3486	3466	) !Node.PseudoSectionMapIndex {
3487	3467	const gpa = coff.base.comp.gpa;
...	...	@@ -3506,11 +3486,11 @@ fn pseudoSectionMapIndex(
3506	3486
3507	3487	try coff.nodes.ensureUnusedCapacity(gpa, 1);
3508	3488	try coff.symbols.ensureUnusedCapacity(gpa, 1);
3509		const ni = try coff.mf.addLastChildNode(gpa, parent.node(coff), .{ .alignment = alignment });
	3489	const ni = try parent.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment });
3510	3490	const si = coff.addSymbolAssumeCapacity();
3511	3491	pseudo_section_gop.value_ptr.* = si;
3512	3492	const sym = si.get(coff);
3513		sym.ni = ni;
	3493	sym.ni = .wrap(ni);
3514	3494	sym.rva = coff.computeNodeRva(ni);
3515	3495	sym.section_number = parent.get(coff).section_number;
3516	3496	assert(sym.loc_relocs == .none);
...	...	@@ -3543,7 +3523,7 @@ fn objectSectionParentName(coff: *Coff, name: []const u8) []const u8 {
3543	3523	fn objectSectionMapIndex(
3544	3524	coff: *Coff,
3545	3525	name: String,
3546		alignment: std.mem.Alignment,
	3526	alignment: Alignment,
3547	3527	attributes: ObjectSectionAttributes,
3548	3528	) !Node.ObjectSectionMapIndex {
3549	3529	const gpa = coff.base.comp.gpa;
...	...	@@ -3565,31 +3545,28 @@ fn objectSectionMapIndex(
3565	3545	try coff.nodes.ensureUnusedCapacity(gpa, 1);
3566	3546	try coff.symbols.ensureUnusedCapacity(gpa, 1);
3567	3547	const parent_ni = parent.node(coff);
3568		var prev_ni: MappedFile.Node.Index = .none;
3569		var next_it = parent_ni.children(&coff.mf);
3570		while (next_it.next()) \|next_ni\| switch (std.mem.order(
3571		u8,
3572		name_slice,
3573		coff.getNode(next_ni).object_section.name(coff).toSlice(coff),
3574		)) {
3575		.lt => break,
3576		.eq => unreachable,
3577		.gt => prev_ni = next_ni,
3578		};
3579		const ni = switch (prev_ni) {
3580		.none => try coff.mf.addFirstChildNode(gpa, parent_ni, .{
3581		.alignment = alignment,
3582		.fixed = true,
3583		}),
3584		else => try coff.mf.addNodeAfter(gpa, prev_ni, .{
3585		.alignment = alignment,
3586		.fixed = true,
3587		}),
3588		};
	3548	var prev_oni: MappedFile.Node.Index.Optional = .none;
	3549	{
	3550	var child_oni = parent_ni.first(&coff.mf);
	3551	while (child_oni.unwrap()) \|child_ni\| : (child_oni = child_ni.next(&coff.mf)) {
	3552	switch (std.mem.order(
	3553	u8,
	3554	name_slice,
	3555	coff.getNode(child_ni).object_section.name(coff).toSlice(coff),
	3556	)) {
	3557	.lt => break,
	3558	.eq => unreachable,
	3559	.gt => prev_oni = .wrap(child_ni),
	3560	}
	3561	}
	3562	}
	3563	const ni = try parent_ni.addHeaderChildAfter(&coff.mf, gpa, prev_oni, .{
	3564	.alignment = alignment,
	3565	});
3589	3566	const si = coff.addSymbolAssumeCapacity();
3590	3567	object_section_gop.value_ptr.* = si;
3591	3568	const sym = si.get(coff);
3592		sym.ni = ni;
	3569	sym.ni = .wrap(ni);
3593	3570	sym.rva = coff.computeNodeRva(ni);
3594	3571	sym.section_number = parent.get(coff).section_number;
3595	3572	assert(sym.loc_relocs == .none);
...	...	@@ -3598,17 +3575,17 @@ fn objectSectionMapIndex(
3598	3575	break :sym sym;
3599	3576	} else object_section_gop.value_ptr.get(coff);
3600	3577
3601		const parent_ni = sym.ni.parent(&coff.mf);
	3578	const parent_ni = sym.ni.unwrap().?.parent(&coff.mf).unwrap().?;
3602	3579	const parent_alignment = parent_ni.alignment(&coff.mf);
3603	3580	if (alignment.compare(.gt, parent_alignment)) {
3604	3581	log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment });
3605		try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true });
	3582	try parent_ni.realign(&coff.mf, gpa, alignment);
3606	3583	}
3607	3584
3608		const old_alignment = sym.ni.alignment(&coff.mf);
	3585	const old_alignment = sym.ni.unwrap().?.alignment(&coff.mf);
3609	3586	if (alignment.compare(.gt, old_alignment)) {
3610	3587	log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment });
3611		try sym.ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true });
	3588	try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment);
3612	3589	}
3613	3590
3614	3591	try coff.verifyParentSectionAttributes(
...	...	@@ -3764,20 +3741,16 @@ fn addRelocAssumeCapacity(
3764	3741	if (coff.symbolTableSectionAuxEntryPtr(loc_sn.symbol(coff).sti(coff))) \|aux_ptr\|
3765	3742	coff.targetStore(&aux_ptr.number_of_relocations, new_num_relocations);
3766	3743
3767		if (section.relocation_table_ni == .none) {
3768		section.relocation_table_ni = try coff.mf.addLastChildNode(
3769		gpa,
3770		coff.sectionParent(),
3771		.{
3772		.size = new_size,
3773		.alignment = .@"2",
3774		.moved = true,
3775		.resized = true,
3776		},
3777		);
3778		coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn });
	3744	if (section.relocation_table_ni.unwrap()) \|relocation_table_ni\| {
	3745	try relocation_table_ni.resizeLeaf(&coff.mf, gpa, new_size);
3779	3746	} else {
3780		try section.relocation_table_ni.resize(&coff.mf, gpa, new_size);
	3747	section.relocation_table_ni = .wrap(try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{
	3748	.size = new_size,
	3749	.alignment = .@"2",
	3750	.moved = true,
	3751	.resized = true,
	3752	}));
	3753	coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn });
3781	3754	}
3782	3755
3783	3756	// TODO: These need to allocate from a free list, once deleting relocs from the table is supported
...	...	@@ -4581,7 +4554,7 @@ fn loadObject(
4581	4554	},
4582	4555	.SAME_SIZE => {
4583	4556	// TODO: Verify that this node isn't resized after creation
4584		_, const size = si.get(coff).ni.location(&coff.mf).resolve(&coff.mf);
	4557	_, const size = si.get(coff).ni.unwrap().?.location(&coff.mf).resolve(&coff.mf);
4585	4558	if (size == section.header.size_of_raw_data) {
4586	4559	symbol.si = si;
4587	4560	break :comdat .skip;
...	...	@@ -4598,9 +4571,9 @@ fn loadObject(
4598	4571	},
4599	4572	.EXACT_MATCH => {
4600	4573	const sym = si.get(coff);
4601		const existing_crc = switch (coff.getNode(sym.ni)) {
	4574	const existing_crc = switch (coff.getNode(sym.ni.unwrap().?)) {
4602	4575	.input_section => \|isi\| isi.inputSection(coff).crc,
4603		else => Crc32.hash(sym.ni.sliceConst(&coff.mf)),
	4576	else => Crc32.hash(sym.ni.unwrap().?.sliceConst(&coff.mf)),
4604	4577	};
4605	4578
4606	4579	if (existing_crc == section.comdat_crc) {
...	...	@@ -4666,7 +4639,7 @@ fn loadObject(
4666	4639
4667	4640	section.parent_si = (try coff.objectSectionMapIndex(
4668	4641	section.name,
4669		section.header.flags.ALIGN.alignment() orelse .@"1",
	4642	.fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1),
4670	4643	.fromFlags(section.header.flags),
4671	4644	)).symbol(coff);
4672	4645	}
...	...	@@ -4679,9 +4652,10 @@ fn loadObject(
4679	4652	for (sections) \|*section\| {
4680	4653	if (section.parent_si == .null) continue;
4681	4654
4682		const ni = try coff.mf.addLastChildNode(gpa, section.parent_si.node(coff), .{
4683		.size = section.header.size_of_raw_data,
4684		.alignment = section.header.flags.ALIGN.alignment() orelse .@"1",
	4655	const alignment: Alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1);
	4656	const ni = try section.parent_si.node(coff).addFloatingChild(&coff.mf, gpa, .{
	4657	.size = alignment.forward(section.header.size_of_raw_data),
	4658	.alignment = alignment,
4685	4659	.moved = true,
4686	4660	});
4687	4661	coff.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(coff.input_sections.items.len)) });
...	...	@@ -4691,7 +4665,7 @@ fn loadObject(
4691	4665	pending_symbols.values()[psi].si = section.si;
4692	4666
4693	4667	const sym = section.si.get(coff);
4694		sym.ni = ni;
	4668	sym.ni = .wrap(ni);
4695	4669	sym.section_number = section.parent_si.get(coff).section_number;
4696	4670
4697	4671	coff.input_sections.addOneAssumeCapacity().* = .{
...	...	@@ -4852,7 +4826,7 @@ fn loadObject(
4852	4826	}
4853	4827
4854	4828	if (section.comdat_psi.unwrap() == @as(u32, @intCast(i)))
4855		coff.getNode(section.si.get(coff).ni).input_section.inputSection(coff).comdat_si = symbol.si;
	4829	coff.getNode(section.si.get(coff).ni.unwrap().?).input_section.inputSection(coff).comdat_si = symbol.si;
4856	4830	}
4857	4831
4858	4832	if (symbol.weak_external_psi.unwrap()) \|weak_external_i\| {
...	...	@@ -4967,14 +4941,14 @@ fn loadObject(
4967	4941	const section = &sections[symbol.section_number.toIndex()];
4968	4942	include_section = section.comdat_result == .include;
4969	4943	if (include_section) {
4970		const isi = coff.getNode(section.si.get(coff).ni).input_section;
	4944	const isi = coff.getNode(section.si.get(coff).ni.unwrap().?).input_section;
4971	4945	isi.inputSection(coff).first_li = @fromBackingInt(@intCast(coff.input_symbols.items.len));
4972	4946	}
4973	4947	}
4974	4948	}
4975	4949
4976	4950	if (include_section) {
4977		assert(coff.getNode(symbol.si.get(coff).ni) == .input_section);
	4951	assert(coff.getNode(symbol.si.get(coff).ni.unwrap().?) == .input_section);
4978	4952	symbol.si.get(coff).setExtra(.{ .isli = @fromBackingInt(@intCast(coff.input_symbols.items.len)) });
4979	4953	coff.input_symbols.addOneAssumeCapacity().* = .{
4980	4954	.si = symbol.si,
...	...	@@ -5002,7 +4976,7 @@ fn failMultipleDefinitions(
5002	4976	var err = try coff.base.comp.link_diags.addErrorWithNotes(num_notes);
5003	4977	try err.addMsg("multiple definitions of '{s}'", .{name.toSlice(coff)});
5004	4978
5005		switch (coff.getNode(existing_si.get(coff).ni)) {
	4979	switch (coff.getNode(existing_si.get(coff).ni.unwrap().?)) {
5006	4980	.input_section => \|isi\| {
5007	4981	const other_ioi = isi.input(coff);
5008	4982	err.addNote("first seen in input '{f}{f}'", .{
...	...	@@ -5473,13 +5447,13 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde
5473	5447	const sec_si = try coff.navSection(zcu, nav.resolved.?);
5474	5448	try coff.nodes.ensureUnusedCapacity(gpa, 1);
5475	5449	if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1);
5476		const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{
5477		.alignment = zcu.navAlignment(nav_index).toStdMem(),
	5450	const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{
	5451	.alignment = .fromIp(zcu.navAlignment(nav_index)),
5478	5452	.moved = true,
5479	5453	});
5480	5454	coff.nodes.appendAssumeCapacity(.{ .nav = nmi });
5481	5455	const sym = si.get(coff);
5482		sym.ni = ni;
	5456	sym.ni = .wrap(ni);
5483	5457	sym.section_number = sec_si.get(coff).section_number;
5484	5458	},
5485	5459	else => si.deleteLocationRelocs(coff),
...	...	@@ -5490,7 +5464,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde
5490	5464	if (!isImage(coff) and sym.target_relocs != .none)
5491	5465	try coff.pendingSymbolTableEntry(si);
5492	5466
5493		break :ni sym.ni;
	5467	break :ni sym.ni.unwrap().?;
5494	5468	};
5495	5469
5496	5470	{
...	...	@@ -5512,21 +5486,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde
5512	5486	}
5513	5487
5514	5488	if (nav.resolved.?.@"linksection".unwrap()) \|_\| {
5515		try ni.resize(&coff.mf, gpa, si.get(coff).extra.size);
5516		var parent_ni = ni;
5517		while (true) {
5518		parent_ni = parent_ni.parent(&coff.mf);
5519		switch (coff.getNode(parent_ni)) {
5520		else => unreachable,
5521		.image_section, .pseudo_section => break,
5522		.object_section => {
5523		var child_it = parent_ni.reverseChildren(&coff.mf);
5524		const last_offset, const last_size =
5525		child_it.next().?.location(&coff.mf).resolve(&coff.mf);
5526		try parent_ni.resize(&coff.mf, gpa, last_offset + last_size);
5527		},
5528		}
5529		}
	5489	try ni.resizeLeaf(&coff.mf, gpa, si.get(coff).extra.size);
5530	5490	}
5531	5491	}
5532	5492
...	...	@@ -5542,10 +5502,11 @@ pub fn lowerUav(
5542	5502	try coff.pending_uavs.ensureUnusedCapacity(gpa, 1);
5543	5503	const umi = try coff.uavMapIndex(uav_val);
5544	5504	const si = umi.symbol(coff);
5545		if (switch (si.get(coff).ni) {
5546		.none => true,
5547		else => \|ni\| uav_align.toStdMem().order(ni.alignment(&coff.mf)).compare(.gt),
5548		}) {
	5505	const need_update: bool = update: {
	5506	const existing_ni = si.get(coff).ni.unwrap() orelse break :update true;
	5507	break :update Alignment.compare(.fromIp(uav_align), .gt, existing_ni.alignment(&coff.mf));
	5508	};
	5509	if (need_update) {
5549	5510	const gop = coff.pending_uavs.getOrPutAssumeCapacity(umi);
5550	5511	if (gop.found_existing) {
5551	5512	gop.value_ptr.alignment = gop.value_ptr.alignment.max(uav_align);
...	...	@@ -5597,22 +5558,22 @@ fn updateFuncInner(
5597	5558	if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1);
5598	5559	const mod = zcu.navFileScope(func.owner_nav).mod.?;
5599	5560	const target = &mod.resolved_target.result;
5600		const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{
	5561	const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{
5601	5562	.alignment = switch (nav.resolved.?.@"align") {
5602	5563	.none => switch (mod.optimize_mode) {
5603	5564	.debug,
5604	5565	.safe,
5605	5566	.fast,
5606		=> target_util.defaultFunctionAlignment(target),
5607		.small => target_util.minFunctionAlignment(target),
	5567	=> .fromIp(target_util.defaultFunctionAlignment(target)),
	5568	.small => .fromIp(target_util.minFunctionAlignment(target)),
5608	5569	},
5609		else => \|a\| a.maxStrict(target_util.minFunctionAlignment(target)),
5610		}.toStdMem(),
	5570	else => \|a\| .fromIp(a.maxStrict(target_util.minFunctionAlignment(target))),
	5571	},
5611	5572	.moved = true,
5612	5573	});
5613	5574	coff.nodes.appendAssumeCapacity(.{ .nav = nmi });
5614	5575	const sym = si.get(coff);
5615		sym.ni = ni;
	5576	sym.ni = .wrap(ni);
5616	5577	sym.section_number = sec_si.get(coff).section_number;
5617	5578	},
5618	5579	else => si.deleteLocationRelocs(coff),
...	...	@@ -5622,7 +5583,7 @@ fn updateFuncInner(
5622	5583	sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len));
5623	5584	if (!isImage(coff) and sym.target_relocs != .none)
5624	5585	try coff.pendingSymbolTableEntry(si);
5625		break :ni sym.ni;
	5586	break :ni sym.ni.unwrap().?;
5626	5587	};
5627	5588
5628	5589	var nw: MappedFile.Node.Writer = undefined;
...	...	@@ -5662,7 +5623,6 @@ fn flushImplib(
5662	5623	implib_file: []const u8,
5663	5624	) !void {
5664	5625	// Emitting implibs is only valid for images
5665		assert(coff.export_table.ni != .none);
5666	5626
5667	5627	const comp = coff.base.comp;
5668	5628	const gpa = comp.gpa;
...	...	@@ -5797,7 +5757,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void {
5797	5757	const loc_sym = loc_si.get(coff);
5798	5758
5799	5759	// TODO: Make this a helper for anything that needs to report "referenced by" notes
5800		switch (coff.getNode(loc_sym.ni)) {
	5760	switch (coff.getNode(loc_sym.ni.unwrap().?)) {
5801	5761	.data_directories => {
5802	5762	const dir: std.coff.IMAGE.DIRECTORY_ENTRY =
5803	5763	@fromBackingInt(@intCast(reloc.offset / @sizeOf(std.coff.ImageDataDirectory)));
...	...	@@ -5808,7 +5768,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void {
5808	5768	const other_ioi = isi.input(coff);
5809	5769	if (loc_sym.gmi == .none) {
5810	5770	const section = isi.inputSection(coff);
5811		const section_name = coff.getNode(loc_sym.ni.parent(&coff.mf))
	5771	const section_name = coff.getNode(loc_sym.ni.unwrap().?.parent(&coff.mf).unwrap().?)
5812	5772	.object_section.name(coff).toSlice(coff);
5813	5773
5814	5774	if (section.comdat_si != .null) {
...	...	@@ -5902,15 +5862,17 @@ pub fn flush(
5902	5862	coff.symbol_table.pending_shrink = false;
5903	5863
5904	5864	const number_of_symbols = coff.targetLoad(&coff.headerPtr().number_of_symbols);
5905		coff.symbol_table.ni.shrink(
	5865	coff.symbol_table.ni.resizeLeaf(
5906	5866	&coff.mf,
5907	5867	comp.gpa,
5908	5868	number_of_symbols * std.coff.Symbol.sizeOf(),
5909		true,
5910		) catch \|err\| return comp.link_diags.fail(
5911		"linker failed to compact symbol table: {t}",
5912		.{err},
5913		);
	5869	) catch \|err\| switch (err) {
	5870	else => \|e\| return e,
	5871	error.MappedFileIo => return comp.link_diags.fail(
	5872	"linker failed to compact symbol table: {t}",
	5873	.{coff.mf.io_err.?},
	5874	),
	5875	};
5914	5876	}
5915	5877	while (try coff.idle(tid)) {}
5916	5878
...	...	@@ -6055,8 +6017,8 @@ fn resolve(coff: *Coff, tid: Zcu.PerThread.Id) !bool {
6055	6017	const sub_prog_node = coff.idleProgNode(
6056	6018	tid,
6057	6019	coff.symbol_prog_node,
6058		if (sym.ni != .none)
6059		coff.getNode(sym.ni)
	6020	if (sym.ni.unwrap()) \|sym_ni\|
	6021	coff.getNode(sym_ni)
6060	6022	else
6061	6023	.{ .import_thunk = sym.gmi },
6062	6024	);
...	...	@@ -6173,7 +6135,7 @@ fn idleProgNode(
6173	6135	break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{
6174	6136	ioi.path(coff).fmtEscapeString(),
6175	6137	fmtMemberNameString(ioi.memberName(coff)),
6176		coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff),
	6138	coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff),
6177	6139	}) catch &name;
6178	6140	},
6179	6141	.import_thunk => \|gmi\| gmi.name(coff).toSlice(coff),
...	...	@@ -6213,17 +6175,22 @@ fn flushUav(
6213	6175	try coff.nodes.ensureUnusedCapacity(gpa, 1);
6214	6176	if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1);
6215	6177	const sym = si.get(coff);
6216		const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{
6217		.alignment = uav_align.toStdMem(),
	6178	const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{
	6179	.alignment = .fromIp(uav_align),
6218	6180	.moved = true,
6219	6181	});
6220	6182	coff.nodes.appendAssumeCapacity(.{ .uav = umi });
6221		sym.ni = ni;
	6183	sym.ni = .wrap(ni);
6222	6184	sym.section_number = sec_si.get(coff).section_number;
6223	6185	},
6224	6186	else => {
6225		if (si.get(coff).ni.alignment(&coff.mf).order(uav_align.toStdMem()).compare(.gte))
	6187	if (Alignment.compare(
	6188	si.get(coff).ni.unwrap().?.alignment(&coff.mf),
	6189	.gte,
	6190	.fromIp(uav_align),
	6191	)) {
6226	6192	return;
	6193	}
6227	6194	si.deleteLocationRelocs(coff);
6228	6195	},
6229	6196	}
...	...	@@ -6233,7 +6200,7 @@ fn flushUav(
6233	6200	if (!isImage(coff) and sym.target_relocs != .none)
6234	6201	try coff.pendingSymbolTableEntry(si);
6235	6202
6236		break :ni sym.ni;
	6203	break :ni sym.ni.unwrap().?;
6237	6204	};
6238	6205
6239	6206	var nw: MappedFile.Node.Writer = undefined;
...	...	@@ -6497,23 +6464,23 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool {
6497	6464	lib_name,
6498	6465	ImportTable.Adapter{ .coff = coff },
6499	6466	);
6500		const import_hint_name_align: std.mem.Alignment = .@"2";
	6467	const import_hint_name_align: Alignment = .@"2";
6501	6468	if (!gop.found_existing) {
6502	6469	errdefer _ = coff.import_table.entries.pop();
6503		try coff.import_table.ni.resize(
	6470	try coff.import_table.ni.resizeLeaf(
6504	6471	&coff.mf,
6505	6472	gpa,
6506	6473	@sizeOf(std.coff.ImportDirectoryEntry) * (gop.index + 2),
6507	6474	);
6508	6475	const import_hint_name_table_len =
6509	6476	import_hint_name_align.forward(lib_name.len + ".dll".len + 1);
6510		const idata_section_ni = coff.import_table.ni.parent(&coff.mf);
6511		const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{
	6477	const idata_section_ni = coff.import_table.ni.parent(&coff.mf).unwrap().?;
	6478	const import_lookup_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{
6512	6479	.size = addr_info.size * 2,
6513	6480	.alignment = addr_info.alignment,
6514	6481	.moved = true,
6515	6482	});
6516		const import_address_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{
	6483	const import_address_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{
6517	6484	.size = addr_info.size * 2,
6518	6485	.alignment = addr_info.alignment,
6519	6486	.moved = true,
...	...	@@ -6521,13 +6488,13 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool {
6521	6488	const import_address_table_si = coff.addSymbolAssumeCapacity();
6522	6489	{
6523	6490	const import_address_table_sym = import_address_table_si.get(coff);
6524		import_address_table_sym.ni = import_address_table_ni;
	6491	import_address_table_sym.ni = .wrap(import_address_table_ni);
6525	6492	assert(import_address_table_sym.loc_relocs == .none);
6526	6493	import_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len));
6527	6494	import_address_table_sym.section_number =
6528	6495	coff.getNode(idata_section_ni).object_section.symbol(coff).get(coff).section_number;
6529	6496	}
6530		const import_hint_name_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{
	6497	const import_hint_name_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{
6531	6498	.size = import_hint_name_table_len,
6532	6499	.alignment = import_hint_name_align,
6533	6500	.moved = true,
...	...	@@ -6583,9 +6550,9 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool {
6583	6550	gop.value_ptr.len = import_symbol_index + 1;
6584	6551	const new_symbol_table_size = addr_info.size * (import_symbol_index + 2);
6585	6552
6586		try gop.value_ptr.import_lookup_table_ni.resize(&coff.mf, gpa, new_symbol_table_size);
	6553	try gop.value_ptr.import_lookup_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size);
6587	6554	const import_address_table_ni = gop.value_ptr.import_address_table_si.node(coff);
6588		try import_address_table_ni.resize(&coff.mf, gpa, new_symbol_table_size);
	6555	try import_address_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size);
6589	6556
6590	6557	const opt_imp_name = import.name.toSlice(coff);
6591	6558	const opt_import_hint_name_index = if (opt_imp_name) \|imp_name\| blk: {
...	...	@@ -6593,7 +6560,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool {
6593	6560	gop.value_ptr.hint_name_len = @intCast(
6594	6561	import_hint_name_align.forward(import_hint_name_index + 2 + imp_name.len + 1),
6595	6562	);
6596		try gop.value_ptr.import_hint_name_table_ni.resize(&coff.mf, gpa, gop.value_ptr.hint_name_len);
	6563	try gop.value_ptr.import_hint_name_table_ni.resizeLeaf(&coff.mf, gpa, gop.value_ptr.hint_name_len);
6597	6564	break :blk import_hint_name_index;
6598	6565	} else null;
6599	6566
...	...	@@ -6648,13 +6615,13 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool {
6648	6615	sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len));
6649	6616
6650	6617	const target = &comp.root_mod.resolved_target.result;
6651		const alignment = switch (comp.root_mod.optimize_mode) {
	6618	const alignment: Alignment = switch (comp.root_mod.optimize_mode) {
6652	6619	.debug,
6653	6620	.safe,
6654	6621	.fast,
6655		=> target_util.defaultFunctionAlignment(target),
6656		.small => target_util.minFunctionAlignment(target),
6657		}.toStdMem();
	6622	=> .fromIp(target_util.defaultFunctionAlignment(target)),
	6623	.small => .fromIp(target_util.minFunctionAlignment(target)),
	6624	};
6658	6625	const parent_si = (try coff.pseudoSectionMapIndex(
6659	6626	.@".thunks",
6660	6627	alignment,
...	...	@@ -6668,12 +6635,12 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool {
6668	6635	else => \|tag\| @panic(@tagName(tag)),
6669	6636	.AMD64 => {
6670	6637	const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 };
6671		const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni, .{
	6638	const ni = try parent_sym.ni.unwrap().?.addFloatingChild(&coff.mf, gpa, .{
6672	6639	.alignment = alignment,
6673		.size = init.len,
	6640	.size = alignment.forward(init.len),
6674	6641	});
6675	6642	@memcpy(ni.slice(&coff.mf)[0..init.len], &init);
6676		sym.ni = ni;
	6643	sym.ni = .wrap(ni);
6677	6644	sym.extra.size = init.len;
6678	6645	try coff.addReloc(
6679	6646	si,
...	...	@@ -6736,7 +6703,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol {
6736	6703	try coff.symbols.ensureUnusedCapacity(gpa, 1);
6737	6704	const optional_hdr_si = coff.addSymbolAssumeCapacity();
6738	6705	const optional_hdr_sym = optional_hdr_si.get(coff);
6739		optional_hdr_sym.ni = Node.known.optional_header;
	6706	optional_hdr_sym.ni = .wrap(Node.known.optional_header);
6740	6707	assert(optional_hdr_sym.loc_relocs == .none);
6741	6708	optional_hdr_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len));
6742	6709
...	...	@@ -6783,7 +6750,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol {
6783	6750	try coff.symbols.ensureUnusedCapacity(gpa, 1);
6784	6751	const data_dir_si = coff.addSymbolAssumeCapacity();
6785	6752	const data_dir_sym = data_dir_si.get(coff);
6786		data_dir_sym.ni = Node.known.data_directories;
	6753	data_dir_sym.ni = .wrap(Node.known.data_directories);
6787	6754	assert(data_dir_sym.loc_relocs == .none);
6788	6755	data_dir_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len));
6789	6756
...	...	@@ -6821,12 +6788,12 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void {
6821	6788	.code => .text,
6822	6789	.const_data => .rdata,
6823	6790	};
6824		const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ .moved = true });
	6791	const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .moved = true });
6825	6792	coff.nodes.appendAssumeCapacity(switch (lazy.kind) {
6826	6793	.code => .{ .lazy_code = @fromBackingInt(@intCast(lmr.index)) },
6827	6794	.const_data => .{ .lazy_const_data = @fromBackingInt(@intCast(lmr.index)) },
6828	6795	});
6829		sym.ni = ni;
	6796	sym.ni = .wrap(ni);
6830	6797	sym.section_number = sec_si.get(coff).section_number;
6831	6798	},
6832	6799	else => si.deleteLocationRelocs(coff),
...	...	@@ -6836,7 +6803,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void {
6836	6803	if (!isImage(coff) and sym.target_relocs != .none)
6837	6804	try coff.pendingSymbolTableEntry(si);
6838	6805
6839		break :ni sym.ni;
	6806	break :ni sym.ni.unwrap().?;
6840	6807	};
6841	6808
6842	6809	var required_alignment: InternPool.Alignment = .none;
...	...	@@ -6914,7 +6881,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void {
6914	6881	const flags = coff.targetLoad(&sym.section_number.header(coff).flags);
6915	6882	if (!flags.CNT_UNINITIALIZED_DATA) {
6916	6883	const file_offset = if (isArchive(coff))
6917		sym.ni.location(&coff.mf).resolve(&coff.mf)[0]
	6884	sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[0]
6918	6885	else
6919	6886	ni.fileLocation(&coff.mf, false).offset;
6920	6887
...	...	@@ -6927,7 +6894,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void {
6927	6894	.input_section => \|isi\| {
6928	6895	try isi.symbol(coff).flushMoved(coff);
6929	6896	for (coff.input_symbols.items[@backingInt(isi.firstSymbol(coff))..]) \|input_symbol\| {
6930		if (input_symbol.si.get(coff).ni != ni) break;
	6897	if (input_symbol.si.get(coff).ni != ni.toOptional()) break;
6931	6898	try input_symbol.si.flushMoved(coff);
6932	6899	}
6933	6900	},
...	...	@@ -7062,7 +7029,7 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void {
7062	7029	if (coff.isArchive() and coff.members.items.len > 0) {
7063	7030	const last_member = coff.members.items[coff.members.items.len - 1];
7064	7031	// See .archive_member branch for reasoning
7065		assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni);
	7032	assert(Node.known.file.last(&coff.mf).unwrap().? == last_member.content_ni);
7066	7033	try coff.flushResized(last_member.content_ni);
7067	7034	}
7068	7035	},
...	...	@@ -7090,19 +7057,15 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void {
7090	7057	=> unreachable,
7091	7058	.archive_member => \|mi\| {
7092	7059	const content_ni = mi.get(coff).content_ni;
7093		const next_ni = content_ni.next(&coff.mf);
7094	7060	const content_offset, _ = content_ni.location(&coff.mf).resolve(&coff.mf);
7095		const next_offset = switch (next_ni) {
7096		.none => offset: {
7097		assert(content_ni.parent(&coff.mf) == Node.known.file);
7098		// This must take into account the final file size. If there are trailing
7099		// bytes, they will be expected to contain another valid member header
7100		break :offset coff.mf.memory_map.memory.len;
7101		},
7102		else => offset: {
7103		assert(coff.getNode(next_ni) == .archive_member_header);
7104		break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0];
7105		},
	7061	const next_offset = if (content_ni.next(&coff.mf).unwrap()) \|next_ni\| offset: {
	7062	assert(coff.getNode(next_ni) == .archive_member_header);
	7063	break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0];
	7064	} else offset: {
	7065	assert(content_ni.parent(&coff.mf) == Node.known.file.toOptional());
	7066	// This must take into account the final file size. If there are trailing
	7067	// bytes, they will be expected to contain another valid member header
	7068	break :offset coff.mf.memory_map.memory.len;
7106	7069	};
7107	7070
7108	7071	// Not inserting IMAGE_ARCHIVE_PAD `\n` byte here, because we are expanding to full size
...	...	@@ -7356,7 +7319,7 @@ fn virtualSlide(coff: *Coff, start_section_index: usize, start_rva: u32) !void {
7356	7319	const section_sym = section.si.get(coff);
7357	7320	section_sym.rva = rva;
7358	7321	coff.targetStore(&header.virtual_address, rva);
7359		try section_sym.ni.childrenMoved(coff.base.comp.gpa, &coff.mf);
	7322	try section_sym.ni.unwrap().?.childrenMoved(coff.base.comp.gpa, &coff.mf);
7360	7323	rva += coff.targetLoad(&header.virtual_size);
7361	7324	}
7362	7325	switch (coff.optionalHeaderPtr()) {
...	...	@@ -7430,7 +7393,7 @@ fn updateExportInner(
7430	7393	// TODO: add an errMsg if this conflicts with an existing symbol
7431	7394	const export_si = try coff.globalSymbol(.{ .name = name });
7432	7395	const export_sym = export_si.get(coff);
7433		export_sym.ni = exported_ni;
	7396	export_sym.ni = .wrap(exported_ni);
7434	7397	export_sym.rva = exported_sym.rva;
7435	7398	export_sym.section_number = exported_sym.section_number;
7436	7399	if (@"export".opts.linkage == .weak and !coff.isImage()) {
...	...	@@ -7481,7 +7444,7 @@ fn updateExportInner(
7481	7444	if (new_name_table_size > std.math.maxInt(@FieldType(ExportTable.Entry, "name_index")))
7482	7445	return coff.base.comp.link_diags.fail("exports name table limit reached", .{});
7483	7446
7484		try coff.export_table.name_table_ni.resize(&coff.mf, gpa, new_name_table_size);
	7447	try coff.export_table.name_table_ni.resizeLeaf(&coff.mf, gpa, new_name_table_size);
7485	7448
7486	7449	const name_table_slice = coff.export_table.name_table_ni.slice(&coff.mf);
7487	7450	@memcpy(name_table_slice[name_index..][0 .. name.len + 1], name[0 .. name.len + 1]);
...	...	@@ -7504,19 +7467,19 @@ fn updateExportInner(
7504	7467
7505	7468	// TODO: These should all be resized ahead of time to fit all exports
7506	7469	// after https://github.com/ziglang/zig/issues/23616
7507		try coff.export_table.export_address_table_si.node(coff).resize(
	7470	try coff.export_table.export_address_table_si.node(coff).resizeLeaf(
7508	7471	&coff.mf,
7509	7472	gpa,
7510	7473	export_count * @sizeOf(std.coff.ExportAddressTableEntry),
7511	7474	);
7512	7475
7513		try coff.export_table.name_pointer_table_ni.resize(
	7476	try coff.export_table.name_pointer_table_ni.resizeLeaf(
7514	7477	&coff.mf,
7515	7478	gpa,
7516	7479	export_count * @sizeOf(std.coff.ExportNamePointerTableEntry),
7517	7480	);
7518	7481
7519		try coff.export_table.ordinal_table_ni.resize(
	7482	try coff.export_table.ordinal_table_ni.resizeLeaf(
7520	7483	&coff.mf,
7521	7484	gpa,
7522	7485	export_count * @sizeOf(std.coff.ExportOrdinalTableEntry),
...	...	@@ -7599,14 +7562,13 @@ fn printSymbol(
7599	7562	si: Symbol.Index,
7600	7563	) !void {
7601	7564	const sym = si.get(coff);
7602		const node = coff.getNode(sym.ni);
7603		try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{t: <26} \| {x:08} ", .{
	7565	try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{s: <26} \| {x:08} ", .{
7604	7566	si,
7605	7567	sym.section_number,
7606	7568	if (sym.flags.extra_tag == .size)
7607	7569	@as(u64, sym.extra.size)
7608		else if (sym.ni != .none)
7609		sym.ni.location(&coff.mf).resolve(&coff.mf)[1]
	7570	else if (sym.ni.unwrap()) \|ni\|
	7571	ni.location(&coff.mf).resolve(&coff.mf)[1]
7610	7572	else
7611	7573	0,
7612	7574	switch (sym.flags.value_tag) {
...	...	@@ -7627,7 +7589,7 @@ fn printSymbol(
7627	7589	},
7628	7590	sym.ni,
7629	7591	if (sym.flags.value_tag == .node_offset) sym.value.node_offset else 0,
7630		node,
	7592	if (sym.ni.unwrap()) \|ni\| @tagName(coff.getNode(ni)) else "",
7631	7593	sym.rva,
7632	7594	});
7633	7595
...	...	@@ -7635,7 +7597,7 @@ fn printSymbol(
7635	7597	try w.print("G {f}\n", .{fmtGlobalName(coff, sym.gmi)});
7636	7598	} else {
7637	7599	try w.writeAll("\| ");
7638		try coff.printNodeName(w, tid, node);
	7600	try coff.printNodeName(w, tid, coff.getNode(sym.ni.unwrap().?));
7639	7601	if (sym.flags.extra_tag == .isli)
7640	7602	try w.print(" \| {s}", .{sym.extra.isli.name(coff).toSlice(coff)});
7641	7603	try w.writeByte('\n');
...	...	@@ -7672,7 +7634,7 @@ fn printNodeName(
7672	7634	try w.print("({f}{f}, {s}", .{
7673	7635	ioi.path(coff).fmtEscapeString(),
7674	7636	fmtMemberNameString(ioi.memberName(coff)),
7675		coff.getNode(is.si.node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff),
	7637	coff.getNode(is.si.node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff),
7676	7638	});
7677	7639	if (is.comdat_si != .null) {
7678	7640	const comdat_sym = is.comdat_si.get(coff);
...	...	@@ -7748,41 +7710,42 @@ pub fn printNode(
7748	7710	{
7749	7711	const mf_node = &coff.mf.nodes.items[@backingInt(ni)];
7750	7712	const off, const size = mf_node.location().resolve(&coff.mf);
7751		try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{
	7713	try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}\n", .{
7752	7714	@backingInt(ni),
7753	7715	off,
7754	7716	size,
7755	7717	mf_node.flags.alignment.toByteUnits(),
7756		if (mf_node.flags.fixed) " fixed" else "",
	7718	mf_node.flags.position,
7757	7719	if (mf_node.flags.moved) " moved" else "",
7758	7720	if (mf_node.flags.resized) " resized" else "",
7759	7721	if (mf_node.flags.has_content) " has_content" else "",
7760	7722	});
7761	7723	}
7762		var leaf = true;
7763		var child_it = ni.children(&coff.mf);
7764		while (child_it.next()) \|child_ni\| {
7765		leaf = false;
7766		try coff.printNode(tid, w, child_ni, indent + 1);
7767		}
7768		if (leaf) {
7769		const file_loc = ni.fileLocation(&coff.mf, false);
7770		if (file_loc.size == 0) return;
7771		var address = file_loc.offset;
7772		const line_len = 0x10;
7773		var line_it = std.mem.window(
7774		u8,
7775		coff.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],
7776		line_len,
7777		line_len,
7778		);
7779		while (line_it.next()) \|line_bytes\| : (address += line_len) {
7780		try w.splatByteAll(' ', indent + 1);
7781		try w.print("{x:0>8} ", .{address});
7782		for (line_bytes) \|byte\| try w.print("{x:0>2} ", .{byte});
7783		try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1);
7784		for (line_bytes) \|byte\| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.');
7785		try w.writeByte('\n');
	7724	if (ni.first(&coff.mf).unwrap()) \|first_ni\| {
	7725	// non-leaf, just print children
	7726	var child_ni = first_ni;
	7727	while (true) {
	7728	try coff.printNode(tid, w, child_ni, indent + 1);
	7729	child_ni = child_ni.next(&coff.mf).unwrap() orelse break;
7786	7730	}
	7731	return;
	7732	}
	7733	const file_loc = ni.fileLocation(&coff.mf, false);
	7734	if (file_loc.size == 0) return;
	7735	var address = file_loc.offset;
	7736	const line_len = 0x10;
	7737	var line_it = std.mem.window(
	7738	u8,
	7739	coff.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)],
	7740	line_len,
	7741	line_len,
	7742	);
	7743	while (line_it.next()) \|line_bytes\| : (address += line_len) {
	7744	try w.splatByteAll(' ', indent + 1);
	7745	try w.print("{x:0>8} ", .{address});
	7746	for (line_bytes) \|byte\| try w.print("{x:0>2} ", .{byte});
	7747	try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1);
	7748	for (line_bytes) \|byte\| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.');
	7749	try w.writeByte('\n');
7787	7750	}
7788	7751	}

src/link/Elf2.zig+292-285

...	...	@@ -18,14 +18,16 @@ const tracy = @import("../tracy.zig");
18	18	const Type = @import("../Type.zig");
19	19	const Value = @import("../Value.zig");
20	20	const Zcu = @import("../Zcu.zig");
	21	const Alignment = MappedFile.Alignment;
21	22
22	23	base: link.File,
23	24	options: link.File.OpenOptions,
24	25	mf: MappedFile,
25	26	ni: Node.Known,
26	27	nodes: std.MultiArrayList(Node),
	28	/// Does not contain an item for `SHN_UNDEF`.
27	29	shdrs: std.ArrayList(Section),
28		phdrs: std.ArrayList(MappedFile.Node.Index),
	30	phdrs: std.ArrayList(MappedFile.Node.Index.Optional),
29	31	shndx: struct {
30	32	got: Section.Index,
31	33	/// Always `.UNDEF` on some targets (e.g. SPARC).
...	...	@@ -99,7 +101,7 @@ dso_globals: std.array_hash_map.Auto(String(.strtab), struct {
99	101	/// the section containing the symbol, and the symbol's offset within the section. I know this
100	102	/// sounds like a terrible hack, but it is genuinely how you're supposed to do this. Copy
101	103	/// relocations suck.
102		alignment: std.mem.Alignment,
	104	alignment: Alignment,
103	105	}),
104	106	shstrtab: StringTable,
105	107	strtab: StringTable,
...	...	@@ -175,7 +177,7 @@ symbol_relocs: std.ArrayList(SymbolReloc),
175	177	got_relocs: std.ArrayList(GotReloc),
176	178	/// Set of relocations which must be re-applied if the size of the TLS segment changes.
177	179	tls_size_symbol_relocs: std.array_hash_map.Auto(SymbolReloc.Index, void),
178		/// Index matches the index into `shdrs`.
	180	/// Index matches the index into `shdrs`. Like `shdrs`, this map excludes `SHN_UNDEF`.
179	181	section_by_name: std.array_hash_map.Auto(String(.shstrtab), void),
180	182	/// Key is the name of a global symbol which has been moved to a new symtab index. Any relocation
181	183	/// entries which target that symbol must be updated to reference the correct symbol index.
...	...	@@ -201,6 +203,8 @@ const Node = union(enum) {
201	203	archive,
202	204	/// This includes the archive magic and long file member.
203	205	archive_header,
	206	/// This is a footer of the `.elf` node, and contains the next archive entry's file header.
	207	archive_elf_footer,
204	208	elf,
205	209	ehdr,
206	210	shdr,
...	...	@@ -339,8 +343,6 @@ const Node = union(enum) {
339	343	};
340	344
341	345	pub const Known = struct {
342		archive: MappedFile.Node.Index,
343		archive_header: MappedFile.Node.Index,
344	346	elf: MappedFile.Node.Index,
345	347	ehdr: MappedFile.Node.Index,
346	348	shdr: MappedFile.Node.Index,
...	...	@@ -349,7 +351,7 @@ const Node = union(enum) {
349	351	text: MappedFile.Node.Index,
350	352	data: MappedFile.Node.Index,
351	353	data_rel_ro: MappedFile.Node.Index,
352		tls: MappedFile.Node.Index,
	354	tls: MappedFile.Node.Index.Optional,
353	355	};
354	356
355	357	comptime {
...	...	@@ -505,7 +507,7 @@ const Section = struct {
505	507	}
506	508
507	509	fn get(s: Index, elf: Elf) Section {
508		return &elf.shdrs.items[@backingInt(s)];
	510	return &elf.shdrs.items[@backingInt(s) - 1]; // overflow means you tried to get the `.UNDEF` section
509	511	}
510	512
511	513	fn name(s: Index, elf: *Elf) String(.shstrtab) {
...	...	@@ -539,7 +541,7 @@ const Section = struct {
539	541	}
540	542	}
541	543
542		fn ensureAligned(shndx: Index, elf: *Elf, min_align: std.mem.Alignment) Error!void {
	544	fn ensureAligned(shndx: Index, elf: *Elf, min_align: Alignment) Error!void {
543	545	switch (elf.shdrPtr(shndx)) {
544	546	inline else => \|shdr\| {
545	547	if (elf.targetLoad(&shdr.addralign) >= min_align.toByteUnits()) {
...	...	@@ -550,9 +552,9 @@ const Section = struct {
550	552	}
551	553	const ni = shndx.get(elf).ni;
552	554	if (min_align.compare(.gt, ni.alignment(&elf.mf))) {
553		try ni.realign(&elf.mf, elf.base.comp.gpa, min_align, .{});
	555	try ni.realign(&elf.mf, elf.base.comp.gpa, min_align);
554	556	}
555		switch (elf.getNode(ni.parent(&elf.mf))) {
	557	switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) {
556	558	.elf => {},
557	559	.segment => \|phndx\| try elf.ensureSegmentAligned(phndx, min_align),
558	560	else => unreachable,
...	...	@@ -583,7 +585,7 @@ const Section = struct {
583	585	break :need_size cur_size + need_additional * ent_size;
584	586	},
585	587	};
586		try elf.ensureNodeSize(node, need_size);
	588	try node.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, need_size);
587	589	}
588	590
589	591	/// Asserts that `rela_shndx` is a `SHT_RELA` section and deletes the `ElfN.Rela` entry at
...	...	@@ -818,7 +820,7 @@ const GotReloc = struct {
818	820	/// * A section
819	821	/// * A NAV, UAV, or lazy code/data
820	822	/// * `.none`, if this relocation was deleted (in which case it should be ignored)
821		node: MappedFile.Node.Index,
	823	node: MappedFile.Node.Index.Optional,
822	824	/// The offset of the relocation inside of `node`.
823	825	offset: u64,
824	826	target: GotKey,
...	...	@@ -942,8 +944,10 @@ const GotReloc = struct {
942	944
943	945	fn apply(reloc: GotReloc, elf: Elf) void {
944	946	assert(elf.ehdrType() != .REL);
945		if (reloc.node == .none) return; // deleted
946		if (reloc.node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) {
	947	const node = reloc.node.unwrap() orelse {
	948	return; // deleted
	949	};
	950	if (node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) {
947	951	// There's no point applying the relocation now, because it will be re-applied by
948	952	// `flushMoved` at some point anyway.
949	953	return;
...	...	@@ -968,8 +972,9 @@ const GotReloc = struct {
968	972	}
969	973	}
970	974	fn applyInner(reloc: const GotReloc, elf: Elf) error{ RelocationOverflow, RelocationMisaligned }!void {
971		const dest_vaddr = elf.getNodeVAddr(reloc.node) + reloc.offset;
972		const dest_slice = reloc.node.slice(&elf.mf)[@intCast(reloc.offset)..];
	975	const node = reloc.node.unwrap().?;
	976	const dest_vaddr = elf.getNodeVAddr(node) + reloc.offset;
	977	const dest_slice = node.slice(&elf.mf)[@intCast(reloc.offset)..];
973	978
974	979	const got_vaddr = elf.shndx.got.vaddr(elf);
975	980	const got_index: u64 = elf.got.getIndex(reloc.target).?;
...	...	@@ -1587,7 +1592,7 @@ const SymbolReloc = struct {
1587	1592	}
1588	1593	},
1589	1594	.sparc_le_hix22 => {
1590		const tls_phndx = elf.getNode(elf.ni.tls).segment;
	1595	const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment;
1591	1596	const tls_size: u64 = switch (elf.phdrSlice()) {
1592	1597	inline else => \|phdr\| tls_size: {
1593	1598	assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS);
...	...	@@ -1646,7 +1651,6 @@ const SymbolReloc = struct {
1646	1651
1647	1652	fn apply(reloc: SymbolReloc, elf: Elf) void {
1648	1653	assert(elf.ehdrType() != .REL);
1649		assert(reloc.node != .none);
1650	1654	if (reloc.node.hasMoved(&elf.mf) or reloc.target.hasMoved(elf)) {
1651	1655	// There's no point applying the relocation now, because it will be re-applied by
1652	1656	// `flushMoved` at some point anyway.
...	...	@@ -1692,7 +1696,7 @@ const SymbolReloc = struct {
1692	1696	.I_original => \|tls\| tls.tcb_size +% reloc.target.value(elf) +% addend,
1693	1697	.I_modified => \|tls\| 0 -% tls.tp_off +% reloc.target.value(elf) +% addend,
1694	1698	.II => {
1695		const tls_phndx = elf.getNode(elf.ni.tls).segment;
	1699	const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment;
1696	1700	const tls_size: u64 = switch (elf.phdrSlice()) {
1697	1701	inline else => \|phdr\| tls_size: {
1698	1702	assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS);
...	...	@@ -1785,6 +1789,8 @@ const SymbolReloc = struct {
1785	1789	};
1786	1790
1787	1791	fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void {
	1792	const gpa = elf.base.comp.gpa;
	1793
1788	1794	const min_buckets = max_dynsym_count / 2;
1789	1795
1790	1796	const cur_dynsym_count: u32 = switch (elf.shdrPtr(elf.shndx.dynsym)) {
...	...	@@ -1805,7 +1811,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void {
1805	1811	// We don't need to add any buckets, but we still need to make sure the section is large
1806	1812	// enough to fit `max_dynsym_count` chains.
1807	1813	const need_size = @sizeOf(info.Header()) + (nbucket + max_dynsym_count) * 4;
1808		try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size);
	1814	try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size);
1809	1815	return;
1810	1816	}
1811	1817	// We need more buckets, so we'll have to rebuild the hash table.
...	...	@@ -1817,7 +1823,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void {
1817	1823
1818	1824	{
1819	1825	const need_size = @sizeOf(info.Header()) + (new_nbucket + max_dynsym_count) * 4;
1820		try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size);
	1826	try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size);
1821	1827	}
1822	1828
1823	1829	elf.mf.nodes_lock.lock();
...	...	@@ -1963,7 +1969,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe
1963	1969	const need_node_size: u64 = switch (elf.shdrPtr(.symtab)) {
1964	1970	inline else => \|shdr, class\| elf.targetLoad(&shdr.size) + len * @sizeOf(class.ElfN().Sym),
1965	1971	};
1966		try elf.ensureNodeSize(Section.Index.symtab.get(elf).ni, need_node_size);
	1972	try Section.Index.symtab.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_node_size);
1967	1973	}
1968	1974
1969	1975	switch (kind) {
...	...	@@ -1986,7 +1992,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe
1986	1992	const dynsym_cur_len: u32 = @intCast(@divExact(dynsym_cur_size, dynsym_ent_size));
1987	1993
1988	1994	const dynsym_need_size: u64 = (dynsym_cur_len + len) * dynsym_ent_size;
1989		try elf.ensureNodeSize(elf.shndx.dynsym.get(elf).ni, dynsym_need_size);
	1995	try elf.shndx.dynsym.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, dynsym_need_size);
1990	1996
1991	1997	try elf.ensureDynsymHashCapacity(dynsym_cur_len + len);
1992	1998
...	...	@@ -2008,19 +2014,19 @@ fn ensureUnusedPltCapacity(elf: *Elf, len: u32) Error!void {
2008	2014	// Ensure the `.plt` section's node is big enough:
2009	2015	{
2010	2016	const need_size: usize = plt.entry_size * (1 + need_plt_count);
2011		try elf.ensureNodeSize(elf.shndx.plt.get(elf).ni, need_size);
	2017	try elf.shndx.plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size);
2012	2018	}
2013	2019
2014	2020	// If there is a `.got.plt` section, ensure its node is big enough
2015	2021	if (plt.got_plt) \|got_plt\| {
2016	2022	const need_size: usize = elf.targetPtrSize() * (got_plt.header_entries + need_plt_count);
2017		try elf.ensureNodeSize(elf.shndx.got_plt.get(elf).ni, need_size);
	2023	try elf.shndx.got_plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size);
2018	2024	}
2019	2025
2020	2026	// If there is a `.plt.sec` section, ensure its node is big enough
2021	2027	if (plt.plt_sec) \|plt_sec\| {
2022	2028	const need_size: usize = plt_sec.entry_size * need_plt_count;
2023		try elf.ensureNodeSize(elf.shndx.plt_sec.get(elf).ni, need_size);
	2029	try elf.shndx.plt_sec.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size);
2024	2030	}
2025	2031	}
2026	2032	/// Given an index into the PLT, returns whether that PLT entry is dead, meaning it may be reused at
...	...	@@ -2044,7 +2050,7 @@ fn pltEntryIsDead(elf: *Elf, plt_index: usize) bool {
2044	2050	}
2045	2051
2046	2052	const AddLocalSymbolOptions = struct {
2047		node: MappedFile.Node.Index,
	2053	node: MappedFile.Node.Index.Optional,
2048	2054	name: String(.strtab),
2049	2055	value: u64,
2050	2056	size: u64,
...	...	@@ -2126,7 +2132,7 @@ const AddGlobalSymbolOptions = struct {
2126	2132	}
2127	2133	};
2128	2134
2129		node: MappedFile.Node.Index,
	2135	node: MappedFile.Node.Index.Optional,
2130	2136	name: Name,
2131	2137	lib_name: ?[]const u8 = null,
2132	2138	value: u64,
...	...	@@ -2294,8 +2300,8 @@ fn addGlobalSymbolAssumeCapacity(elf: *Elf, opts: AddGlobalSymbolOptions) error{
2294	2300	}
2295	2301
2296	2302	const old_head: String(.strtab) = old_head: {
2297		if (opts.node == .none) break :old_head .empty;
2298		const gop = elf.node_global_symbols.getOrPutAssumeCapacity(opts.node);
	2303	const node = opts.node.unwrap() orelse break :old_head .empty;
	2304	const gop = elf.node_global_symbols.getOrPutAssumeCapacity(node);
2299	2305	const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty;
2300	2306	gop.value_ptr.* = opts.name.strtab;
2301	2307	break :old_head old_head;
...	...	@@ -2363,7 +2369,7 @@ fn setGlobalSymbolValue(
2363	2369	global_name: String(.strtab),
2364	2370	global_ptr: *Symbol.Global,
2365	2371	new: struct {
2366		node: MappedFile.Node.Index,
	2372	node: MappedFile.Node.Index.Optional,
2367	2373	value: u64,
2368	2374	size: u64,
2369	2375	type: std.elf.STT,
...	...	@@ -2371,18 +2377,17 @@ fn setGlobalSymbolValue(
2371	2377	},
2372	2378	) void {
2373	2379	assert(new.shndx != .UNDEF);
2374		const old_node = global_ptr.symtab_index.ptr(elf).node;
2375		if (old_node != .none) {
	2380	if (global_ptr.symtab_index.ptr(elf).node.unwrap()) \|old_node\| {
2376	2381	if (global_ptr.next_in_node != .empty) {
2377	2382	const next = elf.globalByName(global_ptr.next_in_node).?;
2378	2383	assert(next.prev_in_node == global_name);
2379		assert(next.symtab_index.ptr(elf).node == old_node);
	2384	assert(next.symtab_index.ptr(elf).node.unwrap().? == old_node);
2380	2385	next.prev_in_node = global_ptr.prev_in_node;
2381	2386	}
2382	2387	if (global_ptr.prev_in_node != .empty) {
2383	2388	const prev = elf.globalByName(global_ptr.prev_in_node).?;
2384	2389	assert(prev.next_in_node == global_name);
2385		assert(prev.symtab_index.ptr(elf).node == old_node);
	2390	assert(prev.symtab_index.ptr(elf).node.unwrap().? == old_node);
2386	2391	prev.next_in_node = global_ptr.next_in_node;
2387	2392	} else {
2388	2393	// We're the start of the linked list, so we need to change the head.
...	...	@@ -2417,8 +2422,8 @@ fn setGlobalSymbolValue(
2417	2422	global_ptr.symtab_index.ptr(elf).node = new.node;
2418	2423
2419	2424	const old_head: String(.strtab) = old_head: {
2420		if (new.node == .none) break :old_head .empty;
2421		const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new.node);
	2425	const new_node = new.node.unwrap() orelse break :old_head .empty;
	2426	const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new_node);
2422	2427	const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty;
2423	2428	gop.value_ptr.* = global_name;
2424	2429	break :old_head old_head;
...	...	@@ -2644,7 +2649,7 @@ const Symbol = struct {
2644	2649	/// * A section (the symbol's value is some vaddr in that section)
2645	2650	/// * An input section (the symbol's value is some vaddr in that input section)
2646	2651	/// * A NAV, UAV, or lazy code/data (the symbol's value is exactly the vaddr of that node)
2647		node: MappedFile.Node.Index,
	2652	node: MappedFile.Node.Index.Optional,
2648	2653
2649	2654	/// The head of a linked list of relocations targeting this symbol.
2650	2655	first_target_reloc: SymbolReloc.Index,
...	...	@@ -2852,8 +2857,7 @@ const Symbol = struct {
2852	2857	/// Returns `true` if the target of `s` has moved, meaning the symbol's value will change at
2853	2858	/// some point due to a call to `flushMoved`.
2854	2859	fn hasMoved(s: Symbol.Id, elf: *Elf) bool {
2855		const node = s.index(elf).ptr(elf).node;
2856		if (node != .none) {
	2860	if (s.index(elf).ptr(elf).node.unwrap()) \|node\| {
2857	2861	return node.hasMoved(&elf.mf);
2858	2862	}
2859	2863	switch (s.unwrap()) {
...	...	@@ -2950,6 +2954,7 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId {
2950	2954	const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) {
2951	2955	.archive,
2952	2956	.archive_header,
	2957	.archive_elf_footer,
2953	2958	.elf,
2954	2959	.ehdr,
2955	2960	.shdr,
...	...	@@ -2989,7 +2994,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol
2989	2994	.code => .{ .text, .FUNC },
2990	2995	.const_data => .{ .rodata, .OBJECT },
2991	2996	};
2992		const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{});
	2997	const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{});
2993	2998	var name_buf: [64]u8 = undefined;
2994	2999	const name = std.fmt.bufPrint(
2995	3000	&name_buf,
...	...	@@ -2998,7 +3003,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol
2998	3003	) catch unreachable;
2999	3004	gop.value_ptr.* = .{
3000	3005	.lsi = elf.addLocalSymbolAssumeCapacity(.{
3001		.node = node,
	3006	.node = .wrap(node),
3002	3007	.name = try elf.string(.strtab, name),
3003	3008	.value = 0,
3004	3009	.size = 0,
...	...	@@ -3248,7 +3253,7 @@ const StringTable = struct {
3248	3253	break :size .{ old_size, new_size };
3249	3254	},
3250	3255	};
3251		try elf.ensureNodeSize(ni, new_size);
	3256	try ni.ensureMinimumSize(&elf.mf, gpa, new_size);
3252	3257	const slice = ni.slice(&elf.mf)[old_size..];
3253	3258	@memcpy(slice[0..key.len], key);
3254	3259	slice[key.len] = 0;
...	...	@@ -3349,16 +3354,14 @@ fn create(
3349	3354	.options = options,
3350	3355	.mf = try .init(file, comp.gpa, io),
3351	3356	.ni = .{
3352		.archive = .root,
3353		.archive_header = .none,
3354		.elf = .root,
3355		.ehdr = .none,
3356		.shdr = .none,
3357		.rodata = .none,
3358		.phdr = .none,
3359		.text = .none,
3360		.data = .none,
3361		.data_rel_ro = .none,
	3357	.elf = undefined,
	3358	.ehdr = undefined,
	3359	.shdr = undefined,
	3360	.rodata = undefined,
	3361	.phdr = undefined,
	3362	.text = undefined,
	3363	.data = undefined,
	3364	.data_rel_ro = undefined,
3362	3365	.tls = .none,
3363	3366	},
3364	3367	.nodes = .empty,
...	...	@@ -3489,7 +3492,7 @@ fn initHeaders(
3489	3492	.EXEC => comp.config.link_mode == .dynamic,
3490	3493	.DYN => true,
3491	3494	};
3492		const addr_align: std.mem.Alignment = switch (class) {
	3495	const addr_align: Alignment = switch (class) {
3493	3496	.NONE, _ => unreachable,
3494	3497	.@"32" => .@"4",
3495	3498	.@"64" => .@"8",
...	...	@@ -3503,7 +3506,7 @@ fn initHeaders(
3503	3506	//
3504	3507	// It can be handy to temporarily set this to `.@"1"` when working on the linker, because it
3505	3508	// prevents alignment bugs from being hidden by your filesystem's block alignment.
3506		const node_block_align: std.mem.Alignment = elf.mf.flags.block_size;
	3509	const node_block_align: Alignment = elf.mf.flags.block_size;
3507	3510
3508	3511	const plt: PltInfo = .fromMachine(machine);
3509	3512
...	...	@@ -3599,28 +3602,31 @@ fn initHeaders(
3599	3602	}, phnum };
3600	3603	};
3601	3604
3602		const expected_nodes_len = @as(usize, if (is_archive) 2 else 0) + // .archive, .archive_header
3603		3 + // `.file`, `.ehdr`, and `.shdr` nodes
3604		(shnum - 1) + // -1 because the null shdr does not have a `.section` node
	3605	const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer
	3606	3 + // `.elf`, `.ehdr`, and `.shdr` nodes
	3607	(shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node
3605	3608	(phnum -\| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node
3606	3609
3607	3610	try elf.nodes.ensureTotalCapacity(gpa, expected_nodes_len);
3608		try elf.shdrs.ensureTotalCapacity(gpa, shnum);
3609		try elf.section_by_name.ensureUnusedCapacity(gpa, shnum);
	3611	try elf.shdrs.ensureTotalCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF
	3612	try elf.section_by_name.ensureUnusedCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF
3610	3613	try elf.phdrs.resize(gpa, phnum);
3611	3614	try elf.symtab.ensureTotalCapacity(gpa, 1);
3612	3615
3613	3616	if (is_archive) {
3614	3617	elf.nodes.appendAssumeCapacity(.archive);
3615		elf.ni.archive_header = try elf.mf.addOnlyChildNode(gpa, elf.ni.archive, .{
	3618
	3619	const archive_ni: MappedFile.Node.Index = .root;
	3620
	3621	const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{
3616	3622	.size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2,
3617	3623	.alignment = .@"2",
3618		.fixed = true,
3619	3624	.next_moved = true,
3620	3625	.bubbles_moved = false,
3621	3626	.enable_next_moved = true,
3622	3627	});
3623		const archive_header_slice = elf.ni.archive_header.slice(&elf.mf);
	3628	elf.nodes.appendAssumeCapacity(.archive_header);
	3629	const archive_header_slice = archive_header_ni.slice(&elf.mf);
3624	3630	@memcpy(archive_header_slice[0..std.elf.ARMAG.len], std.elf.ARMAG);
3625	3631	const strtab_ar_hdr: *std.elf.ar_hdr = @ptrCast(archive_header_slice[std.elf.ARMAG.len..]);
3626	3632	strtab_ar_hdr.* = .{
...	...	@@ -3633,15 +3639,23 @@ fn initHeaders(
3633	3639	.ar_fmag = std.elf.ARFMAG.*,
3634	3640	};
3635	3641
3636		elf.nodes.appendAssumeCapacity(.archive_header);
3637		elf.ni.elf = try elf.mf.addLastChildNode(gpa, elf.ni.archive, .{
	3642	elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{
3638	3643	.alignment = node_block_align.max(.@"2"),
3639	3644	.next_moved = true,
3640	3645	.bubbles_moved = false,
3641	3646	.enable_next_moved = true,
3642	3647	});
	3648	elf.nodes.appendAssumeCapacity(.elf);
	3649
	3650	_ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{
	3651	.alignment = .@"2",
	3652	.size = @sizeOf(std.elf.ar_hdr),
	3653	});
	3654	elf.nodes.appendAssumeCapacity(.archive_elf_footer);
	3655	} else {
	3656	elf.ni.elf = .root;
	3657	elf.nodes.appendAssumeCapacity(.elf);
3643	3658	}
3644		elf.nodes.appendAssumeCapacity(.elf);
3645	3659
3646	3660	const entsize: struct { ph: u32, sh: u32 } = switch (class) {
3647	3661	.NONE, _ => unreachable,
...	...	@@ -3655,19 +3669,18 @@ fn initHeaders(
3655	3669	// the rodata segment. Although to my knowledge neither ELF nor any ELF-based OS strictly
3656	3670	// requires this, it is highly conventional and therefore sometimes relied upon.
3657	3671	if (@"type" != .REL) {
3658		elf.ni.rodata = try elf.mf.addOnlyChildNode(gpa, elf.ni.elf, .{
	3672	// This node will contain the ehdr, which must be at the start of the ELF file, so this
	3673	// node must itself be a header of the `.elf` node.
	3674	elf.ni.rodata = try elf.ni.elf.addOnlyHeaderChild(&elf.mf, gpa, .{
3659	3675	// Must be at least `addr_align` for `elf.ni.phdr` to be placed inside this node
3660	3676	.alignment = node_block_align.max(addr_align),
3661		// This node will contain the ehdr, which must be at the start of the ELF file, so this
3662		// node must itself be fixed.
3663		.fixed = true,
3664	3677	.moved = true,
3665	3678	.bubbles_moved = false,
3666	3679	});
3667	3680	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.rodata });
3668		elf.phdrs.items[phndx.rodata] = elf.ni.rodata;
	3681	elf.phdrs.items[phndx.rodata] = .wrap(elf.ni.rodata);
3669	3682
3670		elf.ni.phdr = try elf.mf.addOnlyChildNode(gpa, elf.ni.rodata, .{
	3683	elf.ni.phdr = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{
3671	3684	.size = @as(u64, phnum) * entsize.ph,
3672	3685	.alignment = addr_align, // keep in sync with `elf.ni.rodata` alignment above
3673	3686	.moved = true,
...	...	@@ -3675,36 +3688,36 @@ fn initHeaders(
3675	3688	.bubbles_moved = false,
3676	3689	});
3677	3690	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.phdr });
3678		elf.phdrs.items[phndx.phdr] = elf.ni.phdr;
	3691	elf.phdrs.items[phndx.phdr] = .wrap(elf.ni.phdr);
3679	3692
3680		elf.ni.text = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{
	3693	elf.ni.text = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{
3681	3694	.alignment = node_block_align,
3682	3695	.moved = true,
3683	3696	.bubbles_moved = false,
3684	3697	});
3685	3698	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.text });
3686		elf.phdrs.items[phndx.text] = elf.ni.text;
	3699	elf.phdrs.items[phndx.text] = .wrap(elf.ni.text);
3687	3700
3688		elf.ni.data = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{
	3701	elf.ni.data = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{
3689	3702	// Must be at least `addr_align` for `elf.ni.data_rel_ro` to be placed inside this node
3690	3703	.alignment = node_block_align.max(addr_align),
3691	3704	.moved = true,
3692	3705	.bubbles_moved = false,
3693	3706	});
3694	3707	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.data });
3695		elf.phdrs.items[phndx.data] = elf.ni.data;
	3708	elf.phdrs.items[phndx.data] = .wrap(elf.ni.data);
3696	3709
3697	3710	if (plt.got_plt == null) {
3698		const plt_ni = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{
	3711	const plt_ni = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{
3699	3712	.alignment = node_block_align,
3700	3713	.moved = true,
3701	3714	.bubbles_moved = false,
3702	3715	});
3703	3716	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.plt });
3704		elf.phdrs.items[phndx.plt] = plt_ni;
	3717	elf.phdrs.items[phndx.plt] = .wrap(plt_ni);
3705	3718	}
3706	3719
3707		elf.ni.data_rel_ro = try elf.mf.addOnlyChildNode(gpa, elf.ni.data, .{
	3720	elf.ni.data_rel_ro = try elf.ni.data.addFloatingChild(&elf.mf, gpa, .{
3708	3721	// Must be at least `addr_align` for the `PT_DYNAMIC` node to be placed inside this one
3709	3722	// later (if `have_dynamic_section`). Keep in sync with `elf.ni.data` alignment above.
3710	3723	.alignment = node_block_align.max(addr_align),
...	...	@@ -3712,19 +3725,27 @@ fn initHeaders(
3712	3725	.bubbles_moved = false,
3713	3726	});
3714	3727	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.relro });
3715		elf.phdrs.items[phndx.relro] = elf.ni.data_rel_ro;
	3728	elf.phdrs.items[phndx.relro] = .wrap(elf.ni.data_rel_ro);
3716	3729
3717	3730	if (comp.config.any_non_single_threaded) {
3718		elf.ni.tls = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{
	3731	elf.ni.tls = .wrap(try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{
3719	3732	.alignment = node_block_align,
3720	3733	.moved = true,
3721	3734	.bubbles_moved = false,
3722		});
	3735	}));
3723	3736	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.tls });
3724	3737	elf.phdrs.items[phndx.tls] = elf.ni.tls;
3725	3738	}
3726	3739
3727	3740	elf.phdrs.items[phndx.gnu_stack] = .none;
	3741	} else {
	3742	elf.ni.rodata = elf.ni.elf;
	3743	elf.ni.text = elf.ni.elf;
	3744	elf.ni.data = elf.ni.elf;
	3745	elf.ni.data_rel_ro = elf.ni.elf;
	3746	if (comp.config.any_non_single_threaded) {
	3747	elf.ni.tls = .wrap(elf.ni.elf);
	3748	}
3728	3749	}
3729	3750
3730	3751	switch (class) {
...	...	@@ -3736,10 +3757,9 @@ fn initHeaders(
3736	3757	.REL => elf.ni.elf,
3737	3758	.DYN, .EXEC => elf.ni.rodata,
3738	3759	};
3739		elf.ni.ehdr = try elf.mf.addFirstChildNode(gpa, parent_ni, .{
	3760	elf.ni.ehdr = try parent_ni.addOnlyHeaderChild(&elf.mf, gpa, .{
3740	3761	.size = @sizeOf(ElfN.Ehdr),
3741	3762	.alignment = addr_align,
3742		.fixed = true,
3743	3763	});
3744	3764	elf.nodes.appendAssumeCapacity(.ehdr);
3745	3765
...	...	@@ -3785,14 +3805,14 @@ fn initHeaders(
3785	3805	ehdr.phentsize = @sizeOf(ElfN.Phdr);
3786	3806	ehdr.phnum = @min(phnum, std.elf.PN_XNUM);
3787	3807	ehdr.shentsize = @sizeOf(ElfN.Shdr);
3788		ehdr.shnum = 1; // Only the null shdr initially---will be incremented by `addSection`
	3808	ehdr.shnum = 1; // Only the SHN_UNDEF shdr initially---will be incremented by `addSection`
3789	3809	ehdr.shstrndx = std.elf.SHN_UNDEF;
3790	3810	if (elf.targetEndian() != native_endian) std.mem.byteSwapAllFields(ElfN.Ehdr, ehdr);
3791	3811	},
3792	3812	}
3793	3813
3794		elf.ni.shdr = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{
3795		.size = 1 * entsize.sh, // as above, only the null shdr initially
	3814	elf.ni.shdr = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{
	3815	.size = node_block_align.forward(1 * entsize.sh), // as above, only the SHN_UNDEF initially
3796	3816	.alignment = addr_align.max(node_block_align),
3797	3817	.moved = true,
3798	3818	.resized = true,
...	...	@@ -3916,7 +3936,7 @@ fn initHeaders(
3916	3936	};
3917	3937	}
3918	3938
3919		if (comp.config.any_non_single_threaded) {
	3939	if (elf.ni.tls.unwrap()) \|tls_segment_ni\| {
3920	3940	const ph_tls = &phdr[phndx.tls];
3921	3941	ph_tls.* = .{
3922	3942	.type = .TLS,
...	...	@@ -3926,7 +3946,7 @@ fn initHeaders(
3926	3946	.filesz = 0,
3927	3947	.memsz = 0,
3928	3948	.flags = .{ .R = true },
3929		.@"align" = @intCast(elf.ni.tls.alignment(&elf.mf).toByteUnits()),
	3949	.@"align" = @intCast(tls_segment_ni.alignment(&elf.mf).toByteUnits()),
3930	3950	};
3931	3951	}
3932	3952
...	...	@@ -3987,7 +4007,6 @@ fn initHeaders(
3987	4007	.entsize = 0,
3988	4008	};
3989	4009	if (target_endian != native_endian) std.mem.byteSwapAllFields(ElfN.Shdr, sh_undef);
3990		elf.shdrs.appendAssumeCapacity(.{ .lsi = .null, .ni = .none, .rela = .{ .shndx = .UNDEF } });
3991	4010
3992	4011	elf.symtab.addOneAssumeCapacity().* = .{
3993	4012	.node = .none,
...	...	@@ -4092,7 +4111,7 @@ fn initHeaders(
4092	4111	.node_align = node_block_align,
4093	4112	});
4094	4113	} else {
4095		elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt], .{
	4114	elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{
4096	4115	.name = ".plt",
4097	4116	.type = .PROGBITS,
4098	4117	.flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true },
...	...	@@ -4108,14 +4127,14 @@ fn initHeaders(
4108	4127	.node_align = node_block_align,
4109	4128	});
4110	4129	if (maybe_interp) \|interp\| {
4111		const interp_ni = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{
	4130	const interp_ni = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{
4112	4131	.size = interp.len + 1,
4113	4132	.moved = true,
4114	4133	.resized = true,
4115	4134	.bubbles_moved = false,
4116	4135	});
4117	4136	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.interp });
4118		elf.phdrs.items[phndx.interp] = interp_ni;
	4137	elf.phdrs.items[phndx.interp] = .wrap(interp_ni);
4119	4138
4120	4139	const sec_interp_shndx = try elf.addSection(interp_ni, .{
4121	4140	.name = ".interp",
...	...	@@ -4129,13 +4148,13 @@ fn initHeaders(
4129	4148	}
4130	4149	if (have_dynamic_section) {
4131	4150	assert(elf.ni.data_rel_ro.alignment(&elf.mf).compare(.gte, addr_align));
4132		const dynamic_ni = try elf.mf.addLastChildNode(gpa, elf.ni.data_rel_ro, .{
	4151	const dynamic_ni = try elf.ni.data_rel_ro.addFloatingChild(&elf.mf, gpa, .{
4133	4152	.alignment = addr_align,
4134	4153	.moved = true,
4135	4154	.bubbles_moved = false,
4136	4155	});
4137	4156	elf.nodes.appendAssumeCapacity(.{ .segment = phndx.dynamic });
4138		elf.phdrs.items[phndx.dynamic] = dynamic_ni;
	4157	elf.phdrs.items[phndx.dynamic] = .wrap(dynamic_ni);
4139	4158
4140	4159	const dynstr_shndx = try elf.addSection(elf.ni.rodata, .{
4141	4160	.name = ".dynstr",
...	...	@@ -4207,7 +4226,7 @@ fn initHeaders(
4207	4226	.flags = .{ .ALLOC = true, .WRITE = true },
4208	4227	.link = dynstr_shndx.toSection().?,
4209	4228	.entsize = @intCast(addr_align.toByteUnits() * 2),
4210		.node_align = addr_align,
	4229	.addralign = addr_align,
4211	4230	});
4212	4231	switch (elf.targetDynsymHashInfo()) {
4213	4232	inline else => \|info\| {
...	...	@@ -4230,8 +4249,8 @@ fn initHeaders(
4230	4249	if (elf.targetEndian() != std.lang.Endian.native) {
4231	4250	std.mem.byteSwapAllFields(info.Header(), header);
4232	4251	}
4233		// The initial bucket and chain values are all 0, but `MappedFile` initialized
4234		// the node with zeroes anyway, so no need to memset.
	4252	// The initial bucket and chain values are all 0.
	4253	@memset(hash_slice[@sizeOf(info.Header())..], 0);
4235	4254	},
4236	4255	}
4237	4256
...	...	@@ -4347,7 +4366,7 @@ fn initHeaders(
4347	4366	try elf.ensureUnusedSymbolCapacity(10, .maybe_global);
4348	4367	// Despite the name, `__dso_handle` is necessary even in static binaries.
4349	4368	_ = elf.addGlobalSymbolAssumeCapacity(.{
4350		.node = Section.Index.text.get(elf).ni,
	4369	.node = .wrap(Section.Index.text.get(elf).ni),
4351	4370	.name = try .string(elf, "__dso_handle"),
4352	4371	.value = Section.Index.text.vaddr(elf),
4353	4372	.size = 0,
...	...	@@ -4359,7 +4378,7 @@ fn initHeaders(
4359	4378	error.MultipleDefinitions => unreachable, // no inputs are processed yet
4360	4379	};
4361	4380	_ = elf.addGlobalSymbolAssumeCapacity(.{
4362		.node = elf.shndx.plt.get(elf).ni,
	4381	.node = .wrap(elf.shndx.plt.get(elf).ni),
4363	4382	.name = try .string(elf, "_PROCEDURE_LINKAGE_TABLE_"),
4364	4383	.value = elf.shndx.plt.vaddr(elf),
4365	4384	.size = 0,
...	...	@@ -4371,7 +4390,7 @@ fn initHeaders(
4371	4390	error.MultipleDefinitions => unreachable, // no inputs are processed yet
4372	4391	};
4373	4392	_ = elf.addGlobalSymbolAssumeCapacity(.{
4374		.node = elf.shndx.got.get(elf).ni,
	4393	.node = .wrap(elf.shndx.got.get(elf).ni),
4375	4394	.name = try .string(elf, "_GLOBAL_OFFSET_TABLE_"),
4376	4395	.value = switch (machine) {
4377	4396	.AARCH64,
...	...	@@ -4468,7 +4487,7 @@ fn initHeaders(
4468	4487	};
4469	4488	if (have_dynamic_section) {
4470	4489	_ = elf.addGlobalSymbolAssumeCapacity(.{
4471		.node = elf.shndx.dynamic.get(elf).ni,
	4490	.node = .wrap(elf.shndx.dynamic.get(elf).ni),
4472	4491	.name = try .string(elf, "_DYNAMIC"),
4473	4492	.value = elf.shndx.dynamic.vaddr(elf),
4474	4493	.size = 0,
...	...	@@ -4484,16 +4503,16 @@ fn initHeaders(
4484	4503	assert(maybe_interp == null);
4485	4504	assert(!have_dynamic_section);
4486	4505	}
4487		if (comp.config.any_non_single_threaded) elf.shndx.tdata = try elf.addSection(elf.ni.tls, .{
	4506	if (elf.ni.tls.unwrap()) \|tls_segment_ni\| elf.shndx.tdata = try elf.addSection(tls_segment_ni, .{
4488	4507	.name = ".tdata",
4489	4508	.flags = .{ .WRITE = true, .ALLOC = true, .TLS = true },
4490	4509	.node_align = node_block_align,
4491	4510	});
4492	4511
4493	4512	assert(elf.nodes.len == expected_nodes_len);
4494		assert(elf.shdrs.items.len == shnum);
	4513	assert(elf.shdrs.items.len == shnum - 1); // -1 to exclude SHN_UNDEF
4495	4514
4496		for (0..shnum) \|shndx_raw\| {
	4515	for (1..shnum) \|shndx_raw\| { // start at 1 to exclude SHN_UNDEF
4497	4516	const shndx: Section.Index = @fromBackingInt(@intCast(shndx_raw));
4498	4517	elf.section_by_name.putAssumeCapacityNoClobber(shndx.name(elf), {});
4499	4518	}
...	...	@@ -4520,8 +4539,6 @@ fn initHeaders(
4520	4539	break :str try elf.string(.dynstr, slice);
4521	4540	},
4522	4541	};
4523
4524		try elf.ensureElfNodeSize();
4525	4542	}
4526	4543
4527	4544	pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void {
...	...	@@ -4556,6 +4573,7 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index {
4556	4573	return switch (elf.getNode(ni)) {
4557	4574	.archive,
4558	4575	.archive_header,
	4576	.archive_elf_footer,
4559	4577	.elf,
4560	4578	.ehdr,
4561	4579	.shdr,
...	...	@@ -4569,13 +4587,14 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index {
4569	4587	.uav,
4570	4588	.lazy_code,
4571	4589	.lazy_const_data,
4572		=> elf.getNode(ni.parent(&elf.mf)).section,
	4590	=> elf.getNode(ni.parent(&elf.mf).unwrap().?).section,
4573	4591	};
4574	4592	}
4575	4593	fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4576	4594	return switch (elf.getNode(ni)) {
4577	4595	.archive,
4578	4596	.archive_header,
	4597	.archive_elf_footer,
4579	4598	.elf,
4580	4599	.ehdr,
4581	4600	.shdr,
...	...	@@ -4593,8 +4612,8 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4593	4612	};
4594	4613	}
4595	4614	fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4596		const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf))) {
4597		.archive, .archive_header => unreachable,
	4615	const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) {
	4616	.archive, .archive_header, .archive_elf_footer => unreachable,
4598	4617	.elf => return 0,
4599	4618	.ehdr, .shdr => unreachable,
4600	4619	.segment => \|phndx\| switch (elf.phdrSlice()) {
...	...	@@ -4620,6 +4639,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void {
4620	4639	const symbol_relocs: SymbolReloc.Index, const got_relocs: ?GotReloc.Index = switch (elf.getNode(ni)) {
4621	4640	.archive,
4622	4641	.archive_header,
	4642	.archive_elf_footer,
4623	4643	.elf,
4624	4644	.ehdr,
4625	4645	.shdr,
...	...	@@ -4660,7 +4680,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void {
4660	4680	if (got_relocs) \|ptr\| {
4661	4681	if (ptr.* != .none) {
4662	4682	for (elf.got_relocs.items[@backingInt(ptr.)..]) \|reloc\| {
4663		if (reloc.node != ni) break;
	4683	if (reloc.node != ni.toOptional()) break;
4664	4684	reloc.delete(elf);
4665	4685	}
4666	4686	}
...	...	@@ -4691,7 +4711,7 @@ fn flushMovedNodeRelocs(
4691	4711
4692	4712	if (first_got_reloc != .none) {
4693	4713	for (elf.got_relocs.items[@backingInt(first_got_reloc)..]) \|*reloc\| {
4694		if (reloc.node != node) break;
	4714	if (reloc.node != node.toOptional()) break;
4695	4715	reloc.apply(elf);
4696	4716	}
4697	4717	}
...	...	@@ -4756,7 +4776,7 @@ fn targetPtrSize(elf: *const Elf) u8 {
4756	4776	/// Page alignment for the target platform.
4757	4777	/// Usually this returns the maximum page size supported on the
4758	4778	/// target to maximize compatibility but there can be exceptions.
4759		fn targetPageAlign(elf: *const Elf) std.mem.Alignment {
	4779	fn targetPageAlign(elf: *const Elf) Alignment {
4760	4780	return .fromByteUnits(switch (elf.ehdrMachine()) {
4761	4781	.AARCH64 => 0x10000,
4762	4782	.LOONGARCH => 0x10000,
...	...	@@ -4810,7 +4830,7 @@ const PltInfo = struct {
4810	4830	/// entry, not the `.plt` entry. The `.plt.sec` section has no header entries, and is aligned to
4811	4831	/// the same boundary as the `.plt` section.
4812	4832	plt_sec: ?struct { entry_size: u8 },
4813		@"align": std.mem.Alignment,
	4833	@"align": Alignment,
4814	4834	entry_size: u8,
4815	4835	header_entries: u8,
4816	4836
...	...	@@ -4868,7 +4888,7 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo {
4868	4888	// TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`.
4869	4889	};
4870	4890	}
4871		fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {
	4891	pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {
4872	4892	const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer;
4873	4893	const Child = pointer_ty.child;
4874	4894	const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child);
...	...	@@ -4941,8 +4961,9 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr {
4941	4961	switch (elf.identClass()) {
4942	4962	.NONE, _ => unreachable,
4943	4963	inline else => \|class\| {
	4964	const shdrs_len = elf.shdrs.items.len + 1; // +1 for SHN_UNDEF
4944	4965	const shdr_slice: []class.ElfN().Shdr = @ptrCast(@alignCast(
4945		raw_slice[0 .. elf.shdrs.items.len * @sizeOf(class.ElfN().Shdr)],
	4966	raw_slice[0 .. shdrs_len * @sizeOf(class.ElfN().Shdr)],
4946	4967	));
4947	4968	const shdr_ptr = &shdr_slice[@backingInt(shndx)];
4948	4969	return @unionInit(ShdrPtr, @tagName(class), shdr_ptr);
...	...	@@ -4951,7 +4972,7 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr {
4951	4972	}
4952	4973
4953	4974	fn arHdrPtr(elf: Elf, ni: MappedFile.Node.Index) align(2) std.elf.ar_hdr {
4954		assert(elf.ni.elf != MappedFile.Node.Index.root);
	4975	assert(elf.ni.elf != .root);
4955	4976	const file_offset = ni.fileLocation(&elf.mf, false).offset;
4956	4977	return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) {
4957	4978	else => unreachable,
...	...	@@ -5049,13 +5070,13 @@ fn mapInputSection(elf: *Elf, opts: struct {
5049	5070	const name_shstrtab = try elf.string(.shstrtab, name);
5050	5071	const gop = try elf.section_by_name.getOrPut(gpa, name_shstrtab);
5051	5072	if (gop.found_existing) {
5052		break :existing @fromBackingInt(@intCast(gop.index));
	5073	break :existing @fromBackingInt(@intCast(gop.index + 1)); // +1 to account for SHN_UDNEF
5053	5074	}
5054	5075	errdefer assert(elf.section_by_name.pop().?.key == name_shstrtab);
5055	5076	const parent_node: MappedFile.Node.Index = parent: {
5056	5077	if (!opts.flags.ALLOC) break :parent elf.ni.elf;
5057	5078	if (opts.flags.EXECINSTR) break :parent elf.ni.text;
5058		if (opts.flags.TLS) break :parent elf.ni.tls;
	5079	if (opts.flags.TLS) break :parent elf.ni.tls.unwrap().?;
5059	5080	if (opts.flags.WRITE) break :parent elf.ni.data;
5060	5081	break :parent elf.ni.rodata;
5061	5082	};
...	...	@@ -5148,12 +5169,12 @@ fn navMapIndex(elf: Elf, zcu: Zcu, nav_index: InternPool.Nav.Index) Error!Node
5148	5169	break :section .data_rel_ro; // TODO: it would be better to use `.rodata` if the NAV value doesn't have relocs
5149	5170	}
5150	5171	};
5151		const alignment: InternPool.Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) {
	5172	const alignment: Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) {
5152	5173	.@"fn" => a: {
5153	5174	const mod = zcu.navFileScope(nav_index).mod.?;
5154	5175	const target = &mod.resolved_target.result;
5155	5176	const min = target_util.minFunctionAlignment(target);
5156		break :a switch (nav.resolved.?.@"align") {
	5177	break :a .fromIp(switch (nav.resolved.?.@"align") {
5157	5178	else => \|a\| a.maxStrict(min),
5158	5179	.none => switch (mod.optimize_mode) {
5159	5180	.debug,
...	...	@@ -5162,20 +5183,20 @@ fn navMapIndex(elf: Elf, zcu: Zcu, nav_index: InternPool.Nav.Index) Error!Node
5162	5183	=> target_util.defaultFunctionAlignment(target),
5163	5184	.small => min,
5164	5185	}.maxStrict(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)),
5165		};
	5186	});
5166	5187	},
5167	5188	else => switch (nav.resolved.?.@"align") {
5168		.none => Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu),
5169		else => \|a\| a,
	5189	.none => .fromIp(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)),
	5190	else => \|a\| .fromIp(a),
5170	5191	},
5171	5192	};
5172		try shndx.ensureAligned(elf, alignment.toStdMem());
5173		const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{
5174		.alignment = alignment.toStdMem(),
	5193	try shndx.ensureAligned(elf, alignment);
	5194	const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{
	5195	.alignment = alignment,
5175	5196	});
5176	5197	nav_gop.value_ptr.* = .{
5177	5198	.lsi = elf.addLocalSymbolAssumeCapacity(.{
5178		.node = node,
	5199	.node = .wrap(node),
5179	5200	.name = try elf.string(.strtab, nav.fqn.toSlice(ip)),
5180	5201	.value = 0,
5181	5202	.size = 0,
...	...	@@ -5204,19 +5225,19 @@ fn uavMapIndex(
5204	5225	try elf.pending_uavs.ensureUnusedCapacity(gpa, 1);
5205	5226
5206	5227	const abi_align = Value.fromInterned(uav_val).typeOf(zcu).abiAlignment(zcu);
5207		const resolved_align: InternPool.Alignment = switch (uav_align) {
5208		.none => abi_align,
5209		else => \|a\| a.minStrict(abi_align),
	5228	const resolved_align: Alignment = switch (uav_align) {
	5229	.none => .fromIp(abi_align),
	5230	else => \|a\| .fromIp(a.minStrict(abi_align)),
5210	5231	};
5211	5232
5212	5233	const uav_gop = elf.uavs.getOrPutAssumeCapacity(uav_val);
5213	5234	const umi: Node.UavMapIndex = @fromBackingInt(@intCast(uav_gop.index));
5214	5235	if (!uav_gop.found_existing) {
5215	5236	const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs
5216		try shndx.ensureAligned(elf, resolved_align.toStdMem());
5217		const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{
	5237	try shndx.ensureAligned(elf, resolved_align);
	5238	const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{
5218	5239	.moved = true, // see assert at end of `genUav`
5219		.alignment = resolved_align.toStdMem(),
	5240	.alignment = resolved_align,
5220	5241	});
5221	5242	var name_buf: [32]u8 = undefined;
5222	5243	const name = std.fmt.bufPrint(
...	...	@@ -5226,7 +5247,7 @@ fn uavMapIndex(
5226	5247	) catch unreachable;
5227	5248	uav_gop.value_ptr.* = .{
5228	5249	.lsi = elf.addLocalSymbolAssumeCapacity(.{
5229		.node = node,
	5250	.node = .wrap(node),
5230	5251	.name = try elf.string(.strtab, name),
5231	5252	.value = 0,
5232	5253	.size = 0,
...	...	@@ -5239,11 +5260,11 @@ fn uavMapIndex(
5239	5260	elf.const_prog_node.increaseEstimatedTotalItems(1);
5240	5261	elf.pending_uavs.appendAssumeCapacity(umi);
5241	5262	} else {
5242		const node = uav_gop.value_ptr.lsi.index().ptr(elf).node;
5243		const shndx = elf.getNode(node.parent(&elf.mf)).section;
5244		try shndx.ensureAligned(elf, resolved_align.toStdMem());
5245		if (resolved_align.toStdMem().order(node.alignment(&elf.mf)).compare(.gt)) {
5246		try node.realign(&elf.mf, gpa, resolved_align.toStdMem(), .{});
	5263	const node = uav_gop.value_ptr.lsi.index().ptr(elf).node.unwrap().?;
	5264	const shndx = elf.getNode(node.parent(&elf.mf).unwrap().?).section;
	5265	try shndx.ensureAligned(elf, resolved_align);
	5266	if (resolved_align.order(node.alignment(&elf.mf)).compare(.gt)) {
	5267	try node.realign(&elf.mf, gpa, resolved_align);
5247	5268	}
5248	5269	}
5249	5270	return umi;
...	...	@@ -5459,10 +5480,11 @@ fn loadObject(
5459	5480	.member = if (member) \|m\| try gpa.dupe(u8, m) else null,
5460	5481	.extra = undefined,
5461	5482	};
5462		if (elf.ni.elf != MappedFile.Node.Index.root) {
	5483	if (elf.ni.elf != .root) {
	5484	const archive_ni: MappedFile.Node.Index = .root;
5463	5485	try elf.nodes.ensureUnusedCapacity(gpa, 1);
5464		input.extra = .{ .node = try elf.mf.addLastChildNode(gpa, .root, .{
5465		.size = fl.size + @sizeOf(std.elf.ar_hdr),
	5486	input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{
	5487	.size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)),
5466	5488	.alignment = .@"2",
5467	5489	.next_moved = true,
5468	5490	.bubbles_moved = false,
...	...	@@ -5640,16 +5662,28 @@ fn loadObject(
5640	5662	.node_fixed = true,
5641	5663	},
5642	5664	};
5643		const need_align: std.mem.Alignment = .fromByteUnits(
	5665	const need_align: Alignment = .fromByteUnits(
5644	5666	std.math.ceilPowerOfTwoAssert(usize, @intCast(@max(section.shdr.addralign, 1))),
5645	5667	);
5646	5668	try opts.shndx.ensureAligned(elf, need_align);
5647		const ni = try elf.mf.addLastChildNode(gpa, opts.shndx.get(elf).ni, .{
5648		.size = section.shdr.size,
	5669	const add_node_opts: MappedFile.Node.AddOptions = .{
	5670	.size = need_align.forward(section.shdr.size),
5649	5671	.alignment = need_align,
5650	5672	.moved = true, // see assert at end of `flushInputSection`
5651		.fixed = opts.node_fixed,
5652		});
	5673	};
	5674	const ni = if (opts.node_fixed) ni: {
	5675	const shndx_ni = opts.shndx.get(elf).ni;
	5676	const after_oni: MappedFile.Node.Index.Optional = after: {
	5677	const last_ni = shndx_ni.last(&elf.mf).unwrap() orelse break :after .none;
	5678	break :after switch (last_ni.position(&elf.mf)) {
	5679	.header => .wrap(last_ni),
	5680	.footer, .floating => .none,
	5681	};
	5682	};
	5683	break :ni try shndx_ni.addHeaderChildAfter(&elf.mf, gpa, after_oni, add_node_opts);
	5684	} else ni: {
	5685	break :ni try opts.shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, add_node_opts);
	5686	};
5653	5687	elf.nodes.appendAssumeCapacity(.{
5654	5688	.input_section = @fromBackingInt(@intCast(elf.input_sections.items.len)),
5655	5689	});
...	...	@@ -5754,7 +5788,7 @@ fn loadObject(
5754	5788	),
5755	5789	.LOCAL => {
5756	5790	const lsi = elf.addLocalSymbolAssumeCapacity(.{
5757		.node = input_section_node,
	5791	.node = .wrap(input_section_node),
5758	5792	.name = try elf.string(.strtab, name),
5759	5793	.value = input_sym.value,
5760	5794	.size = input_sym.size,
...	...	@@ -5765,7 +5799,7 @@ fn loadObject(
5765	5799	},
5766	5800	.GLOBAL, .WEAK, .GNU_UNIQUE => \|bind\| {
5767	5801	si.* = elf.addGlobalSymbolAssumeCapacity(.{
5768		.node = input_section_node,
	5802	.node = .wrap(input_section_node),
5769	5803	.name = try .string(elf, name),
5770	5804	.value = input_sym.value,
5771	5805	.size = input_sym.size,
...	...	@@ -5893,7 +5927,7 @@ fn loadDso(elf: Elf, path: std.Build.Cache.Path, fr: Io.File.Reader) (LoadPars
5893	5927	return diags.failParse(path, "bad machine", .{});
5894	5928	if (ehdr.shnum > 0) try fr.seekTo(ehdr.shoff);
5895	5929	// We're going to need to know the alignment of every section later.
5896		const section_aligns = try gpa.alloc(std.mem.Alignment, ehdr.shnum);
	5930	const section_aligns = try gpa.alloc(Alignment, ehdr.shnum);
5897	5931	defer gpa.free(section_aligns);
5898	5932	const dynamic_sh: ElfN.Shdr, const dynsym_sh: ElfN.Shdr = sh: {
5899	5933	var dynamic_sh: ?ElfN.Shdr = null;
...	...	@@ -5999,7 +6033,7 @@ fn loadDso(elf: Elf, path: std.Build.Cache.Path, fr: Io.File.Reader) (LoadPars
5999	6033
6000	6034	// We need to guess the worst-case alignment of the symbol. Yes, I know this seems
6001	6035	// insane---refer to the doc comment on `alignment` in `Elf.dso_globals`.
6002		const sym_align: std.mem.Alignment = switch (sym.value) {
	6036	const sym_align: Alignment = switch (sym.value) {
6003	6037	0 => section_aligns[sym.shndx],
6004	6038	else => section_aligns[sym.shndx].min(@fromBackingInt(@intCast(@ctz(sym.value)))),
6005	6039	};
...	...	@@ -6017,8 +6051,8 @@ fn loadDso(elf: Elf, path: std.Build.Cache.Path, fr: Io.File.Reader) (LoadPars
6017	6051	// We have a copy relocation for this global, but the amount of space we
6018	6052	// reserved for it could be too small or underaligned!
6019	6053	try Section.Index.data.ensureAligned(elf, gop.value_ptr.alignment);
6020		try copied_global.node.resize(&elf.mf, gpa, gop.value_ptr.size);
6021		try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment, .{});
	6054	try copied_global.node.resizeLeaf(&elf.mf, gpa, gop.value_ptr.alignment.forward(gop.value_ptr.size));
	6055	try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment);
6022	6056	const global_ptr = elf.globalByName(name).?;
6023	6057	switch (elf.symPtr(global_ptr.symtab_index)) {
6024	6058	inline else => \|sym_ptr\| elf.targetStore(&sym_ptr.size, @intCast(gop.value_ptr.size)),
...	...	@@ -6158,7 +6192,7 @@ fn createInitFiniArraySection(
6158	6192	) Error!void {
6159	6193	assert(shndx.* == .UNDEF);
6160	6194	const gpa = elf.base.comp.gpa;
6161		const addr_align: std.mem.Alignment = switch (elf.identClass()) {
	6195	const addr_align: Alignment = switch (elf.identClass()) {
6162	6196	.NONE, _ => unreachable,
6163	6197	.@"32" => .@"4",
6164	6198	.@"64" => .@"8",
...	...	@@ -6178,14 +6212,14 @@ fn createInitFiniArraySection(
6178	6212	const start_sym_name = try elf.string(.strtab, "__" ++ name ++ "_start");
6179	6213	const end_sym_name = try elf.string(.strtab, "__" ++ name ++ "_end");
6180	6214	elf.setGlobalSymbolValue(start_sym_name, elf.globals.strong_def.getPtr(start_sym_name).?, .{
6181		.node = shndx.get(elf).ni,
	6215	.node = .wrap(shndx.get(elf).ni),
6182	6216	.value = shndx.vaddr(elf),
6183	6217	.size = 0,
6184	6218	.type = .NOTYPE,
6185	6219	.shndx = shndx.*,
6186	6220	});
6187	6221	elf.setGlobalSymbolValue(end_sym_name, elf.globals.strong_def.getPtr(end_sym_name).?, .{
6188		.node = shndx.get(elf).ni,
	6222	.node = .wrap(shndx.get(elf).ni),
6189	6223	.value = shndx.vaddr(elf),
6190	6224	.size = 0,
6191	6225	.type = .NOTYPE,
...	...	@@ -6218,7 +6252,7 @@ fn prelinkInner(elf: *Elf) Error!void {
6218	6252	const comp = elf.base.comp;
6219	6253	const gpa = comp.gpa;
6220	6254
6221		if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == MappedFile.Node.Index.root) {
	6255	if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == .root) {
6222	6256	// We're using self-hosted codegen---add an input representing the Zig "object".
6223	6257	try elf.ensureUnusedSymbolCapacity(1, .all_local);
6224	6258	try elf.inputs.ensureUnusedCapacity(gpa, 1);
...	...	@@ -6239,8 +6273,6 @@ fn prelinkInner(elf: *Elf) Error!void {
6239	6273	};
6240	6274	elf.input_pending_index += 1;
6241	6275	}
6242
6243		try elf.ensureElfNodeSize();
6244	6276	}
6245	6277
6246	6278	fn prepareDynamic(elf: *Elf) Error!void {
...	...	@@ -6265,7 +6297,7 @@ fn prepareDynamic(elf: *Elf) Error!void {
6265	6297
6266	6298	const dynamic_size = dynamic_len * 2 * elf.targetPtrSize();
6267	6299
6268		try elf.shndx.dynamic.get(elf).ni.resize(&elf.mf, comp.gpa, dynamic_size);
	6300	try elf.shndx.dynamic.get(elf).ni.resizeLeaf(&elf.mf, comp.gpa, dynamic_size);
6269	6301	switch (elf.shdrPtr(elf.shndx.dynamic)) {
6270	6302	inline else => \|shdr\| elf.targetStore(&shdr.size, @intCast(dynamic_size)),
6271	6303	}
...	...	@@ -6388,10 +6420,9 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct {
6388	6420	size: std.elf.Xword = 0,
6389	6421	link: std.elf.Word = 0,
6390	6422	info: std.elf.Word = 0,
6391		addralign: std.mem.Alignment = .@"1",
	6423	addralign: Alignment = .@"1",
6392	6424	entsize: std.elf.Word = 0,
6393		node_align: std.mem.Alignment = .@"1",
6394		fixed: bool = false,
	6425	node_align: Alignment = .@"1",
6395	6426	}) Error!Section.Index {
6396	6427	switch (opts.type) {
6397	6428	.NULL => assert(opts.size == 0),
...	...	@@ -6435,19 +6466,20 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct {
6435	6466	break :shndx .{ @fromBackingInt(shndx), @as(u64, elf.targetLoad(&ehdr.shentsize)) * @as(u64, shnum) };
6436	6467	},
6437	6468	};
6438		try elf.ensureNodeSize(elf.ni.shdr, new_shdr_size);
6439		const ni = try elf.mf.addLastChildNode(gpa, switch (elf.ehdrType()) {
	6469	try elf.ni.shdr.ensureMinimumSize(&elf.mf, gpa, new_shdr_size);
	6470	const parent_ni = switch (elf.ehdrType()) {
6440	6471	.REL => elf.ni.elf,
6441	6472	.EXEC, .DYN => segment_ni,
6442		}, .{
6443		.size = opts.size,
	6473	};
	6474	assert(opts.addralign.check(opts.size));
	6475	const ni = try parent_ni.addFloatingChild(&elf.mf, gpa, .{
	6476	.size = opts.node_align.forward(opts.size),
6444	6477	.alignment = opts.addralign.max(opts.node_align),
6445		.fixed = opts.fixed,
6446	6478	.resized = opts.size > 0,
6447	6479	});
6448	6480	const addr = elf.computeNodeVAddr(ni);
6449	6481	const lsi: Symbol.LocalIndex = if (opts.flags.ALLOC) elf.addLocalSymbolAssumeCapacity(.{
6450		.node = ni,
	6482	.node = .wrap(ni),
6451	6483	.name = .empty,
6452	6484	.value = addr,
6453	6485	.size = 0,
...	...	@@ -6499,7 +6531,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize)
6499	6531
6500	6532	assert(elf.section_by_name.count() == elf.shdrs.items.len);
6501	6533	try elf.section_by_name.ensureUnusedCapacity(gpa, 1);
6502		const rela_shndx = try elf.addSection(.none, .{
	6534	const rela_shndx = try elf.addSection(elf.ni.elf, .{
6503	6535	.name = rela_name,
6504	6536	.type = .RELA,
6505	6537	.link = @backingInt(Section.Index.symtab),
...	...	@@ -6528,7 +6560,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize)
6528	6560	.NONE, _ => unreachable,
6529	6561	inline else => \|ct_class\| (elf.got.count() + new_got_entries) * @sizeOf(ct_class.ElfN().Addr),
6530	6562	};
6531		try elf.ensureNodeSize(elf.shndx.got.get(elf).ni, need_got_size);
	6563	try elf.shndx.got.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_got_size);
6532	6564
6533	6565	if (elf.shndx.dynamic != .UNDEF) {
6534	6566	try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, new_got_entries);
...	...	@@ -6546,7 +6578,6 @@ fn addRelocAssumeCapacity(
6546	6578	addend: i64,
6547	6579	@"type": MachineRelocType,
6548	6580	) (Error \|\| error{ UnknownRelocation, NonStaticRelocation, UnimplementedRelocation })!void {
6549		assert(node != .none);
6550	6581	switch (elf.ehdrType()) {
6551	6582	.REL => {
6552	6583	const rela_shndx = elf.getNodeShndx(node).get(elf).rela.shndx;
...	...	@@ -6894,7 +6925,6 @@ fn addSymbolRelocAssumeCapacity(
6894	6925	@"type": SymbolReloc.Type,
6895	6926	) Error!void {
6896	6927	assert(elf.ehdrType() != .REL);
6897		assert(node != .none);
6898	6928
6899	6929	const rela_index: Section.RelaIndex.Optional = r: {
6900	6930	if (elf.shndx.dynamic == .UNDEF) break :r .none;
...	...	@@ -7042,6 +7072,7 @@ fn addGotRelocAssumeCapacity(
7042	7072	switch (elf.getNode(node)) {
7043	7073	.archive,
7044	7074	.archive_header,
	7075	.archive_elf_footer,
7045	7076	.elf,
7046	7077	.ehdr,
7047	7078	.shdr,
...	...	@@ -7089,7 +7120,7 @@ fn addGotRelocAssumeCapacity(
7089	7120	}
7090	7121
7091	7122	elf.got_relocs.appendAssumeCapacity(.{
7092		.node = node,
	7123	.node = .wrap(node),
7093	7124	.offset = offset,
7094	7125	.target = target,
7095	7126	.addend = addend,
...	...	@@ -7111,7 +7142,7 @@ fn updateGotEntry(elf: *Elf, got_index: usize) void {
7111	7142	.tpoff => \|sym_id\| val: {
7112	7143	// Only the executable's per-module TLS block is at a known offset from the TLS pointer.
7113	7144	if (elf.base.comp.config.output_mode == .Exe and elf.classifySymbolValue(sym_id) != .dynamic) {
7114		const tls_phndx = elf.getNode(elf.ni.tls).segment;
	7145	const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment;
7115	7146	const tls_size: u64 = switch (elf.phdrSlice()) {
7116	7147	inline else => \|phdr\| tls_size: {
7117	7148	assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS);
...	...	@@ -7284,8 +7315,8 @@ fn maybeAddCopyRelocation(elf: *Elf, global_name: String(.strtab)) Error!bool {
7284	7315	try Section.Index.data.ensureAligned(elf, dso_global.alignment);
7285	7316
7286	7317	try elf.nodes.ensureUnusedCapacity(gpa, 1);
7287		const node = try elf.mf.addLastChildNode(gpa, Section.Index.data.get(elf).ni, .{
7288		.size = dso_global.size,
	7318	const node = try Section.Index.data.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{
	7319	.size = dso_global.alignment.forward(dso_global.size),
7289	7320	.alignment = dso_global.alignment,
7290	7321	});
7291	7322	errdefer comptime unreachable;
...	...	@@ -7336,7 +7367,7 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index)
7336	7367	if (!Type.fromInterned(nav.resolved.?.type).hasRuntimeBits(zcu)) return;
7337	7368
7338	7369	const nmi = try elf.navMapIndex(zcu, nav_index);
7339		const ni = nmi.symbol(elf).index().ptr(elf).node;
	7370	const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?;
7340	7371	elf.resetNodeRelocs(ni);
7341	7372
7342	7373	// Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be
...	...	@@ -7392,7 +7423,7 @@ fn updateFuncInner(
7392	7423
7393	7424	const nmi = try elf.navMapIndex(zcu, func.owner_nav);
7394	7425	log.debug("updateFunc({f}) = {d}", .{ nav.fqn.fmt(ip), nmi.symbol(elf) });
7395		const ni = nmi.symbol(elf).index().ptr(elf).node;
	7426	const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?;
7396	7427	elf.resetNodeRelocs(ni);
7397	7428
7398	7429	// Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be
...	...	@@ -7471,7 +7502,6 @@ fn flushInner(
7471	7502
7472	7503	try elf.prepareDynamic();
7473	7504
7474		try elf.ensureElfNodeSize();
7475	7505	while (try elf.idle(tid)) {}
7476	7506
7477	7507	// We've done the final `idle` loop, so everything is at its final place in the file. We have a
...	...	@@ -7677,7 +7707,7 @@ fn idleProgNode(
7677	7707	break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{
7678	7708	ii.path(elf).fmtEscapeString(),
7679	7709	fmtMemberString(ii.member(elf)),
7680		elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf),
	7710	elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf),
7681	7711	}) catch &name;
7682	7712	},
7683	7713	.nav => \|nmi\| {
...	...	@@ -7724,8 +7754,6 @@ fn genPending(elf: *Elf, pt: Zcu.PerThread) Error!void {
7724	7754	};
7725	7755	break;
7726	7756	}
7727
7728		try elf.ensureElfNodeSize();
7729	7757	}
7730	7758
7731	7759	fn genUav(
...	...	@@ -7737,7 +7765,7 @@ fn genUav(
7737	7765	const gpa = comp.gpa;
7738	7766
7739	7767	const uav_val = umi.uavValue(elf);
7740		const ni = umi.symbol(elf).index().ptr(elf).node;
	7768	const ni = umi.symbol(elf).index().ptr(elf).node.unwrap().?;
7741	7769	elf.resetNodeRelocs(ni);
7742	7770
7743	7771	var nw: MappedFile.Node.Writer = undefined;
...	...	@@ -7766,7 +7794,7 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void {
7766	7794	const gpa = zcu.gpa;
7767	7795
7768	7796	const lazy = lmr.lazySymbol(elf);
7769		const ni = lmr.symbol(elf).index().ptr(elf).node;
	7797	const ni = lmr.symbol(elf).index().ptr(elf).node.unwrap().?;
7770	7798	elf.resetNodeRelocs(ni);
7771	7799
7772	7800	// Ensure the lazy node is marked as moved so that once we're done, `flushMoved` will eventually
...	...	@@ -7842,7 +7870,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void {
7842	7870	fr.seekTo(file_loc.offset) catch \|err\| switch (err) {
7843	7871	error.Canceled => \|e\| return e,
7844	7872	else => \|e\| return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{
7845		elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf),
	7873	elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf),
7846	7874	path.fmtEscapeString(),
7847	7875	fmtMemberString(ii.member(elf)),
7848	7876	e,
...	...	@@ -7853,7 +7881,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void {
7853	7881	defer nw.deinit();
7854	7882	const n_bytes = nw.interface.sendFileAll(&fr, .limited(@intCast(file_loc.size))) catch \|err\| switch (err) {
7855	7883	error.ReadFailed => return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{
7856		elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf),
	7884	elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf),
7857	7885	path.fmtEscapeString(),
7858	7886	fmtMemberString(ii.member(elf)),
7859	7887	fr.err orelse (fr.seek_err orelse fr.size_err.?),
...	...	@@ -7861,7 +7889,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void {
7861	7889	error.WriteFailed => return nw.err.?,
7862	7890	};
7863	7891	if (n_bytes != file_loc.size) return diags.fail("failed to read input section '{s}' from \"{f}{f}\": unexpected eof", .{
7864		elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf),
	7892	elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf),
7865	7893	path.fmtEscapeString(),
7866	7894	fmtMemberString(ii.member(elf)),
7867	7895	});
...	...	@@ -7888,8 +7916,10 @@ fn flushElfOffset(elf: *Elf, ni: MappedFile.Node.Index) void {
7888	7916	}
7889	7917	},
7890	7918	}
7891		var child_it = ni.children(&elf.mf);
7892		while (child_it.next()) \|child_ni\| elf.flushElfOffset(child_ni);
	7919	var child_oni = ni.first(&elf.mf);
	7920	while (child_oni.unwrap()) \|child_ni\| : (child_oni = child_ni.next(&elf.mf)) {
	7921	elf.flushElfOffset(child_ni);
	7922	}
7893	7923	},
7894	7924	.section => \|shndx\| switch (elf.shdrPtr(shndx)) {
7895	7925	inline else => \|shdr\| elf.targetStore(&shdr.offset, @intCast(elf_offset)),
...	...	@@ -7906,7 +7936,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
7906	7936
7907	7937	switch (elf.getNode(ni)) {
7908	7938	.archive, .archive_header => unreachable,
7909		.elf => {},
	7939	.archive_elf_footer, .elf => {},
7910	7940	.ehdr, .shdr => elf.flushElfOffset(ni),
7911	7941	.segment => \|phndx\| {
7912	7942	elf.flushElfOffset(ni);
...	...	@@ -7994,7 +8024,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
7994	8024	const ii = isi.input(elf);
7995	8025	var lsi, const end_lsi = ii.localSymbolRange(elf);
7996	8026	while (lsi != end_lsi) : (lsi = @fromBackingInt(@backingInt(lsi) + 1)) {
7997		if (lsi.index().ptr(elf).node != ni) continue;
	8027	if (lsi.index().ptr(elf).node != ni.toOptional()) continue;
7998	8028	const visibility: std.elf.STV = switch (elf.symPtr(lsi.index())) {
7999	8029	inline else => \|sym\| elf.targetLoad(&sym.other).visibility,
8000	8030	};
...	...	@@ -8079,7 +8109,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
8079	8109	/// moving or resizing of a segment could reorder them and thereby affect how we handle future
8080	8110	/// changes to segments.
8081	8111	fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Error!void {
8082		const segment_ni = elf.phdrs.items[orig_phndx];
	8112	const segment_ni = elf.phdrs.items[orig_phndx].unwrap().?;
8083	8113	assert(elf.getNode(segment_ni).segment == orig_phndx);
8084	8114	const page_align = elf.targetPageAlign();
8085	8115	const node_align = segment_ni.alignment(&elf.mf);
...	...	@@ -8165,7 +8195,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
8165	8195	const next_ni = elf.phdrs.items[next_phndx];
8166	8196	elf.phdrs.items[phndx] = next_ni;
8167	8197	elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx };
8168		elf.phdrs.items[next_phndx] = segment_ni;
	8198	elf.phdrs.items[next_phndx] = .wrap(segment_ni);
8169	8199	elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) };
8170	8200	phndx = @intCast(next_phndx);
8171	8201	}
...	...	@@ -8189,9 +8219,10 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo
8189	8219	_, const size = ni.location(&elf.mf).resolve(&elf.mf);
8190	8220	switch (elf.getNode(ni)) {
8191	8221	.archive => {
8192		var child_it = ni.reverseChildren(&elf.mf);
8193		if (child_it.next()) \|last_ni\| {
8194		if (child_it.next()) \|prev_ni\| if (prev_ni.hasNextMoved(&elf.mf)) return;
	8222	if (ni.last(&elf.mf).unwrap()) \|last_ni\| {
	8223	if (last_ni.prev(&elf.mf).unwrap()) \|prev_ni\| {
	8224	if (prev_ni.hasNextMoved(&elf.mf)) return;
	8225	}
8195	8226	const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf);
8196	8227	_ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{
8197	8228	size - offset,
...	...	@@ -8199,11 +8230,11 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo
8199	8230	}
8200	8231	},
8201	8232	.archive_header, .elf => {},
8202		.ehdr => unreachable,
	8233	.ehdr, .archive_elf_footer => unreachable,
8203	8234	.shdr => {},
8204	8235	.segment => \|phndx\| switch (elf.phdrSlice()) {
8205	8236	inline else => \|phdr\| {
8206		assert(elf.phdrs.items[phndx] == ni);
	8237	assert(elf.phdrs.items[phndx].unwrap().? == ni);
8207	8238	const ph = &phdr[phndx];
8208	8239	elf.targetStore(&ph.filesz, @intCast(size));
8209	8240	switch (elf.targetLoad(&ph.type)) {
...	...	@@ -8284,6 +8315,7 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!
8284	8315
8285	8316	switch (elf.getNode(ni)) {
8286	8317	.archive,
	8318	.archive_elf_footer,
8287	8319	.ehdr,
8288	8320	.shdr,
8289	8321	.segment,
...	...	@@ -8301,51 +8333,45 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!
8301	8333	break :member_offset switch (tag) {
8302	8334	else => unreachable,
8303	8335	.archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true },
8304		.elf, .input_member => .{ offset, switch (ni.prev(&elf.mf)) {
8305		.none => unreachable,
8306		else => \|prev_ni\| !prev_ni.hasNextMoved(&elf.mf),
8307		} },
	8336	.elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) },
8308	8337	};
8309	8338	};
8310		const member_size = member_end: switch (ni.next(&elf.mf)) {
8311		else => \|next_ni\| {
8312		const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf);
8313		const next_member_size = next_member_end: switch (next_ni.next(&elf.mf)) {
8314		else => \|next_next_ni\| {
8315		const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf);
8316		break :next_member_end next_next_offset - @sizeOf(std.elf.ar_hdr);
8317		},
8318		.none => {
8319		_, const parent_size =
8320		ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf);
8321		break :next_member_end parent_size;
8322		},
8323		} - next_offset;
8324		const ar_hdr = elf.arHdrPtr(next_ni);
8325		var name_buf: [16]u8 = undefined;
8326		_ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{
8327		switch (elf.getNode(next_ni)) {
8328		else => unreachable,
8329		.elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}),
8330		.input_member => \|ii\| std.mem.print(&name_buf, "{s}/", .{
8331		std.fs.path.basename(ii.path(elf).sub_path),
8332		}),
8333		} catch @panic("TODO: long archive member names"),
8334		}) catch @panic("TODO: long archive member names");
8335		ar_hdr.ar_date = "0 ".*;
8336		ar_hdr.ar_uid = "0 ".*;
8337		ar_hdr.ar_gid = "0 ".*;
8338		ar_hdr.ar_mode = "644 ".*;
8339		_ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch
8340		@panic("archive member too large");
8341		ar_hdr.ar_fmag = std.elf.ARFMAG.*;
8342		break :member_end next_offset - @sizeOf(std.elf.ar_hdr);
8343		},
8344		.none => {
8345		_, const parent_size = ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf);
8346		break :member_end parent_size;
8347		},
8348		} - member_offset;
	8339	const member_size = if (ni.next(&elf.mf).unwrap()) \|next_ni\| member_size: {
	8340	const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf);
	8341	const next_member_size = if (next_ni.next(&elf.mf).unwrap()) \|next_next_ni\| next_member_size: {
	8342	const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf);
	8343	const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr);
	8344	break :next_member_size next_member_end - next_offset;
	8345	} else next_member_size: {
	8346	_, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf);
	8347	const next_member_end = parent_size;
	8348	break :next_member_size next_member_end - next_offset;
	8349	};
	8350	const ar_hdr = elf.arHdrPtr(next_ni);
	8351	var name_buf: [16]u8 = undefined;
	8352	_ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{
	8353	switch (elf.getNode(next_ni)) {
	8354	else => unreachable,
	8355	.elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}),
	8356	.input_member => \|ii\| std.mem.print(&name_buf, "{s}/", .{
	8357	std.fs.path.basename(ii.path(elf).sub_path),
	8358	}),
	8359	} catch @panic("TODO: long archive member names"),
	8360	}) catch @panic("TODO: long archive member names");
	8361	ar_hdr.ar_date = "0 ".*;
	8362	ar_hdr.ar_uid = "0 ".*;
	8363	ar_hdr.ar_gid = "0 ".*;
	8364	ar_hdr.ar_mode = "644 ".*;
	8365	_ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch
	8366	@panic("archive member too large");
	8367	ar_hdr.ar_fmag = std.elf.ARFMAG.*;
	8368	const member_end = next_offset - @sizeOf(std.elf.ar_hdr);
	8369	break :member_size member_end - member_offset;
	8370	} else member_size: {
	8371	_, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf);
	8372	const member_end = parent_size;
	8373	break :member_size member_end - member_offset;
	8374	};
8349	8375	if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{
8350	8376	member_size,
8351	8377	}) catch @panic("archive member too large");
...	...	@@ -8735,8 +8761,6 @@ fn updateExportInner(
8735	8761	.uav => \|uav\| (try elf.uavMapIndex(uav, .none)).symbol(elf),
8736	8762	};
8737	8763
8738		try elf.ensureElfNodeSize();
8739
8740	8764	// Initialize the global symbol with the same values that the local one currently has. If the
8741	8765	// NAV/UAV is updated, then `updateNavInner` or `genUav` will update the global symbol sizes,
8742	8766	// and `flushMoved` will update their values.
...	...	@@ -8775,12 +8799,13 @@ fn updateExportInner(
8775	8799	// only emitting this error if the symbol we're conflicting with comes from an input
8776	8800	// section (as opposed to the ZCU).
8777	8801	const conflicting_global = elf.globalByName(try elf.string(.strtab, name)).?;
8778		const conflicting_node = conflicting_global.symtab_index.ptr(elf).node;
8779		if (elf.getNode(conflicting_node) == .input_section) {
8780		return elf.base.comp.link_diags.fail(
8781		"multiple definitions of '{s}'",
8782		.{name},
8783		);
	8802	if (conflicting_global.symtab_index.ptr(elf).node.unwrap()) \|conflicting_node\| {
	8803	if (elf.getNode(conflicting_node) == .input_section) {
	8804	return elf.base.comp.link_diags.fail(
	8805	"multiple definitions of '{s}'",
	8806	.{name},
	8807	);
	8808	}
8784	8809	}
8785	8810	},
8786	8811	};
...	...	@@ -8842,7 +8867,7 @@ pub fn printNode(
8842	8867	try w.print("({f}{f}, {s})", .{
8843	8868	ii.path(elf).fmtEscapeString(),
8844	8869	fmtMemberString(ii.member(elf)),
8845		elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf),
	8870	elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf),
8846	8871	});
8847	8872	},
8848	8873	.copied_global => \|name\| try w.print("(copy:{s})", .{name.slice(elf)}),
...	...	@@ -8873,25 +8898,27 @@ pub fn printNode(
8873	8898	{
8874	8899	const mf_node = &elf.mf.nodes.items[@backingInt(ni)];
8875	8900	const off, const size = mf_node.location().resolve(&elf.mf);
8876		try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}{s}\n", .{
	8901	try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}{s}\n", .{
8877	8902	@backingInt(ni),
8878	8903	off,
8879	8904	size,
8880	8905	mf_node.flags.alignment.toByteUnits(),
8881		if (mf_node.flags.fixed) " fixed" else "",
	8906	mf_node.flags.position,
8882	8907	if (mf_node.flags.moved) " moved" else "",
8883	8908	if (mf_node.flags.next_moved) " next_moved" else "",
8884	8909	if (mf_node.flags.resized) " resized" else "",
8885	8910	if (mf_node.flags.has_content) " has_content" else "",
8886	8911	});
8887	8912	}
8888		var leaf = true;
8889		var child_it = ni.children(&elf.mf);
8890		while (child_it.next()) \|child_ni\| {
8891		leaf = false;
8892		try elf.printNode(tid, w, child_ni, indent + 1);
	8913	if (ni.first(&elf.mf).unwrap()) \|first_ni\| {
	8914	// non-leaf, just print children
	8915	var child_ni = first_ni;
	8916	while (true) {
	8917	try elf.printNode(tid, w, child_ni, indent + 1);
	8918	child_ni = child_ni.next(&elf.mf).unwrap() orelse break;
	8919	}
	8920	return;
8893	8921	}
8894		if (!leaf) return;
8895	8922	const file_loc = ni.fileLocation(&elf.mf, false);
8896	8923	var address = file_loc.offset;
8897	8924	if (file_loc.size == 0) {
...	...	@@ -8916,16 +8943,16 @@ pub fn printNode(
8916	8943	}
8917	8944	}
8918	8945
8919		fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignment) Error!void {
	8946	fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error!void {
8920	8947	const gpa = elf.base.comp.gpa;
8921	8948	// We need to loop through parent nodes because segments may be nested (e.g. a PT_TLS segment
8922	8949	// inside a PT_LOAD segment).
8923	8950	var phndx = start_phndx;
8924	8951	while (true) {
8925	8952	// Align the actual node
8926		const seg_ni = elf.phdrs.items[phndx];
	8953	const seg_ni = elf.phdrs.items[phndx].unwrap().?;
8927	8954	if (min_align.compare(.gt, seg_ni.alignment(&elf.mf))) {
8928		try seg_ni.realign(&elf.mf, gpa, min_align, .{});
	8955	try seg_ni.realign(&elf.mf, gpa, min_align);
8929	8956	}
8930	8957	// Update the phdr `@"align"` field if necessary
8931	8958	switch (elf.phdrSlice()) {
...	...	@@ -8948,7 +8975,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen
8948	8975	},
8949	8976	}
8950	8977	// Continue on to the parent segment, if any
8951		switch (elf.getNode(seg_ni.parent(&elf.mf))) {
	8978	switch (elf.getNode(seg_ni.parent(&elf.mf).unwrap().?)) {
8952	8979	.segment => \|parent_phndx\| phndx = parent_phndx,
8953	8980	.elf => return,
8954	8981	else => unreachable,
...	...	@@ -8956,26 +8983,6 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen
8956	8983	}
8957	8984	}
8958	8985
8959		/// Must be called deterministically after any call to `MappedFile.Node.Index.resize`
8960		/// (of `elf.ni.elf` or one of its children) before any possible calls to `idle`.
8961		fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void {
8962		if (elf.ni.elf == MappedFile.Node.Index.root) return;
8963		var child_it = elf.ni.elf.reverseChildren(&elf.mf);
8964		const last_end = if (child_it.next()) \|last_ni\| last_end: {
8965		const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf);
8966		break :last_end last_offset + last_size;
8967		} else 0;
8968		try elf.ensureNodeSize(elf.ni.elf, last_end + @sizeOf(std.elf.ar_hdr));
8969		}
8970
8971		fn ensureNodeSize(elf: *Elf, node: MappedFile.Node.Index, need_size: u64) MappedFile.Error!void {
8972		_, const node_size = node.location(&elf.mf).resolve(&elf.mf);
8973		if (need_size <= node_size) return;
8974		const gpa = elf.base.comp.gpa;
8975		const new_size = need_size + need_size / MappedFile.growth_factor;
8976		try node.resize(&elf.mf, gpa, new_size);
8977		}
8978
8979	8986	/// If `sym` has a PLT entry, returns the address of that entry (specifically, the address which a
8980	8987	/// branch to the PLT should target). If `sym` does not have a PLT entry, returns `null`.
8981	8988	fn pltEntryTargetAddr(elf: *Elf, sym: Symbol.Id) ?u64 {

src/link/MappedFile.zig+1906-917

...	...	@@ -13,14 +13,14 @@ const windows = std.os.windows;
13	13
14	14	io: Io,
15	15	flags: packed struct {
16		block_size: std.mem.Alignment,
	16	block_size: Alignment,
17	17	copy_file_range_unsupported: bool,
18	18	fallocate_punch_hole_unsupported: bool,
19	19	fallocate_insert_range_unsupported: bool,
20	20	},
21	21	memory_map: Io.File.MemoryMap,
22	22	nodes: std.ArrayList(Node),
23		free_ni: Node.Index,
	23	free_ni: Node.Index.Optional,
24	24	large: std.ArrayList(u64),
25	25	updates: std.ArrayList(Node.Index),
26	26	/// This progress node's estimated total items is increased once for each node appended to `updates`.
...	...	@@ -62,6 +62,94 @@ pub const Error = Allocator.Error \|\| Io.Cancelable \|\| error{
62	62	MappedFileIo,
63	63	};
64	64
	65	/// This separate `Alignment` type exists because neither of the other options is really suitable:
	66	///
	67	/// * `std.mem.Alignment` is based on `usize`, which---while technically okay since the file is
	68	/// memory-mapped---is in practice very annoying to work with in linker implementations
	69	///
	70	/// * `InternPool.Alignment` is based on `u64`, which is better, but it has the value `.none`, which
	71	/// is also really annoying to handle, because no alignment is ever nullable in this API
	72	///
	73	/// At some point we should probably just change `InternPool.Alignment` to be non-optional, and add
	74	/// a new `InternPool.Alignment.Optional` type for the case where it can actually be `.none`. At
	75	/// that point we can transition this code to using `InternPool.Alignment` (although it should
	76	/// probably be namespaced elsewhere, it has nothing to do with the `InternPool`!).
	77	pub const Alignment = enum(u6) {
	78	@"1" = 0,
	79	@"2" = 1,
	80	@"4" = 2,
	81	@"8" = 3,
	82	@"16" = 4,
	83	@"32" = 5,
	84	@"64" = 6,
	85	_,
	86
	87	pub fn fromIp(a: @import("../InternPool.zig").Alignment) Alignment {
	88	assert(a != .none);
	89	return @bitCast(a);
	90	}
	91
	92	pub fn toLog2Units(a: Alignment) u6 {
	93	return @backingInt(a);
	94	}
	95
	96	pub fn fromLog2Units(a: u6) Alignment {
	97	return @fromBackingInt(a);
	98	}
	99
	100	pub fn toByteUnits(a: Alignment) u64 {
	101	return @as(u64, 1) << @backingInt(a);
	102	}
	103
	104	pub fn fromByteUnits(n: u64) Alignment {
	105	assert(std.math.isPowerOfTwo(n));
	106	return @fromBackingInt(@intCast(@ctz(n)));
	107	}
	108
	109	pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order {
	110	return std.math.order(@backingInt(lhs), @backingInt(rhs));
	111	}
	112
	113	pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool {
	114	return std.math.compare(@backingInt(lhs), op, @backingInt(rhs));
	115	}
	116
	117	pub fn max(lhs: Alignment, rhs: Alignment) Alignment {
	118	return @fromBackingInt(@max(@backingInt(lhs), @backingInt(rhs)));
	119	}
	120
	121	pub fn min(lhs: Alignment, rhs: Alignment) Alignment {
	122	return @fromBackingInt(@min(@backingInt(lhs), @backingInt(rhs)));
	123	}
	124
	125	pub inline fn of(comptime T: type) Alignment {
	126	return comptime .fromByteUnits(@alignOf(T));
	127	}
	128
	129	/// Given that a base address is known to be aligned to `a`, computes the known alignment of
	130	/// that base address plus `off`.
	131	pub fn offset(a: Alignment, off: u64) Alignment {
	132	return .fromLog2Units(@min(a.toLog2Units(), @ctz(off)));
	133	}
	134
	135	/// Align an address forwards to this alignment.
	136	pub fn forward(a: Alignment, addr: u64) u64 {
	137	const x = (@as(u64, 1) << @backingInt(a)) - 1;
	138	return (addr + x) & ~x;
	139	}
	140
	141	/// Align an address backwards to this alignment.
	142	pub fn backward(a: Alignment, addr: u64) u64 {
	143	const x = (@as(u64, 1) << @backingInt(a)) - 1;
	144	return addr & ~x;
	145	}
	146
	147	/// Check if an address is aligned to this amount.
	148	pub fn check(a: Alignment, addr: u64) bool {
	149	return @ctz(addr) >= @backingInt(a);
	150	}
	151	};
	152
65	153	pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error \|\| Io.Cancelable \|\| IoError)!MappedFile {
66	154	var mf: MappedFile = .{
67	155	.io = io,
...	...	@@ -95,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error \|\| Io.Cancel
95	183	.fallocate_insert_range_unsupported = false,
96	184	.fallocate_punch_hole_unsupported = false,
97	185	};
98		try mf.nodes.ensureUnusedCapacity(gpa, 1);
99		const root_ni = try mf.addNode(gpa, .{ .add_node = .{
100		.size = size,
101		.alignment = mf.flags.block_size,
102		.fixed = true,
103		} });
104		assert(root_ni == Node.Index.root);
105		try mf.ensureTotalCapacityInner(@intCast(size));
	186
	187	const root_location: Node.Location = l: {
	188	if (std.math.cast(u32, size)) \|small_size\| {
	189	break :l .{ .small = .{ .offset = 0, .size = small_size } };
	190	}
	191	try mf.large.appendSlice(gpa, &.{ 0, size });
	192	break :l .{ .large = .{ .index = 0 } };
	193	};
	194	try mf.nodes.append(gpa, .{
	195	.parent = .none,
	196	.prev = .none,
	197	.next = .none,
	198	.first = .none,
	199	.last = .none,
	200	.flags = .{
	201	.alignment = mf.flags.block_size,
	202	.position = .floating,
	203	.bubbles_moved = true,
	204	.enable_next_moved = false,
	205	.location_tag = root_location,
	206	.moved = false,
	207	.resized = false,
	208	.next_moved = false,
	209	.has_content = false,
	210	},
	211	.location_payload = switch (root_location) {
	212	.small => \|small\| .{ .small = small },
	213	.large => \|large\| .{ .large = large },
	214	},
	215	});
	216
	217	mf.ensureTotalCapacity(@intCast(size)) catch \|err\| switch (err) {
	218	error.MappedFileIo => return mf.io_err.?,
	219	else => \|e\| return e,
	220	};
	221
106	222	return mf;
107	223	}
108	224
...	...	@@ -117,32 +233,61 @@ pub fn deinit(mf: *MappedFile, gpa: Allocator) void {
117	233	}
118	234
119	235	pub const Node = extern struct {
120		parent: Node.Index,
121		prev: Node.Index,
122		next: Node.Index,
123		first: Node.Index,
124		last: Node.Index,
	236	parent: Node.Index.Optional,
	237	prev: Node.Index.Optional,
	238	next: Node.Index.Optional,
	239	first: Node.Index.Optional,
	240	last: Node.Index.Optional,
125	241	flags: Flags,
126	242	location_payload: Location.Payload,
127	243
	244	/// Any non-leaf node may designate its first N children as "header" nodes. This means that its
	245	/// first N children must be densely packed together and positioned at the start of the parent.
	246	/// The implementation guarantees that it will never re-order these nodes, nor will it introduce
	247	/// padding between them.
	248	///
	249	/// Likewise, any non-leaf node may designate its last M children as "footer" nodes, which are
	250	/// like header nodes except they are positioned at the end of the parent rather than the
	251	/// start.
	252	///
	253	/// Nodes which are neither headers nor footers are called "floating". The implementation is
	254	/// always free to re-order floating nodes relative to one another, and to add or remove padding
	255	/// between them.
	256	pub const Position = enum(u2) {
	257	header,
	258	footer,
	259	floating,
	260	};
	261
128	262	pub const Flags = packed struct(u32) {
	263	/// While the number of header and footer nodes within a parent node is logically a part of
	264	/// that parent, we actually store this information on the child nodes for efficiency: this
	265	/// field indicates whether each child is a header node, a footer node, or a floating node.
	266	///
	267	/// This value is meaningless for the root node, so is arbitrarily set to `.floating`.
	268	position: Position,
	269	/// For floating nodes, this node's offset into its parent will always be aligned to this
	270	/// boundary. (This is not the case for header and footer nodes due to the requirement that
	271	/// they be densely packed against the start/end of the parent node.)
	272	///
	273	/// This node's size will also always be aligned to this boundary. (This applies regardless
	274	/// of whether this is a floating node, a header node, or a footer node.)
	275	alignment: Alignment,
	276	/// Whether `moved` events on this node bubble down to children.
	277	bubbles_moved: bool,
	278	/// Whether `next_moved` events are reported in `updates`.
	279	enable_next_moved: bool,
	280
129	281	location_tag: Location.Tag,
130		alignment: std.mem.Alignment,
131		/// Whether this node can be moved.
132		fixed: bool,
133	282	/// Whether this node has been moved.
134	283	moved: bool,
135	284	/// Whether this node has been resized.
136	285	resized: bool,
137	286	/// Whether the next sibling has moved or is a different node.
138	287	next_moved: bool,
139		/// Whether this node might contain non-zero bytes.
	288	/// Whether this node might contain initialized bytes.
140	289	has_content: bool,
141		/// Whether `moved` events on this node bubble down to children.
142		bubbles_moved: bool,
143		/// Whether `next_moved` events are reported in `updates`.
144		enable_next_moved: bool,
145		unused: @Int(.unsigned, 32 - @bitSizeOf(std.mem.Alignment) - 8) = 0,
	290	unused: u17 = 0,
146	291	};
147	292
148	293	pub const Location = union(enum(u1)) {
...	...	@@ -179,74 +324,183 @@ pub const Node = extern struct {
179	324	}
180	325	};
181	326
	327	pub const AddOptions = struct {
	328	/// Must be aligned to the given `alignment`.
	329	size: u64 = 0,
	330	alignment: Alignment = .@"1",
	331	bubbles_moved: bool = true,
	332	enable_next_moved: bool = false,
	333
	334	moved: bool = false,
	335	resized: bool = false,
	336	next_moved: bool = false,
	337	};
	338
182	339	pub const Index = enum(u32) {
183		none,
	340	root,
184	341	_,
185	342
186		pub const root: Node.Index = .none;
	343	pub const Optional = enum(u32) {
	344	none = std.math.maxInt(u32),
	345	_,
	346
	347	pub fn unwrap(oi: Optional) ?Index {
	348	return switch (oi) {
	349	_ => @fromBackingInt(@backingInt(oi)),
	350	.none => null,
	351	};
	352	}
	353	pub fn wrap(i: Index) Optional {
	354	const oi: Optional = @bitCast(i);
	355	assert(oi != .none);
	356	return oi;
	357	}
	358	};
187	359
188	360	fn get(ni: Node.Index, mf: const MappedFile) Node {
189	361	return &mf.nodes.items[@backingInt(ni)];
190	362	}
191	363
192		pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index {
	364	/// Adds a floating child node to `parent_ni`. Returns the index of the new child.
	365	pub fn addFloatingChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index {
	366	return mf.addNode(gpa, .{
	367	.add_options = opts,
	368	.position = .floating,
	369	.parent = parent_ni,
	370	.prev = parent_ni.lastHeader(mf),
	371	});
	372	}
	373	/// Adds a header child node to `parent_ni`. Returns the index of the new child.
	374	///
	375	/// Asserts that `parent_ni` has no existing header children.
	376	pub fn addOnlyHeaderChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index {
	377	if (parent_ni.first(mf).unwrap()) \|first_ni\| {
	378	assert(first_ni.position(mf) != .header); // `parent_ni` already has a header child
	379	}
	380	return parent_ni.addHeaderChildAfter(mf, gpa, .none, opts);
	381	}
	382	/// Adds a header child node to `parent_ni`. Returns the index of the new child.
	383	///
	384	/// If `prev_oni` is `.none`, the new child is placed at the very start of the parent,
	385	/// before any existing header nodes.
	386	///
	387	/// Otherwise, asserts that `prev_oni` is a header node and a child of `parent_ni`, and
	388	/// places the new child node immediately after `prev_oni`.
	389	pub fn addHeaderChildAfter(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, prev_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index {
	390	return mf.addNode(gpa, .{
	391	.add_options = opts,
	392	.position = .header,
	393	.parent = parent_ni,
	394	.prev = prev_oni,
	395	});
	396	}
	397	/// Adds a footer child node to `parent_ni`. Returns the index of the new child.
	398	///
	399	/// Asserts that `parent_ni` has no existing footer children.
	400	pub fn addOnlyFooterChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index {
	401	if (parent_ni.last(mf).unwrap()) \|last_ni\| {
	402	assert(last_ni.position(mf) != .footer); // `parent_ni` already has a footer child
	403	}
	404	return parent_ni.addFooterChildBefore(mf, gpa, .none, opts);
	405	}
	406	/// Adds a footer child node to `parent_ni`. Returns the index of the new child.
	407	///
	408	/// If `next_oni` is `.none`, the new child is placed at the very end of the parent, after
	409	/// any existing footer nodes.
	410	///
	411	/// Otherwise, asserts that `next_oni` is a footer node and a child of `parent_ni`, and
	412	/// places the new child node immediately before `next_oni`.
	413	pub fn addFooterChildBefore(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, next_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index {
	414	const prev_oni: Node.Index.Optional = prev: {
	415	const next_ni = next_oni.unwrap() orelse {
	416	break :prev parent_ni.last(mf);
	417	};
	418	break :prev next_ni.prev(mf);
	419	};
	420	return mf.addNode(gpa, .{
	421	.add_options = opts,
	422	.position = .footer,
	423	.parent = parent_ni,
	424	.prev = prev_oni,
	425	});
	426	}
	427
	428	/// Alias for `Optional.wrap`, provided for convenience when a result type is not available.
	429	pub const toOptional = Optional.wrap;
	430
	431	pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
193	432	return ni.get(mf).parent;
194	433	}
195	434
196		pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index {
	435	pub fn first(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
	436	return ni.get(mf).first;
	437	}
	438
	439	pub fn last(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
	440	return ni.get(mf).last;
	441	}
	442
	443	fn lastHeader(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
	444	var header_ni = ni.first(mf).unwrap() orelse return .none;
	445	if (header_ni.position(mf) != .header) return .none;
	446	while (true) {
	447	const next_ni = header_ni.next(mf).unwrap() orelse break;
	448	if (next_ni.position(mf) != .header) break;
	449	header_ni = next_ni;
	450	}
	451	return .wrap(header_ni);
	452	}
	453	fn firstFooter(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
	454	var footer_ni = ni.last(mf).unwrap() orelse return .none;
	455	if (footer_ni.position(mf) != .footer) return .none;
	456	while (true) {
	457	const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
	458	if (prev_ni.position(mf) != .footer) break;
	459	footer_ni = prev_ni;
	460	}
	461	return .wrap(footer_ni);
	462	}
	463
	464	/// Asserts that `ni` is not `.root`, because `Position` is meaningless for the root node.
	465	pub fn position(ni: Node.Index, mf: *const MappedFile) Node.Position {
	466	assert(ni != .root);
	467	return ni.get(mf).flags.position;
	468	}
	469
	470	pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
197	471	return ni.get(mf).next;
198	472	}
199	473	fn setNext(
200		prev_ni: Node.Index,
	474	ni: Node.Index,
201	475	gpa: Allocator,
202		next_ni: Node.Index,
	476	next_ni: Node.Index.Optional,
203	477	mf: *MappedFile,
204	478	) Allocator.Error!void {
205		assert(prev_ni != .none);
206		const prev_next = &prev_ni.get(mf).next;
207		if (prev_next.* == next_ni) return;
208		prev_next.* = next_ni;
209		try prev_ni.nextMoved(gpa, mf);
	479	const next_ptr = &ni.get(mf).next;
	480	if (next_ptr.* == next_ni) return;
	481	next_ptr.* = next_ni;
	482	try ni.nextMoved(gpa, mf);
210	483	}
211	484
212		pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index {
	485	pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional {
213	486	return ni.get(mf).prev;
214	487	}
215	488
216		pub fn ChildIterator(comptime direction: enum { prev, next }) type {
217		return struct {
218		mf: *const MappedFile,
219		ni: Node.Index,
220		pub fn next(it: *@This()) ?Node.Index {
221		const ni = it.ni;
222		if (ni == .none) return null;
223		it.ni = @field(ni.get(it.mf), @tagName(direction));
224		return ni;
225		}
226		};
227		}
228		pub fn children(ni: Node.Index, mf: *const MappedFile) ChildIterator(.next) {
229		return .{ .mf = mf, .ni = ni.get(mf).first };
230		}
231		pub fn reverseChildren(ni: Node.Index, mf: *const MappedFile) ChildIterator(.prev) {
232		return .{ .mf = mf, .ni = ni.get(mf).last };
233		}
234
235	489	pub fn childrenMoved(ni: Node.Index, gpa: Allocator, mf: *MappedFile) Allocator.Error!void {
236		var child_ni = ni.get(mf).last;
237		while (child_ni != .none) {
	490	var child_oni = ni.get(mf).last;
	491	while (child_oni.unwrap()) \|child_ni\| {
238	492	try child_ni.moved(gpa, mf);
239		child_ni = child_ni.get(mf).prev;
	493	child_oni = child_ni.get(mf).prev;
240	494	}
241	495	}
242	496
243	497	pub fn hasMoved(ni: Node.Index, mf: *const MappedFile) bool {
244	498	var parent_ni = ni;
245		while (parent_ni != Node.Index.root) {
	499	while (parent_ni != .root) {
246	500	const parent_node = parent_ni.get(mf);
247	501	if (!parent_node.flags.bubbles_moved) break;
248	502	if (parent_node.flags.moved) return true;
249		parent_ni = parent_node.parent;
	503	parent_ni = parent_node.parent.unwrap().?;
250	504	}
251	505	return false;
252	506	}
...	...	@@ -263,9 +517,8 @@ pub const Node = extern struct {
263	517	if (ni.hasMoved(mf)) return;
264	518	const node = ni.get(mf);
265	519	node.flags.moved = true;
266		switch (node.prev) {
267		.none => {},
268		else => \|prev_ni\| prev_ni.nextMovedAssumeCapacity(mf),
	520	if (node.prev.unwrap()) \|prev_ni\| {
	521	prev_ni.nextMovedAssumeCapacity(mf);
269	522	}
270	523	if (node.flags.resized or node.flags.next_moved) return;
271	524	mf.updates.appendAssumeCapacity(ni);
...	...	@@ -314,12 +567,18 @@ pub const Node = extern struct {
314	567	mf.update_prog_node.increaseEstimatedTotalItems(1);
315	568	}
316	569
317		pub fn alignment(ni: Node.Index, mf: *const MappedFile) std.mem.Alignment {
	570	pub fn alignment(ni: Node.Index, mf: *const MappedFile) Alignment {
318	571	return ni.get(mf).flags.alignment;
319	572	}
320	573
321		fn setLocationAssumeCapacity(ni: Node.Index, mf: *MappedFile, offset: u64, size: u64) void {
	574	fn setLocation(ni: Node.Index, mf: *MappedFile, gpa: Allocator, offset: u64, size: u64) Allocator.Error!void {
	575	try mf.large.ensureUnusedCapacity(gpa, 2);
	576	try mf.updates.ensureUnusedCapacity(gpa, 2);
322	577	const node = ni.get(mf);
	578	if (node.flags.position == .floating) {
	579	assert(node.flags.alignment.check(offset));
	580	}
	581	assert(node.flags.alignment.check(size));
323	582	if (size == 0) node.flags.has_content = false;
324	583	switch (node.location()) {
325	584	.small => \|small\| {
...	...	@@ -361,8 +620,11 @@ pub const Node = extern struct {
361	620	while (true) {
362	621	const parent_node = parent_ni.get(mf);
363	622	if (set_has_content) parent_node.flags.has_content = true;
364		if (parent_ni == .none) break;
365		parent_ni = parent_node.parent;
	623	if (parent_ni == .root) {
	624	assert(parent_node.parent == .none);
	625	break;
	626	}
	627	parent_ni = parent_node.parent.unwrap().?;
366	628	const parent_offset, _ = parent_ni.location(mf).resolve(mf);
367	629	offset += parent_offset;
368	630	}
...	...	@@ -379,62 +641,46 @@ pub const Node = extern struct {
379	641	return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)];
380	642	}
381	643
382		pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void {
383		mf.resizeNode(gpa, ni, size) catch \|err\| switch (err) {
384		error.OutOfMemory,
385		error.Canceled,
386		=> \|e\| return e,
387		else => \|e\| {
388		mf.io_err = e;
389		return error.MappedFileIo;
390		},
391		};
392		var writers_it = mf.writers.first;
393		while (writers_it) \|writer_node\| : (writers_it = writer_node.next) {
394		const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node);
395		w.interface.buffer = w.ni.slice(mf);
396		}
	644	/// Ensures that the size of `ni` is at least `min_size`. Valid for any node.
	645	///
	646	/// Applies `growth_factor` if necessary (so the caller should not apply `growth_factor`).
	647	pub fn ensureMinimumSize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, min_size: u64) Error!void {
	648	_, const current_size = ni.location(mf).resolve(mf);
	649	if (current_size >= min_size) return;
	650	const new_size = ni.alignment(mf).forward(min_size +\| min_size / growth_factor);
	651	try mf.growNode(gpa, ni, new_size, .minimum);
	652	mf.updateWriters();
397	653	}
398	654
399		pub const RealignNodeOptions = struct {
400		/// Shift the node backwards if possible
401		try_backwards: bool = false,
402		};
403
404		/// Moves and expands a node such that its offset and size are aligned to `new_alignment`.
405		/// Asserts that `ni` is not `Node.Index.root`.
406		pub fn realign(
407		ni: Node.Index,
408		mf: *MappedFile,
409		gpa: Allocator,
410		new_alignment: std.mem.Alignment,
411		opts: RealignNodeOptions,
412		) Error!void {
413		mf.realignNode(gpa, ni, new_alignment, opts) catch \|err\| switch (err) {
414		error.OutOfMemory,
415		error.Canceled,
416		=> \|e\| return e,
417		else => \|e\| {
418		mf.io_err = e;
419		return error.MappedFileIo;
420		},
421		};
	655	/// Sets the size of `ni` to exactly `size`.
	656	///
	657	/// Asserts that `ni` is a leaf node, i.e. has no children.
	658	///
	659	/// Asserts that `size` is aligned to `ni.alignment(mf)`.
	660	pub fn resizeLeaf(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void {
	661	assert(ni.first(mf) == .none);
	662	// The alignment of `size` is asserted by `shrinkLeafNode` and `growNode`.
	663	_, const old_size = ni.location(mf).resolve(mf);
	664	switch (std.math.order(size, old_size)) {
	665	.lt => try mf.shrinkLeafNode(gpa, ni, size),
	666	.eq => {}, // `old_size` must be well-aligned, so `size` is too
	667	.gt => try mf.growNode(gpa, ni, size, .exact),
	668	}
422	669	mf.updateWriters();
423	670	}
424	671
425		/// Shrink a node to `size`, exactly.
426		/// Asserts that the new size can contain all the children.
427		/// If `shift_next` is set, then the following node is shifted backwards into
428		/// the free space as much as alignment allows.
429		/// Asserts that `size` is >= the end of the last child node.
430		pub fn shrink(
	672	/// Updates a node's alignment to exactly `new_alignment`. Valid for any node.
	673	///
	674	/// If the node's current offset or size is not sufficiently aligned, it will be moved
	675	/// and/or resized to match the new alignment. The node's size may be increased by any
	676	/// amount, as if `ensureMinimumSize` were used.
	677	pub fn realign(
431	678	ni: Node.Index,
432	679	mf: *MappedFile,
433	680	gpa: Allocator,
434		size: u64,
435		shift_next: bool,
	681	new_alignment: Alignment,
436	682	) Error!void {
437		try mf.shrinkNode(gpa, ni, size, shift_next);
	683	try mf.realignNode(gpa, ni, new_alignment);
438	684	mf.updateWriters();
439	685	}
440	686
...	...	@@ -538,16 +784,9 @@ pub const Node = extern struct {
538	784	file_reader.pos,
539	785	w.ni.fileLocation(w.mf, true).offset + interface.end,
540	786	limit.minInt(interface.unusedCapacityLen()),
541		) catch \|err\| switch (err) {
542		error.Canceled => \|e\| {
543		w.err = e;
544		return error.WriteFailed;
545		},
546		else => \|e\| {
547		w.mf.io_err = e;
548		w.err = error.MappedFileIo;
549		return error.WriteFailed;
550		},
	787	) catch \|err\| {
	788	w.err = err;
	789	return error.WriteFailed;
551	790	});
552	791	if (n == 0) return error.Unimplemented;
553	792	file_reader.pos += n;
...	...	@@ -574,10 +813,8 @@ pub const Node = extern struct {
574	813	unused_capacity: usize,
575	814	) Io.Writer.Error!void {
576	815	_ = preserve;
577		const total_capacity = interface.end + unused_capacity;
578		if (interface.buffer.len >= total_capacity) return;
579	816	const w: *Writer = @fieldParentPtr("interface", interface);
580		w.ni.resize(w.mf, w.gpa, total_capacity +\| total_capacity / growth_factor) catch \|err\| {
	817	w.ni.ensureMinimumSize(w.mf, w.gpa, interface.end + unused_capacity) catch \|err\| {
581	818	w.err = err;
582	819	return error.WriteFailed;
583	820	};
...	...	@@ -585,617 +822,1265 @@ pub const Node = extern struct {
585	822	};
586	823
587	824	comptime {
588		if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 32);
	825	if (!std.debug.runtime_safety) assert(@sizeOf(Node) == 32);
589	826	}
590	827	};
591	828
	829	/// Asserts that `opts.position` is compatible with `opts.prev` (i.e. that this addition will not
	830	/// violate the requirement that header nodes come before floating nodes come before footer nodes).
592	831	fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct {
593		parent: Node.Index = .none,
594		prev: Node.Index = .none,
595		next: Node.Index = .none,
596		offset: u64 = 0,
597		add_node: AddNodeOptions,
598		}) (Allocator.Error \|\| Io.Cancelable \|\| IoError)!Node.Index {
	832	add_options: Node.AddOptions,
	833	position: Node.Position,
	834	parent: Node.Index,
	835	/// If `position == .floating`, this is just used as an initial value, and may be immediately
	836	/// replaced when finding a location for this node. In this case, it is still necessary that
	837	/// `prev` be compatible with `position` (so `prev` must be either a floating node or the last
	838	/// header node in `parent`).
	839	prev: Node.Index.Optional,
	840	}) Error!Node.Index {
599	841	mf.nodes_lock.assertUnlocked();
600		const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: {
601		if (std.math.cast(u32, opts.offset)) \|small_offset\| break :location .{ .small, .{
602		.small = .{ .offset = small_offset, .size = 0 },
603		} };
604		try mf.large.ensureUnusedCapacity(gpa, 2);
605		defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 });
606		break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } };
	842
	843	try mf.nodes.ensureUnusedCapacity(gpa, 1);
	844	try mf.large.ensureUnusedCapacity(gpa, 2);
	845
	846	const new_ni: Node.Index = new: {
	847	if (mf.free_ni.unwrap()) \|free_ni\| {
	848	mf.free_ni = free_ni.get(mf).next;
	849	break :new free_ni;
	850	}
	851	const new_ni: Node.Index = @fromBackingInt(@intCast(mf.nodes.items.len));
	852	_ = mf.nodes.addOneAssumeCapacity();
	853	break :new new_ni;
607	854	};
608		const free_ni: Node.Index, const free_node = free: switch (mf.free_ni) {
609		.none => .{ @fromBackingInt(@intCast(mf.nodes.items.len)), mf.nodes.addOneAssumeCapacity() },
610		else => \|free_ni\| {
611		const free_node = free_ni.get(mf);
612		mf.free_ni = free_node.next;
613		break :free .{ free_ni, free_node };
	855
	856	const next_oni: Node.Index.Optional = if (opts.prev.unwrap()) \|prev_ni\| next: {
	857	assert(prev_ni.parent(mf) == opts.parent.toOptional()); // `prev` is not a child of `parent`
	858	break :next prev_ni.get(mf).next;
	859	} else opts.parent.first(mf);
	860
	861	// Validate node ordering
	862	switch (opts.position) {
	863	.floating => {
	864	if (opts.prev.unwrap()) \|prev_ni\| {
	865	assert(prev_ni.position(mf) != .footer); // tried to add floating node after footer node
	866	}
	867	if (next_oni.unwrap()) \|next_ni\| {
	868	assert(next_ni.position(mf) != .header); // tried to add floating node before header node
	869	}
	870	},
	871	.header => if (opts.prev.unwrap()) \|prev_ni\| {
	872	switch (prev_ni.position(mf)) {
	873	.header => {},
	874	.floating => unreachable, // tried to add header node after floating node
	875	.footer => unreachable, // tried to add header node after footer node
	876	}
	877	},
	878	.footer => if (next_oni.unwrap()) \|next_ni\| {
	879	switch (next_ni.position(mf)) {
	880	.header => unreachable, // tried to add footer node before header node
	881	.floating => unreachable, // tried to add footer node before floating node
	882	.footer => {},
	883	}
614	884	},
615		};
616		switch (opts.prev) {
617		.none => opts.parent.get(mf).first = free_ni,
618		else => \|prev_ni\| try prev_ni.setNext(gpa, free_ni, mf),
619		}
620		switch (opts.next) {
621		.none => opts.parent.get(mf).last = free_ni,
622		else => \|next_ni\| next_ni.get(mf).prev = free_ni,
623	885	}
624		free_node.* = .{
625		.parent = opts.parent,
626		.prev = opts.prev,
627		.next = opts.next,
	886
	887	// Initialize the node as empty with alignment 1
	888	const location: Node.Location = loc: {
	889	const offset: u64 = switch (opts.position) {
	890	.header, .floating => offset: {
	891	const prev_ni = opts.prev.unwrap() orelse break :offset 0;
	892	const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf);
	893	break :offset prev_offset + prev_size;
	894	},
	895	.footer => offset: {
	896	const next_ni = next_oni.unwrap() orelse {
	897	_, const parent_size = opts.parent.location(mf).resolve(mf);
	898	break :offset parent_size;
	899	};
	900	const next_offset, _ = next_ni.location(mf).resolve(mf);
	901	break :offset next_offset;
	902	},
	903	};
	904	if (std.math.cast(u32, offset)) \|small_offset\| {
	905	break :loc .{ .small = .{ .offset = small_offset, .size = 0 } };
	906	}
	907	const large_index = mf.large.items.len;
	908	mf.large.appendSliceAssumeCapacity(&.{ offset, 0 });
	909	break :loc .{ .large = .{ .index = large_index } };
	910	};
	911	new_ni.get(mf).* = .{
	912	.parent = .wrap(opts.parent),
	913	.prev = .none,
	914	.next = .none,
628	915	.first = .none,
629	916	.last = .none,
630	917	.flags = .{
631		.location_tag = location_tag,
	918	.position = opts.position,
632	919	.alignment = .@"1",
633		.fixed = opts.add_node.fixed,
634		.moved = true,
635		.resized = true,
636		.next_moved = true,
	920	.bubbles_moved = opts.add_options.bubbles_moved,
	921	.enable_next_moved = opts.add_options.enable_next_moved,
	922	.location_tag = location,
	923	.moved = false,
	924	.resized = false,
	925	.next_moved = false,
637	926	.has_content = false,
638		.bubbles_moved = opts.add_node.bubbles_moved,
639		.enable_next_moved = opts.add_node.enable_next_moved,
640	927	},
641		.location_payload = location_payload,
	928	.location_payload = switch (location) {
	929	.small => \|small\| .{ .small = small },
	930	.large => \|large\| .{ .large = large },
	931	},
642	932	};
643	933
644		{
645		defer {
646		free_node.flags.moved = false;
647		free_node.flags.resized = false;
648		free_node.flags.next_moved = false;
649		}
650		try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{});
651		try mf.resizeNode(gpa, free_ni, opts.add_node.size);
	934	try mf.addNodesToChildListBefore(gpa, next_oni, new_ni, new_ni);
	935
	936	try mf.realignNode(gpa, new_ni, opts.add_options.alignment);
	937	if (opts.add_options.size > 0) {
	938	try mf.growNode(gpa, new_ni, opts.add_options.size, .exact);
652	939	}
653	940	mf.updateWriters();
654		if (opts.add_node.moved) try free_ni.moved(gpa, mf);
655		if (opts.add_node.resized) try free_ni.resized(gpa, mf);
656		if (opts.add_node.next_moved) try free_ni.nextMoved(gpa, mf);
657		return free_ni;
658		}
659	941
660		pub const AddNodeOptions = struct {
661		size: u64 = 0,
662		alignment: std.mem.Alignment = .@"1",
663		fixed: bool = false,
664		moved: bool = false,
665		resized: bool = false,
666		next_moved: bool = false,
667		bubbles_moved: bool = true,
668		enable_next_moved: bool = false,
669		};
	942	new_ni.get(mf).flags.moved = false;
	943	new_ni.get(mf).flags.resized = false;
	944	new_ni.get(mf).flags.next_moved = false;
670	945
671		pub fn addOnlyChildNode(
672		mf: *MappedFile,
673		gpa: Allocator,
674		parent_ni: Node.Index,
675		opts: AddNodeOptions,
676		) Error!Node.Index {
677		try mf.nodes.ensureUnusedCapacity(gpa, 1);
678		const parent = parent_ni.get(mf);
679		assert(parent.first == .none and parent.last == .none);
680		return mf.addNode(gpa, .{
681		.parent = parent_ni,
682		.add_node = opts,
683		}) catch \|err\| switch (err) {
684		error.OutOfMemory,
685		error.Canceled,
686		=> \|e\| return e,
687		else => \|e\| {
688		mf.io_err = e;
689		return error.MappedFileIo;
690		},
691		};
692		}
	946	if (opts.add_options.moved) try new_ni.moved(gpa, mf);
	947	if (opts.add_options.resized) try new_ni.resized(gpa, mf);
	948	if (opts.add_options.next_moved) try new_ni.nextMoved(gpa, mf);
693	949
694		pub fn addFirstChildNode(
695		mf: *MappedFile,
696		gpa: Allocator,
697		parent_ni: Node.Index,
698		opts: AddNodeOptions,
699		) Error!Node.Index {
700		try mf.nodes.ensureUnusedCapacity(gpa, 1);
701		const parent = parent_ni.get(mf);
702		return mf.addNode(gpa, .{
703		.parent = parent_ni,
704		.next = parent.first,
705		.add_node = opts,
706		}) catch \|err\| switch (err) {
707		error.OutOfMemory,
708		error.Canceled,
709		=> \|e\| return e,
710		else => \|e\| {
711		mf.io_err = e;
712		return error.MappedFileIo;
713		},
714		};
	950	return new_ni;
715	951	}
716	952
717		pub fn addLastChildNode(
	953	fn shrinkLeafNode(
718	954	mf: *MappedFile,
719	955	gpa: Allocator,
720		parent_ni: Node.Index,
721		opts: AddNodeOptions,
722		) Error!Node.Index {
723		try mf.nodes.ensureUnusedCapacity(gpa, 1);
724		const parent = parent_ni.get(mf);
725		return mf.addNode(gpa, .{
726		.parent = parent_ni,
727		.prev = parent.last,
728		.offset = offset: switch (parent.last) {
729		.none => 0,
730		else => \|last_ni\| {
731		const last_offset, const last_size = last_ni.location(mf).resolve(mf);
732		break :offset last_offset + last_size;
733		},
734		},
735		.add_node = opts,
736		}) catch \|err\| switch (err) {
737		error.OutOfMemory,
738		error.Canceled,
739		=> \|e\| return e,
740		else => \|e\| {
741		mf.io_err = e;
742		return error.MappedFileIo;
743		},
744		};
745		}
	956	ni: Node.Index,
	957	new_size: u64,
	958	) Error!void {
	959	mf.nodes_lock.assertUnlocked();
746	960
747		pub fn addNodeAfter(
748		mf: *MappedFile,
749		gpa: Allocator,
750		prev_ni: Node.Index,
751		opts: AddNodeOptions,
752		) Error!Node.Index {
753		assert(prev_ni != .none);
754		try mf.nodes.ensureUnusedCapacity(gpa, 1);
755		const prev = prev_ni.get(mf);
756		const prev_offset, const prev_size = prev.location().resolve(mf);
757		return mf.addNode(gpa, .{
758		.parent = prev.parent,
759		.prev = prev_ni,
760		.next = prev.next,
761		.offset = prev_offset + prev_size,
762		.add_node = opts,
763		}) catch \|err\| switch (err) {
764		error.OutOfMemory,
765		error.Canceled,
766		=> \|e\| return e,
767		else => \|e\| {
768		mf.io_err = e;
	961	const old_offset, const old_size = ni.location(mf).resolve(mf);
	962
	963	assert(new_size < old_size);
	964	assert(ni.alignment(mf).check(new_size));
	965	assert(ni.first(mf) == .none); // `ni` must be a leaf node
	966
	967	const parent_ni = ni.parent(mf).unwrap() orelse {
	968	assert(ni == .root);
	969	mf.memory_map.write(mf.io) catch \|err\| {
	970	mf.io_err = switch (err) {
	971	error.Canceled => \|e\| return e,
	972	error.WouldBlock => error.Unexpected, // file was not opened as non-blocking
	973	error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing
	974	else => \|e\| e,
	975	};
769	976	return error.MappedFileIo;
770		},
	977	};
	978	mf.memory_map.file.setLength(mf.io, new_size) catch \|err\| switch (err) {
	979	error.Canceled => \|e\| return e,
	980	else => \|e\| {
	981	mf.io_err = e;
	982	return error.MappedFileIo;
	983	},
	984	};
	985	try mf.ensureTotalCapacityPrecise(@intCast(new_size));
	986	try ni.setLocation(mf, gpa, old_offset, new_size);
	987	return;
771	988	};
772		}
773	989
774		fn shrinkNode(
775		mf: *MappedFile,
776		gpa: Allocator,
777		ni: Node.Index,
778		size: u64,
779		shift_next: bool,
780		) !void {
781		mf.nodes_lock.assertUnlocked();
782		const node = ni.get(mf);
783		const old_offset, _ = node.location().resolve(mf);
	990	switch (ni.position(mf)) {
	991	.header => {
	992	const shift = old_size - new_size;
784	993
785		// This would require unmapping first
786		assert(ni != Node.Index.root);
	994	try ni.setLocation(mf, gpa, old_offset, new_size);
787	995
788		if (node.last != .none) {
789		const last = node.last.get(mf);
790		const last_offset, const last_size = last.location().resolve(mf);
791		assert(last_offset + last_size > size);
792		}
	996	// We need to shift backwards all header nodes following us.
	997	const next_header_ni = ni.next(mf).unwrap() orelse return;
	998	if (next_header_ni.position(mf) != .header) return;
793	999
794		try mf.large.ensureUnusedCapacity(gpa, 4);
795		try mf.updates.ensureUnusedCapacity(gpa, 4);
	1000	var header_ni = next_header_ni;
	1001	while (true) {
	1002	const old_header_off, const old_header_size = header_ni.location(mf).resolve(mf);
	1003	try header_ni.setLocation(mf, gpa, old_header_off - shift, old_header_size);
796	1004
797		ni.setLocationAssumeCapacity(mf, old_offset, size);
798		if (!shift_next or node.next == .none) return;
	1005	const next_ni = header_ni.next(mf).unwrap() orelse break;
	1006	if (next_ni.position(mf) != .header) break;
	1007	header_ni = next_ni;
	1008	}
799	1009
800		const next = node.next.get(mf);
801		const old_next_offset, const next_size = next.location().resolve(mf);
802		const padding = old_next_offset - (old_offset + size);
803		const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding));
	1010	// Now we must shift the actual header bytes of those nodes backwards.
	1011	const parent_file_off = parent_ni.fileLocation(mf, false).offset;
	1012	const move_src_off = old_offset + old_size;
	1013	const move_dest_off = old_offset + new_size;
	1014	assert(next_header_ni.location(mf).resolve(mf)[0] == move_dest_off); // `move_dest_off` because we already updated the location
	1015	const move_size = size: {
	1016	// `header_ni` is the last header in the parent.
	1017	const last_off, const last_size = header_ni.location(mf).resolve(mf);
	1018	const move_end = last_off + last_size;
	1019	break :size move_end - move_dest_off; // `move_dest_off` because we already updated the location
	1020	};
	1021	try mf.moveRange(
	1022	parent_file_off + move_src_off,
	1023	parent_file_off + move_dest_off,
	1024	move_size,
	1025	);
	1026	},
	1027	.floating => {
	1028	try ni.setLocation(mf, gpa, old_offset, new_size);
	1029	},
	1030	.footer => {
	1031	const shift = old_size - new_size;
	1032
	1033	const new_offset = old_offset + shift;
	1034	try ni.setLocation(mf, gpa, new_offset, new_size);
	1035
	1036	const prev_footers_size = prev_footers_size: {
	1037	// We need to shift forwards all footer nodes preceding us.
	1038	const prev_footer_ni = ni.prev(mf).unwrap() orelse {
	1039	break :prev_footers_size 0;
	1040	};
	1041	if (prev_footer_ni.position(mf) != .footer) {
	1042	break :prev_footers_size 0;
	1043	}
804	1044
805		if (next.flags.has_content and new_next_offset < old_next_offset) {
806		const old_file_offset = node.next.fileLocation(mf, false).offset;
807		const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset;
808		@memmove(
809		mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)],
810		mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(next_size)],
811		);
812		@memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0);
813		}
	1045	var footer_ni = prev_footer_ni;
	1046	while (true) {
	1047	const old_footer_off, const old_footer_size = footer_ni.location(mf).resolve(mf);
	1048	try footer_ni.setLocation(mf, gpa, old_footer_off + shift, old_footer_size);
814	1049
815		node.next.setLocationAssumeCapacity(mf, new_next_offset, next_size);
	1050	const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
	1051	if (prev_ni.position(mf) != .footer) break;
	1052	footer_ni = prev_ni;
	1053	}
	1054
	1055	// `footer_ni` is the first footer in the parent. This expression gets its new
	1056	// offset because we already did the `setLocation` calls.
	1057	const first_footer_new_offset = footer_ni.location(mf).resolve(mf)[0];
	1058
	1059	break :prev_footers_size new_offset - first_footer_new_offset;
	1060	};
	1061
	1062	// Now we must shift the actual footer bytes forwards, including our own.
	1063	const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
	1064	try mf.moveRange(
	1065	parent_file_offset + old_offset - prev_footers_size,
	1066	parent_file_offset + new_offset - prev_footers_size,
	1067	prev_footers_size + new_size,
	1068	);
	1069	},
	1070	}
816	1071	}
817	1072
818		fn resizeNode(
	1073	const GrowMode = enum { exact, minimum };
	1074
	1075	/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`.
	1076	/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`.
	1077	///
	1078	/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!).
	1079	///
	1080	/// Asserts that `new_size` is greater than the current size of `ni`.
	1081	fn growNode(
819	1082	mf: *MappedFile,
820	1083	gpa: Allocator,
821	1084	ni: Node.Index,
822		requested_size: u64,
823		) (Allocator.Error \|\| Io.Cancelable \|\| IoError)!void {
	1085	new_size: u64,
	1086	grow_mode: GrowMode,
	1087	) Error!void {
824	1088	mf.nodes_lock.assertUnlocked();
825		const io = mf.io;
	1089
826	1090	const node = ni.get(mf);
	1091
827	1092	const old_offset, const old_size = node.location().resolve(mf);
828		const new_size = node.flags.alignment.forward(@intCast(requested_size));
829
830		// Resize the entire file
831		if (ni == Node.Index.root) {
832		try mf.ensureCapacityForSetLocation(gpa);
833		mf.memory_map.write(io) catch \|err\| switch (err) {
834		error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking
835		error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing
836		else => \|e\| return e,
837		};
838		try mf.memory_map.file.setLength(io, new_size);
839		try mf.ensureTotalCapacityInner(@intCast(new_size));
840		ni.setLocationAssumeCapacity(mf, old_offset, new_size);
841		return;
842		}
843		const parent = node.parent.get(mf);
844		_, var old_parent_size = parent.location().resolve(mf);
845		const trailing_end = trailing_end: switch (node.next) {
846		.none => old_parent_size,
847		else => \|next_ni\| {
848		const next_offset, _ = next_ni.location(mf).resolve(mf);
849		break :trailing_end next_offset;
850		},
851		};
852		assert(old_offset + old_size <= trailing_end);
853		if (old_offset + new_size <= trailing_end) {
854		// Expand the node into trailing free space
855		try mf.ensureCapacityForSetLocation(gpa);
856		ni.setLocationAssumeCapacity(mf, old_offset, new_size);
857		return;
858		}
859		insert_range: {
860		if (!is_linux) break :insert_range;
861		if (mf.flags.fallocate_insert_range_unsupported) break :insert_range;
862
863		// We need the node to be aligned to `mf.flags.block_size` in the file in order to use this
864		// fast path. It is not sufficient to check `node.flags.alignment`, because that doesn't
865		// necessarily mean that all parent nodes are equally aligned; instead we must compute the
866		// actual file offset.
867		const range_file_offset = ni.fileLocation(mf, false).offset + old_size;
868		const range_size = node.flags.alignment.forward(
869		@intCast(requested_size +\| requested_size / growth_factor),
870		) - old_size;
871		if (!mf.flags.block_size.check(@intCast(range_file_offset))) break :insert_range;
872		if (!mf.flags.block_size.check(@intCast(range_size))) break :insert_range;
873
874		mf.memory_map.write(io) catch \|err\| switch (err) {
875		error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking
876		error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing
877		else => \|e\| return e,
	1093
	1094	assert(node.flags.alignment.check(old_size));
	1095	assert(node.flags.alignment.check(new_size));
	1096	assert(new_size > old_size);
	1097
	1098	const parent_ni = node.parent.unwrap() orelse {
	1099	assert(ni == .root);
	1100
	1101	if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {
	1102	return;
	1103	}
	1104
	1105	mf.memory_map.write(mf.io) catch \|err\| {
	1106	mf.io_err = switch (err) {
	1107	error.Canceled => \|e\| return e,
	1108	error.WouldBlock => error.Unexpected, // file was not opened as non-blocking
	1109	error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing
	1110	else => \|e\| e,
	1111	};
	1112	return error.MappedFileIo;
878	1113	};
879		// Ask the filesystem driver to insert extents into the file without copying any data
880		const last_offset, const last_size = parent.last.location(mf).resolve(mf);
881		const last_end = last_offset + last_size;
882		assert(last_end <= old_parent_size);
883		_, const file_size = Node.Index.root.location(mf).resolve(mf);
884		while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) {
885		.lt => linux.fallocate(
886		mf.memory_map.file.handle,
887		linux.FALLOC.FL_INSERT_RANGE,
888		@intCast(range_file_offset),
889		@intCast(range_size),
890		),
891		.eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)),
892		.gt => unreachable,
893		})) {
894		.SUCCESS => {
895		var enclosing_ni = ni;
896		while (true) {
897		try mf.ensureCapacityForSetLocation(gpa);
898		const enclosing = enclosing_ni.get(mf);
899		const enclosing_offset, const old_enclosing_size =
900		enclosing.location().resolve(mf);
901		const new_enclosing_size = old_enclosing_size + range_size;
902		enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size);
903		if (enclosing_ni == Node.Index.root) {
904		assert(enclosing_offset == 0);
905		try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size));
906		break;
907		}
908		var after_ni = enclosing.next;
909		while (after_ni != .none) {
910		try mf.ensureCapacityForSetLocation(gpa);
911		const after = after_ni.get(mf);
912		const after_offset, const after_size = after.location().resolve(mf);
913		after_ni.setLocationAssumeCapacity(
914		mf,
915		range_size + after_offset,
916		after_size,
917		);
918		after_ni = after.next;
919		}
920		enclosing_ni = enclosing.parent;
921		}
922		return;
923		},
924		.INTR => continue,
925		.BADF, .FBIG, .INVAL => unreachable,
926		.IO => return error.InputOutput,
927		.NODEV => return error.NotFile,
928		.NOSPC => return error.NoSpaceLeft,
929		.NOSYS, .OPNOTSUPP => {
930		mf.flags.fallocate_insert_range_unsupported = true;
931		break :insert_range;
	1114	mf.memory_map.file.setLength(mf.io, new_size) catch \|err\| switch (err) {
	1115	error.Canceled => \|e\| return e,
	1116	else => \|e\| {
	1117	mf.io_err = e;
	1118	return error.MappedFileIo;
932	1119	},
933		.PERM => return error.PermissionDenied,
934		.SPIPE => return error.Unseekable,
935		.TXTBSY => return error.FileBusy,
936		else => \|e\| return std.posix.unexpectedErrno(e),
937	1120	};
938		}
939		if (node.next == .none) {
940		// As this is the last node, we simply need more space in the parent
941		const new_parent_size = old_offset + new_size;
942		try mf.resizeNode(gpa, node.parent, new_parent_size +\| new_parent_size / growth_factor);
943		try mf.ensureCapacityForSetLocation(gpa);
944		ni.setLocationAssumeCapacity(mf, old_offset, new_size);
945		return;
946		}
947		if (!node.flags.fixed) {
948		// Make space at the end of the parent for this floating node
949		const last = parent.last.get(mf);
950		const last_offset, const last_size = last.location().resolve(mf);
951		const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size));
952		const new_parent_size = new_offset + new_size;
953		if (new_parent_size > old_parent_size)
954		try mf.resizeNode(gpa, node.parent, new_parent_size +\| new_parent_size / growth_factor);
955		try mf.ensureCapacityForSetLocation(gpa);
956		const next_ni = node.next;
957		next_ni.get(mf).prev = node.prev;
958		switch (node.prev) {
959		.none => parent.first = next_ni,
960		else => \|prev_ni\| try prev_ni.setNext(gpa, next_ni, mf),
961		}
962		try parent.last.setNext(gpa, ni, mf);
963		node.prev = parent.last;
964		try ni.setNext(gpa, .none, mf);
965		parent.last = ni;
966		if (node.flags.has_content) {
967		const parent_file_offset = node.parent.fileLocation(mf, false).offset;
	1121	try mf.ensureTotalCapacityPrecise(@intCast(new_size));
	1122	try ni.setLocation(mf, gpa, old_offset, new_size);
	1123	// We need to move any footers to be at the new end of the file.
	1124	if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
	1125	const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1126	const footers_size = old_size - old_footers_offset;
968	1127	try mf.moveRange(
969		parent_file_offset + old_offset,
970		parent_file_offset + new_offset,
971		old_size,
	1128	old_footers_offset,
	1129	old_footers_offset + (new_size - old_size),
	1130	footers_size,
972	1131	);
	1132	// Also update the footers' locations.
	1133	var cur_ni = first_footer_ni;
	1134	while (true) {
	1135	const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
	1136	try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
	1137	cur_ni = cur_ni.next(mf).unwrap() orelse break;
	1138	}
973	1139	}
974		ni.setLocationAssumeCapacity(mf, new_offset, new_size);
975	1140	return;
976		}
977		// Search for the first floating node following this fixed node
978		var last_fixed_ni = ni;
979		var first_floating_ni = node.next;
980		var shift = new_size - old_size;
981		var max_shift_align: std.mem.Alignment = .@"1";
982		var direction: enum { forward, reverse } = .forward;
983		while (true) {
984		assert(last_fixed_ni != .none);
985		const last_fixed = last_fixed_ni.get(mf);
986		assert(last_fixed.flags.fixed);
987		const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf);
988		const new_last_fixed_offset = old_last_fixed_offset + shift;
989		make_space: switch (first_floating_ni) {
990		else => {
991		const first_floating = first_floating_ni.get(mf);
992		const old_first_floating_offset, const first_floating_size =
993		first_floating.location().resolve(mf);
994		assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset);
995		if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset)
996		break :make_space;
997		assert(direction == .forward);
998		max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment));
999		if (first_floating.flags.fixed) {
1000		shift = max_shift_align.forward(@intCast(
1001		@max(shift, first_floating_size),
1002		));
1003
1004		// Not enough space, try the next node
1005		last_fixed_ni = first_floating_ni;
1006		first_floating_ni = first_floating.next;
1007		continue;
	1141	};
	1142
	1143	switch (node.flags.position) {
	1144	.header => {
	1145	if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {
	1146	return;
	1147	}
	1148
	1149	try mf.ensureAdditionalHeaderCapacity(gpa, parent_ni, new_size - old_size);
	1150
	1151	// `old_offset` is still valid because header nodes don't move when the parent resizes.
	1152
	1153	const last_header_ni: Node.Index = last_header: {
	1154	var header_ni = ni;
	1155	while (true) {
	1156	const next_ni = header_ni.next(mf).unwrap() orelse break;
	1157	if (next_ni.position(mf) != .header) break;
	1158	header_ni = next_ni;
1008	1159	}
1009		// Move the found floating node to make space for preceding fixed nodes
1010		const last = parent.last.get(mf);
1011		const last_offset, const last_size = last.location().resolve(mf);
1012		const new_first_floating_offset = max_shift_align.forward(
1013		@intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)),
1014		);
1015		const new_parent_size = new_first_floating_offset + first_floating_size;
1016		if (new_parent_size > old_parent_size) {
1017		try mf.resizeNode(
	1160	break :last_header header_ni;
	1161	};
	1162	const last_header_offset, const last_header_size = last_header_ni.location(mf).resolve(mf);
	1163	const old_headers_size = last_header_offset + last_header_size;
	1164
	1165	// This is the first footer inside of `ni`.
	1166	const first_sub_footer_oni = ni.firstFooter(mf);
	1167	const sub_footers_size = size: {
	1168	const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
	1169	const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
	1170	break :size old_size - first_sub_footer_offset;
	1171	};
	1172
	1173	// We need to shift two things forwards; any header nodes which follow us, and any
	1174	// footer nodes within us (since they need to be at the end of our new size).
	1175	const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
	1176	try mf.moveRange(
	1177	parent_file_offset + old_offset + old_size - sub_footers_size,
	1178	parent_file_offset + old_offset + new_size - sub_footers_size,
	1179	old_headers_size - (old_offset + old_size - sub_footers_size),
	1180	);
	1181
	1182	// Any footers inside of us have had their offsets changed due to us growing:
	1183	if (first_sub_footer_oni.unwrap()) \|first_sub_footer_ni\| {
	1184	var cur_ni = first_sub_footer_ni;
	1185	while (true) {
	1186	const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf);
	1187	try cur_ni.setLocation(
	1188	mf,
1018	1189	gpa,
1019		node.parent,
1020		new_parent_size +\| new_parent_size / growth_factor,
	1190	old_sub_footer_offset + (new_size - old_size),
	1191	sub_footer_size,
1021	1192	);
1022		_, old_parent_size = parent.location().resolve(mf);
	1193	cur_ni = cur_ni.next(mf).unwrap() orelse break;
1023	1194	}
1024		try mf.ensureCapacityForSetLocation(gpa);
1025		if (parent.last != first_floating_ni) {
1026		const old_last = parent.last;
1027		first_floating.prev = old_last;
1028		parent.last = first_floating_ni;
1029		try old_last.setNext(gpa, first_floating_ni, mf);
1030		try last_fixed_ni.setNext(gpa, first_floating.next, mf);
1031		switch (first_floating.next) {
1032		.none => {},
1033		else => \|next_ni\| next_ni.get(mf).prev = last_fixed_ni,
1034		}
1035		try first_floating_ni.setNext(gpa, .none, mf);
	1195	}
	1196
	1197	// Update the offsets of all header nodes following us:
	1198	{
	1199	var moved_header_ni = last_header_ni;
	1200	while (moved_header_ni != ni) {
	1201	assert(moved_header_ni.position(mf) == .header);
	1202	const moved_header_offset, const moved_header_size = moved_header_ni.location(mf).resolve(mf);
	1203	try moved_header_ni.setLocation(
	1204	mf,
	1205	gpa,
	1206	moved_header_offset - old_size + new_size,
	1207	moved_header_size,
	1208	);
	1209	moved_header_ni = moved_header_ni.prev(mf).unwrap().?;
1036	1210	}
1037		if (first_floating.flags.has_content) {
1038		const parent_file_offset =
1039		node.parent.fileLocation(mf, false).offset;
1040		try mf.moveRange(
1041		parent_file_offset + old_first_floating_offset,
1042		parent_file_offset + new_first_floating_offset,
1043		first_floating_size,
	1211	}
	1212
	1213	// Finally, update our own size:
	1214	try ni.setLocation(mf, gpa, old_offset, new_size);
	1215	return;
	1216	},
	1217	.floating => {
	1218	try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode);
	1219	},
	1220	.footer => {
	1221	if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {
	1222	return;
	1223	}
	1224
	1225	try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size);
	1226
	1227	const first_footer_ni: Node.Index = first_footer: {
	1228	var footer_ni = ni;
	1229	while (true) {
	1230	const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
	1231	if (prev_ni.position(mf) != .footer) break;
	1232	footer_ni = prev_ni;
	1233	}
	1234	break :first_footer footer_ni;
	1235	};
	1236
	1237	// This is the first footer inside of `ni` (unrelated to the fact that `ni` is itself
	1238	// a footer within its parent).
	1239	const first_sub_footer_oni = ni.firstFooter(mf);
	1240	const sub_footers_size = size: {
	1241	const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
	1242	const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
	1243	break :size old_size - first_sub_footer_offset;
	1244	};
	1245
	1246	_, const parent_size = parent_ni.location(mf).resolve(mf);
	1247
	1248	const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0];
	1249	const new_footers_size = old_footers_size - old_size + new_size;
	1250
	1251	// Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node
	1252	// itself needs to shift its contents, because our offset was shifted backwards by
	1253	// `new_size - old_size`, and the added bytes should go at the end of this footer node.
	1254	// However, if we contain any footer nodes, they need to stay at the end of `ni`, so
	1255	// we shouldn't shift that data.
	1256	const old_footers_start = parent_size - old_footers_size;
	1257	const new_footers_start = parent_size - new_footers_size;
	1258	const end_offset = node.location().resolve(mf)[0] + old_size;
	1259	const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
	1260	try mf.moveRange(
	1261	parent_file_offset + old_footers_start,
	1262	parent_file_offset + new_footers_start,
	1263	end_offset - old_footers_start - sub_footers_size,
	1264	);
	1265
	1266	// Update our own offset and size:
	1267	try ni.setLocation(mf, gpa, end_offset - new_size, new_size);
	1268
	1269	// Any footers inside of us have had their offsets changed due to us growing:
	1270	if (first_sub_footer_oni.unwrap()) \|first_sub_footer_ni\| {
	1271	var cur_ni = first_sub_footer_ni;
	1272	while (true) {
	1273	const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf);
	1274	try cur_ni.setLocation(
	1275	mf,
	1276	gpa,
	1277	old_sub_footer_offset + (new_size - old_size),
	1278	sub_footer_size,
1044	1279	);
	1280	cur_ni = cur_ni.next(mf).unwrap() orelse break;
1045	1281	}
1046		first_floating_ni.setLocationAssumeCapacity(
1047		mf,
1048		new_first_floating_offset,
1049		first_floating_size,
1050		);
1051		// Continue the search after the just-moved floating node
1052		first_floating_ni = last_fixed.next;
1053		continue;
1054		},
1055		.none => {
1056		assert(direction == .forward);
1057		const new_parent_size = new_last_fixed_offset + last_fixed_size;
1058		if (new_parent_size > old_parent_size) {
1059		try mf.resizeNode(
	1282	}
	1283
	1284	// Finally, update the offsets of every footer before us:
	1285	if (node.prev.unwrap()) \|prev_ni\| {
	1286	var maybe_footer_ni = prev_ni;
	1287	while (true) {
	1288	switch (maybe_footer_ni.position(mf)) {
	1289	.header, .floating => break,
	1290	.footer => {},
	1291	}
	1292	const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf);
	1293	try maybe_footer_ni.setLocation(
	1294	mf,
1060	1295	gpa,
1061		node.parent,
1062		new_parent_size +\| new_parent_size / growth_factor,
	1296	moved_footer_offset + old_size - new_size,
	1297	moved_footer_size,
1063	1298	);
1064		_, old_parent_size = parent.location().resolve(mf);
	1299	maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break;
1065	1300	}
1066		},
1067		}
1068		try mf.ensureCapacityForSetLocation(gpa);
1069		if (last_fixed_ni == ni) {
1070		// The original fixed node now has enough space
1071		last_fixed_ni.setLocationAssumeCapacity(
1072		mf,
1073		old_last_fixed_offset,
1074		new_size,
1075		);
	1301	}
	1302
1076	1303	return;
1077		}
1078		// Move a fixed node into trailing free space
1079		if (last_fixed.flags.has_content) {
1080		const parent_file_offset = node.parent.fileLocation(mf, false).offset;
1081		try mf.moveRange(
1082		parent_file_offset + old_last_fixed_offset,
1083		parent_file_offset + new_last_fixed_offset,
1084		last_fixed_size,
1085		);
1086		}
1087		last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size);
1088		// Retry the previous nodes now that there is enough space
1089		first_floating_ni = last_fixed_ni;
1090		last_fixed_ni = last_fixed.prev;
1091		direction = .reverse;
	1304	},
1092	1305	}
1093	1306	}
1094	1307
1095		fn realignNode(
	1308	/// Moves a floating node to an unused region with the given size, which may be greater than the
	1309	/// current size. If `new_alignment` is not `null`, then the offset and size of the new region will
	1310	/// have that alignment instead of `ni.alignment(mf)`.
	1311	///
	1312	/// Asserts that `ni` is a floating node (and not `.root`).
	1313	///
	1314	/// Asserts that `new_size` is aligned to `new_alignment orelse ni.alignment(mf)`.
	1315	///
	1316	/// Asserts that `new_size` is greater than or equal to the current size of `ni`.
	1317	fn growFloatingNodeWithAlignment(
1096	1318	mf: *MappedFile,
1097	1319	gpa: Allocator,
1098	1320	ni: Node.Index,
1099		new_alignment: std.mem.Alignment,
1100		opts: Node.Index.RealignNodeOptions,
1101		) (Allocator.Error \|\| Io.Cancelable \|\| IoError)!void {
	1321	new_alignment: ?Alignment,
	1322	new_size: u64,
	1323	grow_mode: GrowMode,
	1324	) Error!void {
1102	1325	mf.nodes_lock.assertUnlocked();
1103	1326
1104		const node = ni.get(mf);
1105		{
1106		const prev_alignment = node.flags.alignment;
1107		node.flags.alignment = new_alignment;
1108		if (new_alignment.compare(.lte, prev_alignment)) return;
1109		}
	1327	const parent_ni = ni.parent(mf).unwrap().?; // `ni` cannot be `.root`
	1328	const old_offset, const old_size = ni.location(mf).resolve(mf);
1110	1329
1111		const old_offset, const size = node.location().resolve(mf);
1112		if (ni == Node.Index.root) return mf.resizeNode(gpa, ni, size);
	1330	const alignment = new_alignment orelse ni.alignment(mf);
1113	1331
1114		const new_size = new_alignment.forward(@intCast(size));
1115		if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size);
	1332	assert(new_size >= old_size);
	1333	assert(ni.position(mf) == .floating);
	1334	assert(alignment.check(new_size));
1116	1335
1117		_, const parent_size = node.parent.location(mf).resolve(mf);
1118		const trailing_end = trailing_end: switch (node.next) {
1119		.none => parent_size,
1120		else => \|next_ni\| {
	1336	grow_in_place: {
	1337	if (!alignment.check(old_offset)) {
	1338	break :grow_in_place;
	1339	}
	1340	const limit: u64 = limit: {
	1341	const next_ni = ni.next(mf).unwrap() orelse break :limit parent_ni.location(mf).resolve(mf)[1];
1121	1342	const next_offset, _ = next_ni.location(mf).resolve(mf);
1122		break :trailing_end next_offset;
1123		},
1124		};
	1343	break :limit next_offset;
	1344	};
	1345	if (old_offset + new_size > limit) {
	1346	break :grow_in_place; // the parent is not big enough
	1347	}
	1348	// Great, we can grow this node without changing its offset or moving any siblings.
	1349	try ni.setLocation(mf, gpa, old_offset, new_size);
	1350	// If we have any footers, we need to move them to the end of our new size, and update their
	1351	// offsets accordingly.
	1352	if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
	1353	var cur_ni = first_footer_ni;
	1354	var footers_have_content = false;
	1355	while (true) {
	1356	footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;
	1357	const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
	1358	try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
	1359	cur_ni = cur_ni.next(mf).unwrap() orelse break;
	1360	}
	1361	if (footers_have_content) {
	1362	const parent_file_off = parent_ni.fileLocation(mf, false).offset;
	1363	// This gets the new offset because we already updated the offsets above.
	1364	const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1365	const footers_size = new_size - new_footers_offset;
	1366	try mf.moveRange(
	1367	parent_file_off + old_offset + old_size - footers_size,
	1368	parent_file_off + old_offset + new_size - footers_size,
	1369	footers_size,
	1370	);
	1371	}
	1372	}
	1373	return;
	1374	}
	1375
	1376	const new_loc: struct {
	1377	offset: u64,
	1378	prev: Node.Index.Optional,
	1379	} = new_loc: {
	1380	_, const parent_size = parent_ni.location(mf).resolve(mf);
	1381
	1382	{
	1383	// See if there's space at the start of the parent.
	1384	const last_header_oni = parent_ni.lastHeader(mf);
	1385	const headers_end: u64 = if (last_header_oni.unwrap()) \|last_header_ni\| headers_end: {
	1386	const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf);
	1387	break :headers_end last_header_off + last_header_size;
	1388	} else 0;
	1389	const limit: u64 = limit: {
	1390	const after_header_oni: Node.Index.Optional = after_header: {
	1391	if (last_header_oni.unwrap()) \|last_header_ni\| {
	1392	break :after_header last_header_ni.next(mf);
	1393	}
	1394	break :after_header parent_ni.first(mf);
	1395	};
	1396	if (after_header_oni.unwrap()) \|after_header_ni\| {
	1397	break :limit after_header_ni.location(mf).resolve(mf)[0];
	1398	} else {
	1399	break :limit parent_size;
	1400	}
	1401	};
	1402	if (alignment.forward(headers_end) + new_size <= limit) {
	1403	// There's space here!
	1404	break :new_loc .{
	1405	// Put ourselves at the end of this range, so that the free space remains at the start of the parent.
	1406	.offset = alignment.backward(limit - new_size),
	1407	.prev = last_header_oni,
	1408	};
	1409	}
	1410	}
	1411
	1412	// Otherwise, use space at the end of the parent, or make space there if necessary.
	1413
	1414	const first_footer_oni = parent_ni.firstFooter(mf);
1125	1415
1126		if (opts.try_backwards) {
1127		const backward_offset = new_alignment.backward(@intCast(old_offset));
1128		const prev_end = if (node.prev == .none) 0 else prev: {
1129		const prev_offset, const prev_size = node.prev.location(mf).resolve(mf);
1130		break :prev prev_offset + prev_size;
	1416	// We know there is a node before the footer[s], because `ni` itself is such a node.
	1417	const prev_ni: Node.Index = if (first_footer_oni.unwrap()) \|first_footer_ni\| prev: {
	1418	break :prev first_footer_ni.prev(mf).unwrap().?;
	1419	} else prev: {
	1420	break :prev parent_ni.last(mf).unwrap().?;
1131	1421	};
1132	1422
1133		if (backward_offset >= prev_end) {
1134		try mf.ensureCapacityForSetLocation(gpa);
	1423	const result_offset: u64 = result_offset: {
	1424	if (prev_ni == ni and alignment.check(old_offset)) {
	1425	// We're already at the end of the parent, and our offset is already well-aligned.
	1426	// The only reason we didn't simply grow in place earlier is that the parent wasn't
	1427	// big enough---but now we're resizing the parent anyway, so growing in-place stops
	1428	// us from unnecessarily moving!
	1429	break :result_offset old_offset;
	1430	}
	1431	// Otherwise, just move after the last node.
	1432	const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf);
	1433	break :result_offset alignment.forward(prev_offset + prev_size);
	1434	};
1135	1435
1136		if (node.flags.has_content) {
1137		const old_file_offset = ni.fileLocation(mf, false).offset;
1138		const new_file_offset = (old_file_offset - old_offset) + backward_offset;
1139		@memmove(
1140		mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)],
1141		mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)],
1142		);
1143		@memset(mf.memory_map.memory[@intCast(new_file_offset + size)..@intCast(old_file_offset + size)], 0);
	1436	const footers_size: u64 = if (first_footer_oni.unwrap()) \|first_footer_ni\| footers_size: {
	1437	const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1438	break :footers_size parent_size - first_footer_offset;
	1439	} else 0;
	1440
	1441	const min_parent_size = result_offset + new_size + footers_size;
	1442	if (parent_size < min_parent_size) {
	1443	// Okay, at this point we're planning to expand the parent---so before we actually do
	1444	// that, let's first try the Linux "insert range" fast path. We didn't try it before now
	1445	// because it would have been more efficient to just move ourselves into existing space.
	1446	//
	1447	// If we were given a custom alignment, we cannot pass `grow_mode` directly into the
	1448	// "insert range" path, because that function is unaware of `new_alignment`.
	1449	const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact;
	1450	if (alignment.check(old_offset) and
	1451	try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode))
	1452	{
	1453	// The Linux fast path did our job for us!
	1454	return;
1144	1455	}
1145	1456
1146		if (backward_offset + new_size <= trailing_end) {
1147		ni.setLocationAssumeCapacity(mf, backward_offset, new_size);
	1457	// Grow the parent and move to the end of the parent.
	1458	const new_parent_size = parent_ni.alignment(mf).forward(
	1459	min_parent_size +\| min_parent_size / growth_factor,
	1460	);
	1461	try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	1462	}
	1463
	1464	break :new_loc .{
	1465	.offset = result_offset,
	1466	.prev = .wrap(prev_ni),
	1467	};
	1468	};
	1469
	1470	// We've found our new location in `parent_ni`, now to actually move ourselves there.
	1471
	1472	// Footers need to move to a different place than the rest of our content.
	1473	const footers_size: u64, const footers_have_content: bool = footers: {
	1474	const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
	1475	break :footers .{ 0, false };
	1476	};
	1477
	1478	var cur_ni = first_footer_ni;
	1479	var footers_have_content = false;
	1480	while (true) {
	1481	footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;
	1482	const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
	1483	// Our footers' offsets must change to be at the end of our new size.
	1484	try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
	1485	cur_ni = cur_ni.next(mf).unwrap() orelse break;
	1486	}
	1487
	1488	// This is the new offset because we already updated the offsets above.
	1489	const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1490	const footers_size = new_size - new_footers_offset;
	1491
	1492	break :footers .{ footers_size, footers_have_content };
	1493	};
	1494
	1495	if (ni.get(mf).flags.has_content) {
	1496	const parent_file_off = parent_ni.fileLocation(mf, false).offset;
	1497	try mf.moveRange(
	1498	parent_file_off + old_offset,
	1499	parent_file_off + new_loc.offset,
	1500	old_size - footers_size,
	1501	);
	1502	if (footers_have_content) try mf.moveRange(
	1503	parent_file_off + old_offset + old_size - footers_size,
	1504	parent_file_off + new_loc.offset + new_size - footers_size,
	1505	footers_size,
	1506	);
	1507	} else {
	1508	assert(!footers_have_content);
	1509	}
	1510
	1511	try ni.setLocation(mf, gpa, new_loc.offset, new_size);
	1512
	1513	if (new_loc.prev != ni.toOptional()) {
	1514	// We're potentially in a different place in `parent_ni`'s child list, so remove and re-add ourselves.
	1515	try mf.removeNodesFromChildList(gpa, ni, ni);
	1516	try mf.addNodesToChildListAfter(gpa, new_loc.prev, ni, ni);
	1517	}
	1518	}
	1519
	1520	/// Attempts to grow `ni` to `new_size` using `FALLOCATE_FL_INSERT_RANGE` on Linux. This strategy
	1521	/// has the advantage that it does not require manually moving any bytes in the file, but has the
	1522	/// disadvantages that it may increase the file size more than necessary, and that it changes the
	1523	/// offsets of all following nodes, recursively.
	1524	///
	1525	/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false`
	1526	/// without changing any nodes' locations or invalidating any slices.
	1527	///
	1528	/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node
	1529	/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`,
	1530	/// the actual new size of `ni` may be greater than `new_size`.
	1531	fn growNodeViaInsertRange(
	1532	mf: *MappedFile,
	1533	gpa: Allocator,
	1534	ni: Node.Index,
	1535	new_size: u64,
	1536	grow_mode: GrowMode,
	1537	) Error!bool {
	1538	if (!is_linux or mf.flags.fallocate_insert_range_unsupported) {
	1539	return false;
	1540	}
	1541
	1542	_, const old_size = ni.location(mf).resolve(mf);
	1543
	1544	// We don't compute the size of the range yet, because depending on `grow_mode` we might want to
	1545	// bump it based on our sibling and parent nodes' alignments. However, we can do an early check
	1546	// for cases where we should obviously exit.
	1547	const requested_range_size = new_size - old_size;
	1548	if (!mf.flags.block_size.check(requested_range_size)) {
	1549	// The requested size isn't exactly aligned.
	1550	switch (grow_mode) {
	1551	.exact => return false,
	1552	.minimum => {
	1553	// We can still choose to allow it by increasing the size a bit, but we shouldn't do
	1554	// that if it would significantly increase the requested size.
	1555	const block_size = mf.flags.block_size.toByteUnits();
	1556	if (requested_range_size < block_size * 2) {
	1557	// Bumping this size up to the next block boundary would be a quite significant
	1558	// increase; let's not do it.
	1559	return false;
	1560	}
	1561	},
	1562	}
	1563	}
	1564	// If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump
	1565	// the size a little more.
	1566	const min_range_size: u64 = s: {
	1567	const exact_size = new_size - old_size;
	1568	if (mf.flags.block_size.check(exact_size)) {
	1569	break :s exact_size;
	1570	}
	1571	switch (grow_mode) {
	1572	.exact => return false,
	1573	.minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) {
	1574	// We're growing by at least a few blocks, so allow ourselves to bump the size
	1575	// slightly to give it the needed alignment.
	1576	break :s mf.flags.block_size.forward(exact_size);
1148	1577	} else {
1149		ni.setLocationAssumeCapacity(mf, backward_offset, size);
1150		try mf.resizeNode(gpa, ni, new_size);
	1578	return false;
	1579	},
	1580	}
	1581	};
	1582	assert(min_range_size > 0);
	1583	assert(mf.flags.block_size.check(min_range_size));
	1584
	1585	const range_file_offset: u64 = range_file_offset: {
	1586	const node_file_offset = ni.fileLocation(mf, false).offset;
	1587	const last_ni = ni.last(mf).unwrap() orelse {
	1588	// If `ni` has no children (i.e. is a leaf node), we need to insert exactly at its end.
	1589	const range_file_offset = node_file_offset + old_size;
	1590	if (!mf.flags.block_size.check(range_file_offset)) {
	1591	return false;
	1592	}
	1593	break :range_file_offset range_file_offset;
	1594	};
	1595	const first_footer_oni = ni.firstFooter(mf);
	1596	const footers_size: u64 = if (first_footer_oni.unwrap()) \|first_footer_ni\| size: {
	1597	const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);
	1598	break :size old_size - first_footer_offset;
	1599	} else 0;
	1600	const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) \|first_footer_ni\| pre_footer: {
	1601	break :pre_footer first_footer_ni.prev(mf);
	1602	} else .wrap(last_ni);
	1603	const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) \|pre_footer_ni\| end: {
	1604	const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf);
	1605	break :end pre_footer_off + pre_footer_size;
	1606	} else 0;
	1607
	1608	const min_file_offset = node_file_offset + pre_footer_end;
	1609	const max_file_offset = node_file_offset + old_size - footers_size;
	1610	// We can go anywhere between `min_file_offset` and `max_file_offset`.
	1611	const candidate_file_offset = mf.flags.block_size.forward(min_file_offset);
	1612	if (candidate_file_offset > max_file_offset) {
	1613	return false;
	1614	}
	1615	break :range_file_offset candidate_file_offset;
	1616	};
	1617	assert(mf.flags.block_size.check(range_file_offset));
	1618
	1619	const range_size: u64 = range_size: {
	1620	// For this strategy to be valid, the number of bytes we insert needs to be compatible with
	1621	// the alignments of all nodes following us (and following our parents, their parents, etc).
	1622	// We also probably don't want to trigger too many "node moved" events, since doing that
	1623	// repeatedly could result in a lot of extra work. Therefore, while we traverse parents and
	1624	// siblings to check their alignment requirements, we will also set an arbitrary limit on
	1625	// the number of nodes we can move, and give up if we walk more than that.
	1626	const max_moved_nodes = 32;
	1627	var num_moved: u32 = 0;
	1628	var cur_ni = ni;
	1629	// Alignment required for `range_size`: initially the block size (required for the syscall),
	1630	// then updated as we traverse based on how the operation would affect surrounding nodes.
	1631	var need_range_align: Alignment = mf.flags.block_size.max(ni.alignment(mf));
	1632	while (true) {
	1633	// `cur_ni` will grow as a result of the range insertion. Its size must be well-aligned.
	1634	need_range_align = need_range_align.max(cur_ni.alignment(mf));
	1635
	1636	// Siblings following `cur_ni` don't get bigger, but their offsets change.
	1637	while (cur_ni.next(mf).unwrap()) \|next_ni\| {
	1638	// Only floating children need well-aligned offsets.
	1639	if (next_ni.position(mf) == .floating) {
	1640	need_range_align = need_range_align.max(next_ni.alignment(mf));
	1641	}
	1642	num_moved += 1;
	1643	if (num_moved > max_moved_nodes) return false;
	1644	cur_ni = next_ni;
1151	1645	}
1152	1646
1153		return;
	1647	// Move up to the parent.
	1648	cur_ni = cur_ni.parent(mf).unwrap() orelse break;
	1649	}
	1650	// Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment
	1651	// requirement to figure out whether we're actually going to insert a range.
	1652	if (need_range_align.check(requested_range_size)) {
	1653	break :range_size requested_range_size;
	1654	}
	1655	// Perhaps we're allowed to grow by more than `requested_range_size`?
	1656	switch (grow_mode) {
	1657	.exact => return false,
	1658	.minimum => {
	1659	const candidate_range_size = need_range_align.forward(min_range_size);
	1660	// Allow growing by up to 50% more than was requested.
	1661	if (candidate_range_size <= requested_range_size +\| requested_range_size / 2) {
	1662	break :range_size candidate_range_size;
	1663	} else {
	1664	return false;
	1665	}
	1666	},
	1667	}
	1668	};
	1669
	1670	// This `range_size` is compatible with everyone's alignment requirements, and we won't move too
	1671	// many nodes, so let's do it!
	1672
	1673	mf.memory_map.write(mf.io) catch \|err\| {
	1674	mf.io_err = switch (err) {
	1675	error.Canceled => \|e\| return e,
	1676	error.WouldBlock => error.Unexpected, // file was not opened as non-blocking
	1677	error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing
	1678	else => \|e\| e,
	1679	};
	1680	return error.MappedFileIo;
	1681	};
	1682
	1683	// If we happen to be inserting at the very end of the file, we need to resize the file instead
	1684	// of using `FALLOCATE_FL_INSERT_RANGE`.
	1685	if (range_file_offset == Node.Index.root.location(mf).resolve(mf)[1]) {
	1686	mf.memory_map.file.setLength(mf.io, range_file_offset + range_size) catch \|err\| switch (err) {
	1687	error.Canceled => \|e\| return e,
	1688	else => \|e\| {
	1689	mf.io_err = e;
	1690	return error.MappedFileIo;
	1691	},
	1692	};
	1693	} else {
	1694	while (true) switch (linux.errno(linux.fallocate(
	1695	mf.memory_map.file.handle,
	1696	linux.FALLOC.FL_INSERT_RANGE,
	1697	@intCast(range_file_offset),
	1698	@intCast(range_size),
	1699	))) {
	1700	.SUCCESS => break,
	1701	.INTR => continue,
	1702	.NOSYS, .OPNOTSUPP => {
	1703	// After all that setup work, it turns out the operation is actually unsupported!
	1704	mf.flags.fallocate_insert_range_unsupported = true;
	1705	return false;
	1706	},
	1707	else => \|e\| {
	1708	mf.io_err = switch (e) {
	1709	.SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above
	1710	.BADF => unreachable,
	1711	.FBIG => unreachable,
	1712	.INVAL => unreachable,
	1713	.IO => error.InputOutput,
	1714	.NODEV => error.NotFile,
	1715	.NOSPC => error.NoSpaceLeft,
	1716	.PERM => error.PermissionDenied,
	1717	.SPIPE => error.Unseekable,
	1718	.TXTBSY => error.FileBusy,
	1719	else => std.posix.unexpectedErrno(e),
	1720	};
	1721	return error.MappedFileIo;
	1722	},
	1723	};
	1724	}
	1725
	1726	// We did it! Now to update all the sizes and offsets. This loop is exactly the same shape as
	1727	// above, except we're updating locations instead of checking alignments.
	1728	var cur_ni = ni;
	1729	while (true) {
	1730	const this_offset, const this_old_size = cur_ni.location(mf).resolve(mf);
	1731	if (cur_ni == .root) {
	1732	try mf.ensureTotalCapacityPrecise(@intCast(this_old_size + range_size));
1154	1733	}
	1734	try cur_ni.setLocation(mf, gpa, this_offset, this_old_size + range_size);
	1735
	1736	while (cur_ni.next(mf).unwrap()) \|next_ni\| {
	1737	const next_old_offset, const next_size = next_ni.location(mf).resolve(mf);
	1738	try next_ni.setLocation(mf, gpa, next_old_offset + range_size, next_size);
	1739	cur_ni = next_ni;
	1740	}
	1741
	1742	cur_ni = cur_ni.parent(mf).unwrap() orelse break;
1155	1743	}
1156	1744
1157		const forward_offset = new_alignment.forward(@intCast(old_offset));
1158		if (forward_offset + new_size <= trailing_end) {
1159		// Shift into the free space if possible
1160		try mf.ensureCapacityForSetLocation(gpa);
1161		if (node.flags.has_content) {
1162		const old_file_offset = ni.fileLocation(mf, false).offset;
1163		const new_file_offset = (old_file_offset - old_offset) + forward_offset;
1164		if (new_file_offset < old_file_offset + size) {
1165		@memmove(
1166		mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)],
1167		mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)],
1168		);
1169		} else try mf.moveRange(old_file_offset, new_file_offset, size);
1170		@memset(mf.memory_map.memory[@intCast(new_file_offset + size)..][0..@intCast(new_size - size)], 0);
	1745	// The only thing left is to update the offsets of any footers inside of `ni`.
	1746	if (ni.firstFooter(mf).unwrap()) \|first_footer_ni\| {
	1747	var footer_ni = first_footer_ni;
	1748	while (true) {
	1749	const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
	1750	try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);
	1751	footer_ni = footer_ni.next(mf).unwrap() orelse break;
1171	1752	}
	1753	}
1172	1754
1173		ni.setLocationAssumeCapacity(mf, forward_offset, new_size);
1174		} else {
1175		const temp_size = new_alignment.forward(@intCast(new_size + 1));
1176		try mf.resizeNode(gpa, ni, temp_size);
1177		const new_offset, _ = ni.location(mf).resolve(mf);
1178
1179		try mf.ensureCapacityForSetLocation(gpa);
1180
1181		// Non-fixed nodes may now be aligned if the resize moved them
1182		const new_forward_offset = new_alignment.forward(@intCast(new_offset));
1183		const final_offset = if (new_forward_offset != new_offset) final_offset: {
1184		if (node.flags.has_content) {
1185		const old_file_offset = ni.fileLocation(mf, false).offset;
1186		const new_file_offset = (old_file_offset - new_offset) + new_forward_offset;
1187		@memmove(
1188		mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)],
1189		mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)],
1190		);
1191		@memset(mf.memory_map.memory[@intCast(old_file_offset)..@intCast(new_file_offset)], 0);
	1755	return true;
	1756	}
	1757
	1758	/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes following its current
	1759	/// headers, so that the headers can grow into that space.
	1760	fn ensureAdditionalHeaderCapacity(
	1761	mf: *MappedFile,
	1762	gpa: Allocator,
	1763	parent_ni: Node.Index,
	1764	extra_capacity: u64,
	1765	) Error!void {
	1766	_, const parent_size = parent_ni.location(mf).resolve(mf);
	1767
	1768	const last_header_oni = parent_ni.lastHeader(mf);
	1769	const first_footer_oni = parent_ni.firstFooter(mf);
	1770
	1771	const headers_size: u64 = headers_size: {
	1772	const last_header_ni = last_header_oni.unwrap() orelse break :headers_size 0;
	1773	const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf);
	1774	break :headers_size last_header_off + last_header_size;
	1775	};
	1776
	1777	const footers_size: u64 = footers_size: {
	1778	const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0;
	1779	const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);
	1780	break :footers_size parent_size - first_footer_off;
	1781	};
	1782
	1783	const first_floating_oni: Node.Index.Optional = if (last_header_oni.unwrap()) \|last_header_ni\| first_floating: {
	1784	const after_header_ni = last_header_ni.next(mf).unwrap() orelse break :first_floating .none;
	1785	break :first_floating switch (after_header_ni.position(mf)) {
	1786	.header => unreachable,
	1787	.floating => .wrap(after_header_ni),
	1788	.footer => .none,
	1789	};
	1790	} else first_floating: {
	1791	const first_ni = parent_ni.first(mf).unwrap() orelse break :first_floating .none;
	1792	break :first_floating switch (first_ni.position(mf)) {
	1793	.header => unreachable,
	1794	.floating => .wrap(first_ni),
	1795	.footer => .none,
	1796	};
	1797	};
	1798	const first_floating_ni = first_floating_oni.unwrap() orelse {
	1799	// This node has only headers and footers.
	1800	const min_parent_size = headers_size + extra_capacity + footers_size;
	1801	if (parent_size < min_parent_size) {
	1802	const new_parent_size = parent_ni.alignment(mf).forward(
	1803	min_parent_size +\| min_parent_size / growth_factor,
	1804	);
	1805	try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	1806	}
	1807	return;
	1808	};
	1809
	1810	const last_floating_ni = if (first_footer_oni.unwrap()) \|first_footer_ni\| last_floating: {
	1811	break :last_floating first_footer_ni.prev(mf).unwrap().?;
	1812	} else last_floating: {
	1813	break :last_floating parent_ni.last(mf).unwrap().?;
	1814	};
	1815	assert(last_floating_ni.position(mf) == .floating); // we know `parent_ni` contains at least `first_floating_ni`
	1816
	1817	// Find the first floating child, if any, which does not overlap the new header space.
	1818	const first_good_floating_oni: Node.Index.Optional = first_good_floating: {
	1819	var floating_ni = first_floating_ni;
	1820	while (true) {
	1821	const floating_offset, _ = floating_ni.location(mf).resolve(mf);
	1822	if (floating_offset >= headers_size + extra_capacity) {
	1823	break :first_good_floating .wrap(floating_ni);
	1824	}
	1825	const next_ni = floating_ni.next(mf).unwrap() orelse {
	1826	break :first_good_floating .none;
	1827	};
	1828	switch (next_ni.position(mf)) {
	1829	.header => unreachable, // after the last header
	1830	.floating => floating_ni = next_ni,
	1831	.footer => break :first_good_floating .none,
1192	1832	}
	1833	}
	1834	};
1193	1835
1194		break :final_offset new_forward_offset;
1195		} else new_offset;
	1836	if (first_good_floating_oni == first_floating_ni.toOptional()) {
	1837	// None of the floating children are in our way! That means there's already enough space.
	1838	return;
	1839	}
	1840
	1841	const last_moving_ni = if (first_good_floating_oni.unwrap()) \|first_good_floating_ni\| last_moving: {
	1842	break :last_moving first_good_floating_ni.prev(mf).unwrap().?;
	1843	} else last_moving: {
	1844	break :last_moving last_floating_ni;
	1845	};
	1846
	1847	// We are going to move all nodes between `first_floating_ni` and `last_moving_ni` to the end of
	1848	// the parent. We'll move all the node data in one big block.
1196	1849
1197		ni.setLocationAssumeCapacity(mf, final_offset, new_size);
	1850	const moving_offset: u64 = first_floating_ni.location(mf).resolve(mf)[0];
	1851	const moving_size: u64 = size: {
	1852	const last_moving_off, const last_moving_size = last_moving_ni.location(mf).resolve(mf);
	1853	break :size last_moving_off + last_moving_size - moving_offset;
	1854	};
	1855
	1856	var moving_alignment: Alignment = .@"1";
	1857	var moving_has_content = false; // optimization: no need to move data if it's all uninitialized
	1858	{
	1859	var cur_ni = first_floating_ni;
	1860	while (true) {
	1861	moving_alignment = moving_alignment.max(cur_ni.alignment(mf));
	1862	moving_has_content = moving_has_content or cur_ni.get(mf).flags.has_content;
	1863	if (cur_ni == last_moving_ni) break;
	1864	cur_ni = cur_ni.next(mf).unwrap().?;
	1865	}
	1866	}
	1867
	1868	const first_free_offset = free_offset: {
	1869	const last_floating_off, const last_floating_size = last_floating_ni.location(mf).resolve(mf);
	1870	break :free_offset @max(last_floating_off + last_floating_size, headers_size + extra_capacity);
	1871	};
	1872	// Alignment is a little tricky here. We don't necessarily want the new offset to be aligned to
	1873	// `moving_alignment` exactly, because if (e.g.) the first floating node is align(2) and the
	1874	// second is align(4), then the overall range we're moving may not be 4-byte aligned even though
	1875	// one of the nodes is. Instead, the old and new offsets must be congruent modulo the alignment.
	1876	const aligned_dest_offset = moving_alignment.forward(first_free_offset);
	1877	const dest_offset = aligned_dest_offset + (moving_offset - moving_alignment.backward(moving_offset));
	1878	assert(dest_offset % moving_alignment.toByteUnits() == moving_offset % moving_alignment.toByteUnits());
	1879
	1880	// This expression is correct because `dest_offset` is after all floating nodes (except the ones
	1881	// we're moving there of course).
	1882	const min_parent_size = dest_offset + moving_size + footers_size;
	1883	if (parent_size < min_parent_size) {
	1884	const new_parent_size = parent_ni.alignment(mf).forward(
	1885	min_parent_size +\| min_parent_size / growth_factor,
	1886	);
	1887	try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	1888	}
	1889
	1890	if (moving_has_content) {
	1891	const parent_file_off = parent_ni.fileLocation(mf, false).offset;
	1892	try mf.moveRange(
	1893	parent_file_off + moving_offset,
	1894	parent_file_off + dest_offset,
	1895	moving_size,
	1896	);
	1897	}
	1898
	1899	// Remove everything between `first_floating_ni` and `last_moving_ni` from the linked list, then
	1900	// re-insert them in their new position.
	1901	try mf.removeNodesFromChildList(gpa, first_floating_ni, last_moving_ni);
	1902	try mf.addNodesToChildListBefore(gpa, first_footer_oni, first_floating_ni, last_moving_ni);
	1903
	1904	// Finally, we need to update the locations of all of those nodes.
	1905	var cur_ni = first_floating_ni;
	1906	while (true) {
	1907	assert(cur_ni.position(mf) == .floating);
	1908	const old_offset, const old_size = cur_ni.location(mf).resolve(mf);
	1909	const new_offset = old_offset - moving_offset + dest_offset;
	1910	assert(cur_ni.alignment(mf).check(new_offset));
	1911	try cur_ni.setLocation(mf, gpa, new_offset, old_size);
	1912	if (cur_ni == last_moving_ni) break;
	1913	cur_ni = cur_ni.next(mf).unwrap().?;
	1914	}
	1915	}
	1916
	1917	/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current
	1918	/// footers, so that the footers can grow into that space.
	1919	fn ensureAdditionalFooterCapacity(
	1920	mf: *MappedFile,
	1921	gpa: Allocator,
	1922	parent_ni: Node.Index,
	1923	extra_capacity: u64,
	1924	) Error!void {
	1925	// This is way easier than the header case, because we don't need to actually move anything; we
	1926	// just need to expand the parent if there isn't space, and that will add padding after the
	1927	// parent's floating children, which is exactly where we want it.
	1928
	1929	const first_footer_oni = parent_ni.firstFooter(mf);
	1930
	1931	_, const parent_size = parent_ni.location(mf).resolve(mf);
	1932
	1933	const footers_size: u64 = footers_size: {
	1934	const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0;
	1935	const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);
	1936	break :footers_size parent_size - first_footer_off;
	1937	};
	1938
	1939	const header_and_floating_end: u64 = end: {
	1940	const before_footers_oni = if (first_footer_oni.unwrap()) \|first_footer_ni\| before_footers: {
	1941	break :before_footers first_footer_ni.prev(mf);
	1942	} else before_footers: {
	1943	break :before_footers parent_ni.last(mf);
	1944	};
	1945	const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0;
	1946	const offset, const size = before_footers_ni.location(mf).resolve(mf);
	1947	break :end offset + size;
	1948	};
	1949
	1950	assert(header_and_floating_end + footers_size <= parent_size);
	1951
	1952	const min_parent_size = header_and_floating_end + footers_size + extra_capacity;
	1953	if (parent_size < min_parent_size) {
	1954	const new_parent_size = parent_ni.alignment(mf).forward(
	1955	min_parent_size +\| min_parent_size / growth_factor,
	1956	);
	1957	try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
	1958	}
	1959	}
	1960
	1961	fn removeNodesFromChildList(
	1962	mf: *MappedFile,
	1963	gpa: Allocator,
	1964	first_remove_ni: Node.Index,
	1965	last_remove_ni: Node.Index,
	1966	) Allocator.Error!void {
	1967	const parent_ni = first_remove_ni.parent(mf).unwrap().?;
	1968	assert(last_remove_ni.parent(mf).unwrap().? == parent_ni);
	1969
	1970	const prev_oni = first_remove_ni.prev(mf);
	1971	const next_oni = last_remove_ni.next(mf);
	1972
	1973	if (prev_oni.unwrap()) \|prev_ni\| {
	1974	assert(prev_ni.next(mf).unwrap().? == first_remove_ni);
	1975	try prev_ni.setNext(gpa, next_oni, mf);
	1976	} else {
	1977	assert(parent_ni.first(mf).unwrap().? == first_remove_ni);
	1978	parent_ni.get(mf).first = next_oni;
	1979	}
	1980
	1981	if (next_oni.unwrap()) \|next_ni\| {
	1982	assert(next_ni.prev(mf).unwrap().? == last_remove_ni);
	1983	next_ni.get(mf).prev = prev_oni;
	1984	} else {
	1985	assert(parent_ni.last(mf).unwrap().? == last_remove_ni);
	1986	parent_ni.get(mf).last = prev_oni;
	1987	}
	1988	}
	1989	/// Assumes `first_add_ni` and `last_add_ni` are connected, and that all nodes in between them
	1990	/// already have their `parent` field correctly populated.
	1991	///
	1992	/// To add a single node, set `first_add_ni` equal to `last_add_ni`.
	1993	fn addNodesToChildListBefore(
	1994	mf: *MappedFile,
	1995	gpa: Allocator,
	1996	/// `null` means to add at the end of the parent.
	1997	next_oni: Node.Index.Optional,
	1998	first_add_ni: Node.Index,
	1999	last_add_ni: Node.Index,
	2000	) Allocator.Error!void {
	2001	const parent_ni = first_add_ni.parent(mf).unwrap().?;
	2002	assert(last_add_ni.parent(mf).unwrap().? == parent_ni);
	2003	if (next_oni.unwrap()) \|next_ni\| {
	2004	assert(next_ni.parent(mf).unwrap().? == parent_ni);
	2005	}
	2006
	2007	const prev_oni: Node.Index.Optional = if (next_oni.unwrap()) \|next_ni\| prev: {
	2008	break :prev next_ni.prev(mf);
	2009	} else prev: {
	2010	break :prev parent_ni.last(mf);
	2011	};
	2012
	2013	first_add_ni.get(mf).prev = prev_oni;
	2014	try last_add_ni.setNext(gpa, next_oni, mf);
	2015
	2016	if (prev_oni.unwrap()) \|prev_ni\| {
	2017	assert(prev_ni.next(mf) == next_oni);
	2018	try prev_ni.setNext(gpa, .wrap(first_add_ni), mf);
	2019	} else {
	2020	assert(parent_ni.first(mf) == next_oni);
	2021	parent_ni.get(mf).first = .wrap(first_add_ni);
1198	2022	}
	2023
	2024	if (next_oni.unwrap()) \|next_ni\| {
	2025	assert(next_ni.prev(mf) == prev_oni);
	2026	next_ni.get(mf).prev = .wrap(last_add_ni);
	2027	} else {
	2028	assert(parent_ni.last(mf) == prev_oni);
	2029	parent_ni.get(mf).last = .wrap(last_add_ni);
	2030	}
	2031	}
	2032	fn addNodesToChildListAfter(
	2033	mf: *MappedFile,
	2034	gpa: Allocator,
	2035	/// `null` means to add at the start of the parent.
	2036	prev_oni: Node.Index.Optional,
	2037	first_add_ni: Node.Index,
	2038	last_add_ni: Node.Index,
	2039	) Allocator.Error!void {
	2040	const next_oni: Node.Index.Optional = next: {
	2041	if (prev_oni.unwrap()) \|prev_ni\| break :next prev_ni.next(mf);
	2042	const parent_ni = first_add_ni.parent(mf).unwrap().?;
	2043	break :next parent_ni.first(mf);
	2044	};
	2045	return mf.addNodesToChildListBefore(gpa, next_oni, first_add_ni, last_add_ni);
	2046	}
	2047
	2048	fn realignNode(
	2049	mf: *MappedFile,
	2050	gpa: Allocator,
	2051	ni: Node.Index,
	2052	new_alignment: Alignment,
	2053	) Error!void {
	2054	mf.nodes_lock.assertUnlocked();
	2055
	2056	const old_offset, const old_size = ni.location(mf).resolve(mf);
	2057
	2058	if (ni == .root or ni.position(mf) != .floating) {
	2059	// Only this node's size is aligned, not its offset.
	2060	if (!new_alignment.check(old_size)) {
	2061	assert(new_alignment.compare(.gt, ni.alignment(mf)));
	2062	try mf.growNode(
	2063	gpa,
	2064	ni,
	2065	new_alignment.forward(old_size),
	2066	.exact, // because `growNode` is not aware that the size needs to match `new_alignment`
	2067	);
	2068	}
	2069	} else {
	2070	// This is a floating node, so its size and offset are both aligned.
	2071	if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) {
	2072	assert(new_alignment.compare(.gt, ni.alignment(mf)));
	2073	try mf.growFloatingNodeWithAlignment(
	2074	gpa,
	2075	ni,
	2076	new_alignment,
	2077	new_alignment.forward(old_size),
	2078	.minimum,
	2079	);
	2080	}
	2081	}
	2082
	2083	ni.get(mf).flags.alignment = new_alignment;
1199	2084	}
1200	2085
1201	2086	fn updateWriters(mf: *MappedFile) void {
...	...	@@ -1206,10 +2091,47 @@ fn updateWriters(mf: *MappedFile) void {
1206	2091	}
1207	2092	}
1208	2093
1209		fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable \|\| IoError)!void {
1210		// make a copy of this node at the new location
1211		try mf.copyRange(old_file_offset, new_file_offset, size);
1212		// delete the copy of this node at the old location
	2094	fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) Error!void {
	2095	if (old_file_offset == new_file_offset) return;
	2096
	2097	if (old_file_offset >= new_file_offset + size or
	2098	new_file_offset >= old_file_offset + size)
	2099	{
	2100	const n = try mf.copyFileRange(
	2101	mf.memory_map.file,
	2102	old_file_offset,
	2103	new_file_offset,
	2104	size,
	2105	);
	2106	@memcpy(
	2107	mf.memory_map.memory[@intCast(new_file_offset + n)..][0..@intCast(size - n)],
	2108	mf.memory_map.memory[@intCast(old_file_offset + n)..][0..@intCast(size - n)],
	2109	);
	2110
	2111	try mf.zeroRange(old_file_offset, size);
	2112
	2113	return;
	2114	}
	2115
	2116	// TODO: if the non-overlapping region is greater than or equal to a filesystem block, is it
	2117	// ever worth doing multiple `copyFileRange` calls instead of a big `@memmove`?
	2118
	2119	@memmove(
	2120	mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)],
	2121	mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)],
	2122	);
	2123
	2124	if (new_file_offset > old_file_offset) {
	2125	const clear_size = new_file_offset - old_file_offset;
	2126	assert(clear_size < size);
	2127	try mf.zeroRange(old_file_offset, clear_size);
	2128	} else {
	2129	const clear_size = old_file_offset - new_file_offset;
	2130	assert(clear_size < size);
	2131	try mf.zeroRange(new_file_offset + size, clear_size);
	2132	}
	2133	}
	2134	fn zeroRange(mf: *MappedFile, file_offset: u64, size: u64) Error!void {
1213	2135	if (is_linux and
1214	2136	!mf.flags.fallocate_punch_hole_unsupported and
1215	2137	size >= mf.flags.block_size.toByteUnits() * 2 - 1)
...	...	@@ -1217,147 +2139,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size:
1217	2139	while (true) switch (linux.errno(linux.fallocate(
1218	2140	mf.memory_map.file.handle,
1219	2141	linux.FALLOC.FL_PUNCH_HOLE \| linux.FALLOC.FL_KEEP_SIZE,
1220		@intCast(old_file_offset),
	2142	@intCast(file_offset),
1221	2143	@intCast(size),
1222	2144	))) {
1223	2145	.SUCCESS => return,
1224	2146	.INTR => continue,
1225		.BADF, .FBIG, .INVAL => unreachable,
1226		.IO => return error.InputOutput,
1227		.NODEV => return error.NotFile,
1228		.NOSPC => return error.NoSpaceLeft,
1229	2147	.NOSYS, .OPNOTSUPP => {
1230	2148	mf.flags.fallocate_punch_hole_unsupported = true;
1231	2149	break; // fall back to slow path
1232	2150	},
1233		.PERM => return error.PermissionDenied,
1234		.SPIPE => return error.Unseekable,
1235		.TXTBSY => return error.FileBusy,
1236		else => \|e\| return std.posix.unexpectedErrno(e),
	2151	else => \|e\| {
	2152	mf.io_err = switch (e) {
	2153	.SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above
	2154	.BADF => unreachable,
	2155	.FBIG => unreachable,
	2156	.INVAL => unreachable,
	2157	.IO => error.InputOutput,
	2158	.NODEV => error.NotFile,
	2159	.NOSPC => error.NoSpaceLeft,
	2160	.PERM => error.PermissionDenied,
	2161	.SPIPE => error.Unseekable,
	2162	.TXTBSY => error.FileBusy,
	2163	else => std.posix.unexpectedErrno(e),
	2164	};
	2165	return error.MappedFileIo;
	2166	},
1237	2167	};
1238	2168	}
1239		@memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0);
1240		}
1241
1242		fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable \|\| IoError)!void {
1243		const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size);
1244		if (copy_size < size) @memcpy(
1245		mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)],
1246		mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)],
1247		);
	2169	@memset(mf.memory_map.memory[@intCast(file_offset)..][0..@intCast(size)], 0);
1248	2170	}
1249
1250	2171	fn copyFileRange(
1251	2172	mf: *MappedFile,
1252	2173	old_file: Io.File,
1253	2174	old_file_offset: u64,
1254	2175	new_file_offset: u64,
1255	2176	size: u64,
1256		) (Io.Cancelable \|\| IoError)!u64 {
	2177	) Error!u64 {
	2178	if (!is_linux or mf.flags.copy_file_range_unsupported) {
	2179	return 0;
	2180	}
	2181
	2182	const min_size = mf.flags.block_size.toByteUnits() * 2 - 1;
	2183	if (size < min_size) return 0;
	2184
1257	2185	const io = mf.io;
1258		mf.memory_map.write(io) catch \|err\| switch (err) {
1259		error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking
1260		error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing
1261		else => \|e\| return e,
	2186	mf.memory_map.write(io) catch \|err\| {
	2187	mf.io_err = switch (err) {
	2188	error.Canceled => \|e\| return e,
	2189	error.WouldBlock => error.Unexpected, // file was not opened as non-blocking
	2190	error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing
	2191	else => \|e\| e,
	2192	};
	2193	return error.MappedFileIo;
1262	2194	};
1263	2195	var remaining_size = size;
1264		if (is_linux and !mf.flags.copy_file_range_unsupported) {
1265		var old_file_offset_mut: i64 = @intCast(old_file_offset);
1266		var new_file_offset_mut: i64 = @intCast(new_file_offset);
1267		while (remaining_size >= mf.flags.block_size.toByteUnits() * 2 - 1) {
1268		const copy_len = linux.copy_file_range(
1269		old_file.handle,
1270		&old_file_offset_mut,
1271		mf.memory_map.file.handle,
1272		&new_file_offset_mut,
1273		@intCast(remaining_size),
1274		0,
1275		);
1276		switch (linux.errno(copy_len)) {
1277		.SUCCESS => {
1278		if (copy_len == 0) break;
1279		remaining_size -= copy_len;
1280		if (remaining_size == 0) break;
1281		},
1282		.INTR => continue,
1283		.BADF, .FBIG, .INVAL, .OVERFLOW => unreachable,
1284		.IO => return error.InputOutput,
1285		.ISDIR => return error.IsDir,
1286		.NOMEM => return error.SystemResources,
1287		.NOSPC => return error.NoSpaceLeft,
1288		.NOSYS, .OPNOTSUPP, .XDEV => {
1289		mf.flags.copy_file_range_unsupported = true;
1290		break;
1291		},
1292		.PERM => return error.PermissionDenied,
1293		.TXTBSY => return error.FileBusy,
1294		else => \|e\| return std.posix.unexpectedErrno(e),
1295		}
	2196	var old_file_offset_mut: i64 = @intCast(old_file_offset);
	2197	var new_file_offset_mut: i64 = @intCast(new_file_offset);
	2198	while (remaining_size >= min_size) {
	2199	const copy_len = linux.copy_file_range(
	2200	old_file.handle,
	2201	&old_file_offset_mut,
	2202	mf.memory_map.file.handle,
	2203	&new_file_offset_mut,
	2204	@intCast(remaining_size),
	2205	0,
	2206	);
	2207	switch (linux.errno(copy_len)) {
	2208	.SUCCESS => {
	2209	if (copy_len == 0) break;
	2210	remaining_size -= copy_len;
	2211	if (remaining_size == 0) break;
	2212	},
	2213	.INTR => continue,
	2214	.NOSYS, .OPNOTSUPP, .XDEV => {
	2215	mf.flags.copy_file_range_unsupported = true;
	2216	break;
	2217	},
	2218	else => \|e\| {
	2219	mf.io_err = switch (e) {
	2220	.SUCCESS, .INTR, .NOSYS, .OPNOTSUPP, .XDEV => unreachable, // handled above
	2221	.BADF => unreachable,
	2222	.FBIG => unreachable,
	2223	.INVAL => unreachable,
	2224	.OVERFLOW => unreachable,
	2225	.IO => error.InputOutput,
	2226	.ISDIR => error.IsDir,
	2227	.NOMEM => error.SystemResources,
	2228	.NOSPC => error.NoSpaceLeft,
	2229	.PERM => error.PermissionDenied,
	2230	.TXTBSY => error.FileBusy,
	2231	else => std.posix.unexpectedErrno(e),
	2232	};
	2233	return error.MappedFileIo;
	2234	},
1296	2235	}
1297	2236	}
1298	2237	return size - remaining_size;
1299	2238	}
1300	2239
1301		fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: Allocator) Allocator.Error!void {
1302		try mf.large.ensureUnusedCapacity(gpa, 2);
1303		try mf.updates.ensureUnusedCapacity(gpa, 2);
1304		}
1305
1306	2240	pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void {
1307		mf.ensureTotalCapacityInner(new_capacity) catch \|err\| switch (err) {
1308		error.OutOfMemory,
1309		error.Canceled,
1310		=> \|e\| return e,
1311
1312		else => \|e\| {
1313		mf.io_err = e;
1314		return error.MappedFileIo;
1315		},
1316		};
1317		}
1318		fn ensureTotalCapacityInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error \|\| Io.Cancelable \|\| IoError)!void {
1319	2241	if (mf.memory_map.memory.len >= new_capacity) return;
1320		try mf.ensureTotalCapacityPreciseInner(new_capacity +\| new_capacity / growth_factor);
	2242	try mf.ensureTotalCapacityPrecise(new_capacity +\| new_capacity / growth_factor);
1321	2243	}
1322	2244
1323	2245	pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void {
1324		mf.ensureTotalCapacityPreciseInner(new_capacity) catch \|err\| switch (err) {
1325		error.OutOfMemory,
1326		error.Canceled,
1327		=> \|e\| return e,
1328
1329		else => \|e\| {
1330		mf.io_err = e;
1331		return error.MappedFileIo;
1332		},
1333		};
1334		}
1335		fn ensureTotalCapacityPreciseInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error \|\| Io.Cancelable \|\| IoError)!void {
1336	2246	if (mf.memory_map.memory.len >= new_capacity) return;
1337	2247	const io = mf.io;
1338		const aligned_capacity = mf.flags.block_size.forward(new_capacity);
	2248	const aligned_capacity: usize = @intCast(
	2249	mf.flags.block_size.forward(new_capacity),
	2250	);
1339	2251
1340	2252	if (mf.memory_map.memory.len > 0) {
1341	2253	if (mf.memory_map.setLength(io, aligned_capacity)) \|_\| {
1342	2254	return;
1343	2255	} else \|err\| switch (err) {
1344	2256	error.OperationUnsupported => {},
1345		else => \|e\| return e,
	2257	error.OutOfMemory, error.Canceled => \|e\| return e,
	2258	else => \|e\| {
	2259	mf.io_err = e;
	2260	return error.MappedFileIo;
	2261	},
1346	2262	}
1347	2263
1348		mf.memory_map.write(io) catch \|err\| switch (err) {
1349		error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking
1350		error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing
1351		else => \|e\| return e,
	2264	mf.memory_map.write(io) catch \|err\| {
	2265	mf.io_err = switch (err) {
	2266	error.Canceled => \|e\| return e,
	2267	error.WouldBlock => error.Unexpected, // file was not opened as non-blocking
	2268	error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing
	2269	else => \|e\| e,
	2270	};
	2271	return error.MappedFileIo;
1352	2272	};
1353	2273	unmap(mf);
1354	2274	}
1355	2275
1356	2276	const file = mf.memory_map.file;
1357		mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch \|err\| switch (err) {
1358		error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking
1359		error.NotOpenForReading => return error.Unexpected, // we definitely opened the file for writing
1360		else => \|e\| return e,
	2277	mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch \|err\| {
	2278	mf.io_err = switch (err) {
	2279	error.OutOfMemory, error.Canceled => \|e\| return e,
	2280	error.WouldBlock => error.Unexpected, // file was not opened as non-blocking
	2281	error.NotOpenForReading => error.Unexpected, // we definitely opened the file for writing
	2282	else => \|e\| e,
	2283	};
	2284	return error.MappedFileIo;
1361	2285	};
1362	2286	}
1363	2287
...	...	@@ -1376,7 +2300,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable \|\| error{MappedFileIo})!void {
1376	2300
1377	2301	error.WouldBlock, // file was not opened as non-blocking
1378	2302	error.NotOpenForWriting, // we definitely opened the file for writing
1379		error.ReadOnlyFileSystem,
	2303	error.ReadOnlyFileSystem, // again, we opened the file for writing
1380	2304	=> {
1381	2305	mf.io_err = error.Unexpected;
1382	2306	return error.MappedFileIo;
...	...	@@ -1399,213 +2323,278 @@ fn verify(mf: *MappedFile) void {
1399	2323	assert(root.parent == .none);
1400	2324	assert(root.prev == .none);
1401	2325	assert(root.next == .none);
1402		mf.verifyNode(Node.Index.root);
	2326	mf.verifyNode(.root);
1403	2327	}
1404
1405	2328	fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void {
1406	2329	const parent = parent_ni.get(mf);
1407		const parent_offset, const parent_size = parent.location().resolve(mf);
1408		var prev_ni: Node.Index = .none;
	2330	_, const parent_size = parent.location().resolve(mf);
	2331
	2332	var prev_oni: Node.Index.Optional = .none;
1409	2333	var prev_end: u64 = 0;
1410		var ni = parent.first;
1411		while (true) {
1412		if (ni == .none) {
1413		assert(parent.last == prev_ni);
1414		return;
1415		}
	2334	var prev_pos: Node.Position = .header;
	2335	var oni = parent.first;
	2336	while (oni.unwrap()) \|ni\| {
1416	2337	const node = ni.get(mf);
1417		assert(node.parent == parent_ni);
	2338	assert(node.parent == parent_ni.toOptional());
	2339	assert(node.prev == prev_oni);
	2340
1418	2341	const offset, const size = node.location().resolve(mf);
1419		assert(node.flags.alignment.check(@intCast(offset)));
1420		assert(node.flags.alignment.check(@intCast(size)));
1421	2342	const end = offset + size;
1422		assert(end <= parent_offset + parent_size);
	2343
	2344	assert(node.flags.alignment.check(size));
1423	2345	assert(offset >= prev_end);
1424		assert(node.prev == prev_ni);
	2346	assert(end <= parent_size);
	2347
	2348	switch (node.flags.position) {
	2349	.header => {
	2350	assert(prev_pos == .header);
	2351	assert(offset == prev_end);
	2352	},
	2353	.floating => {
	2354	assert(prev_pos != .footer);
	2355	assert(node.flags.alignment.check(offset));
	2356	},
	2357	.footer => {
	2358	if (prev_pos == .footer) assert(offset == prev_end);
	2359	},
	2360	}
	2361
1425	2362	mf.verifyNode(ni);
1426		prev_ni = ni;
	2363
	2364	prev_oni = .wrap(ni);
1427	2365	prev_end = end;
1428		ni = node.next;
	2366	prev_pos = ni.position(mf);
	2367
	2368	oni = node.next;
	2369	}
	2370	assert(parent.last == prev_oni);
	2371	if (prev_pos == .footer) {
	2372	assert(prev_end == parent_size);
1429	2373	}
1430	2374	}
1431	2375
1432		const testing = std.testing;
1433		fn testVerifyContent(mf: *@This(), ni: Node.Index, value: u8, init_len: usize) !void {
1434		// Not using std.mem.allEqual, so we can get useful output
1435		const slice = ni.slice(mf);
1436		var buf: [256]u8 = undefined;
1437		@memset(buf[0..init_len], value);
1438		@memset(buf[init_len..], 0);
1439		try testing.expectEqualSlices(u8, buf[0..slice.len], slice);
	2376	test "fuzz node operations" {
	2377	try std.testing.fuzz({}, fuzzOneNodeOperations, .{});
1440	2378	}
	2379	fn fuzzOneNodeOperations(_: void, smith: *std.testing.Smith) anyerror!void {
	2380	const gpa = std.testing.allocator;
	2381	const io = std.testing.io;
1441	2382
1442		test {
1443		const gpa = testing.allocator;
1444
1445		var tmp_dir = testing.tmpDir(.{});
	2383	var tmp_dir = std.testing.tmpDir(.{});
1446	2384	defer tmp_dir.cleanup();
1447	2385
1448		var file = try tmp_dir.dir.createFile(testing.io, "test.mf", .{ .read = true });
1449		defer file.close(testing.io);
	2386	var tmp_file = try tmp_dir.dir.createFile(io, "test.mf", .{ .read = true });
	2387	defer tmp_file.close(io);
1450	2388
1451		var mf = try init(file, gpa, testing.io);
	2389	var mf: MappedFile = try .init(tmp_file, gpa, io);
1452	2390	defer mf.deinit(gpa);
1453	2391
1454		const a = try mf.addFirstChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" });
1455		const c = try mf.addLastChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" });
1456		const b = try mf.addNodeAfter(gpa, a, .{ .fixed = true, .alignment = .@"16" });
1457		const d = try mf.addNodeAfter(gpa, b, .{ .alignment = .@"4" });
	2392	var nodes: std.array_hash_map.Auto(MappedFile.Node.Index, struct {
	2393	parent: MappedFile.Node.Index.Optional,
	2394	position: MappedFile.Node.Position,
	2395	num_headers: u32,
	2396	num_footers: u32,
	2397	/// For leaf nodes, this value is whether we have initialized the contents of the node or
	2398	/// not. For non-leaf nodes, this value is unspecified and should be ignored.
	2399	initialized: bool,
	2400	}) = .empty;
	2401	defer nodes.deinit(gpa);
	2402
	2403	// When initializing a leaf node, we will place its 4-byte node index at the start of its range,
	2404	// and the bitwise NOT of its node index at the end of its range (both little-endian). This is
	2405	// just a simple way to put distinct values we can validate at all node boundaries.
	2406
	2407	try nodes.putNoClobber(gpa, .root, .{
	2408	.parent = .none,
	2409	.position = .floating,
	2410	.num_headers = 0,
	2411	.num_footers = 0,
	2412	.initialized = false,
	2413	});
	2414
	2415	// Allow a range of alignments, with most nodes having a small alignment of 1--32 bytes (most
	2416	// commonly 1 byte), but with a small chance for some large alignments too.
	2417	const alignment_weights: []const std.testing.Smith.Weight = comptime &.{
	2418	.value(Alignment, .@"1", 20),
	2419	.value(Alignment, .@"2", 5),
	2420	.value(Alignment, .@"4", 5),
	2421	.value(Alignment, .@"8", 5),
	2422	.value(Alignment, .@"16", 5),
	2423	.value(Alignment, .@"32", 5),
	2424	.value(Alignment, .fromByteUnits(0x200), 1),
	2425	.value(Alignment, .fromByteUnits(0x400), 1),
	2426	.value(Alignment, .fromByteUnits(0x800), 1),
	2427	.value(Alignment, .fromByteUnits(0x1000), 1),
	2428	.value(Alignment, .fromByteUnits(0x2000), 1),
	2429	.value(Alignment, .fromByteUnits(0x4000), 1),
	2430	.value(Alignment, .fromByteUnits(0x8000), 1),
	2431	};
1458	2432
1459		const a_init_size = 8;
1460		const b_init_size = 16;
1461		const c_init_size = 24;
1462		const d_init_size = 28;
	2433	const min_nonzero_size = 2 * @sizeOf(MappedFile.Node.Index);
	2434	const max_size = 0x10_000;
	2435	const initial_size_weights: []const std.testing.Smith.Weight = comptime &.{
	2436	// initially, make nodes just as likely to be empty as non-empty
	2437	.value(u64, 0, max_size - min_nonzero_size + 1),
	2438	.rangeAtMost(u64, min_nonzero_size, max_size, 1),
	2439	};
1463	2440
1464		// Resize without content
1465		{
1466		// Verify size is aligned forward
1467		try d.resize(&mf, gpa, d_init_size - 1);
1468		try a.resize(&mf, gpa, a_init_size - 2);
1469		try c.resize(&mf, gpa, c_init_size);
1470		try b.resize(&mf, gpa, b_init_size);
1471		mf.verify();
1472
1473		const a_loc, const a_size = a.location(&mf).resolve(&mf);
1474		const b_loc, const b_size = b.location(&mf).resolve(&mf);
1475		const c_loc, const c_size = c.location(&mf).resolve(&mf);
1476		_, const d_size = d.location(&mf).resolve(&mf);
1477		try testing.expect(a_size >= a_init_size);
1478		try testing.expect(b_size >= b_init_size);
1479		try testing.expect(c_size >= c_init_size);
1480		try testing.expect(d_size >= d_init_size);
1481		try testing.expect(b_loc >= a_loc + a_size);
1482		try testing.expect(c_loc >= b_loc + b_size);
1483		}
	2441	while (!smith.eos()) switch (smith.value(enum { add, resize, realign })) {
	2442	.add => {
	2443	const parent_ni = nodes.keys()[smith.index(nodes.count())];
1484	2444
1485		const a_exp_size = 24;
1486		const b_exp_size = 28;
1487		const c_exp_size = 48;
1488		const d_exp_size = 32;
	2445	const alignment = smith.valueWeighted(Alignment, alignment_weights);
	2446	const size = alignment.forward(smith.valueWeighted(u64, initial_size_weights));
1489	2447
1490		// Resize with content
1491		{
1492		@memset(a.slice(&mf)[0..a_init_size], 0xaa);
1493		@memset(b.slice(&mf)[0..b_init_size], 0xbb);
1494		@memset(c.slice(&mf)[0..c_init_size], 0xcc);
1495		@memset(d.slice(&mf)[0..d_init_size], 0xdd);
1496
1497		try a.resize(&mf, gpa, a_exp_size);
1498		try b.resize(&mf, gpa, b_exp_size);
1499		try c.resize(&mf, gpa, c_exp_size);
1500		try d.resize(&mf, gpa, d_exp_size);
1501		mf.verify();
1502
1503		const a_loc, const a_size = a.location(&mf).resolve(&mf);
1504		const b_loc, const b_size = b.location(&mf).resolve(&mf);
1505		const c_loc, const c_size = c.location(&mf).resolve(&mf);
1506		_, const d_size = d.location(&mf).resolve(&mf);
1507		try testing.expect(a_size >= a_exp_size);
1508		try testing.expect(b_size >= b_exp_size);
1509		try testing.expect(c_size >= c_exp_size);
1510		try testing.expect(d_size >= d_exp_size);
1511		try testing.expect(b_loc >= a_loc + a_size);
1512		try testing.expect(c_loc >= b_loc + b_size);
1513
1514		try testVerifyContent(&mf, a, 0xaa, a_init_size);
1515		try testVerifyContent(&mf, b, 0xbb, b_init_size);
1516		try testVerifyContent(&mf, c, 0xcc, c_init_size);
1517		try testVerifyContent(&mf, d, 0xdd, d_init_size);
1518		}
	2448	const position = smith.valueWeighted(Node.Position, comptime &.{
	2449	// make floating nodes more common than header and footer nodes
	2450	.value(Node.Position, .header, 1),
	2451	.value(Node.Position, .footer, 1),
	2452	.value(Node.Position, .floating, 4),
	2453	});
	2454	const new_ni: Node.Index = switch (position) {
	2455	.header => new_ni: {
	2456	const parent_info = nodes.getPtr(parent_ni).?;
	2457	const prev_oni: Node.Index.Optional = prev_oni: {
	2458	const n = smith.valueRangeAtMost(u32, 0, parent_info.num_headers);
	2459	if (n == 0) break :prev_oni .none;
	2460	var cur_ni = parent_ni.first(&mf).unwrap().?;
	2461	for (1..n) \|_\| cur_ni = cur_ni.next(&mf).unwrap().?;
	2462	break :prev_oni .wrap(cur_ni);
	2463	};
	2464	const new_ni = try parent_ni.addHeaderChildAfter(&mf, gpa, prev_oni, .{
	2465	.size = size,
	2466	.alignment = alignment,
	2467	});
	2468	parent_info.num_headers += 1;
	2469	break :new_ni new_ni;
	2470	},
1519	2471
1520		const child_init: []const struct { std.mem.Alignment, usize } = &.{
1521		.{ .@"16", 16 },
1522		.{ .@"1", 1 },
1523		.{ .@"1", 19 },
1524		.{ .@"1", 3 },
1525		.{ .@"8", 30 },
1526		.{ .@"2", 5 },
1527		.{ .@"1", 60 },
1528		.{ .@"2", 2 },
1529		.{ .@"16", 32 },
1530		};
	2472	.floating => try parent_ni.addFloatingChild(&mf, gpa, .{
	2473	.size = size,
	2474	.alignment = alignment,
	2475	}),
	2476
	2477	.footer => new_ni: {
	2478	const parent_info = nodes.getPtr(parent_ni).?;
	2479	const next_oni: Node.Index.Optional = next_oni: {
	2480	const n = smith.valueRangeAtMost(u32, 0, parent_info.num_footers);
	2481	if (n == 0) break :next_oni .none;
	2482	var cur_ni = parent_ni.last(&mf).unwrap().?;
	2483	for (1..n) \|_\| cur_ni = cur_ni.prev(&mf).unwrap().?;
	2484	break :next_oni .wrap(cur_ni);
	2485	};
	2486	const new_ni = try parent_ni.addFooterChildBefore(&mf, gpa, next_oni, .{
	2487	.size = size,
	2488	.alignment = alignment,
	2489	});
	2490	parent_info.num_footers += 1;
	2491	break :new_ni new_ni;
	2492	},
	2493	};
1531	2494
1532		var children: [child_init.len]Node.Index = undefined;
	2495	const initialize = size > 0 and smith.value(bool);
	2496	if (initialize) {
	2497	const slice = new_ni.slice(&mf);
	2498	std.mem.writeInt(u32, slice[0..4], @backingInt(new_ni), .little);
	2499	std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(new_ni), .little);
	2500	}
1533	2501
1534		// Differently-aligned fixed sibling nodes
1535		{
1536		for (children[0 .. children.len - 1], child_init[0 .. children.len - 1], 0..) \|*ni, opts, i\| {
1537		ni.* = try mf.addLastChildNode(gpa, b, .{
1538		.alignment = opts.@"0",
1539		.size = opts.@"1",
1540		.fixed = true,
	2502	try nodes.putNoClobber(gpa, new_ni, .{
	2503	.parent = .wrap(parent_ni),
	2504	.position = position,
	2505	.num_headers = 0,
	2506	.num_footers = 0,
	2507	.initialized = initialize,
1541	2508	});
	2509	},
1542	2510
1543		@memset(ni.slice(&mf)[0..opts.@"1"], @intCast(i + 1));
1544		}
1545		// Shift differently-aligned nodes by inserting a node
1546		children[children.len - 1] = try mf.addNodeAfter(gpa, children[3], .{
1547		.alignment = child_init[children.len - 1].@"0",
1548		.size = child_init[children.len - 1].@"1",
1549		.fixed = true,
1550		});
1551		@memset(children[children.len - 1].slice(&mf), @intCast(children.len));
1552
1553		mf.verify();
1554		for (children, child_init, 0..) \|ni, opts, i\| {
1555		try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1");
1556		}
1557		}
	2511	.resize => {
	2512	const ni = nodes.keys()[smith.index(nodes.count())];
	2513	const node_info = nodes.getPtr(ni).?;
1558	2514
1559		// Shifting child nodes forward due via resize of parent.prev
1560		{
1561		try testing.expect(a.location(&mf).resolve(&mf)[1] < 64);
1562		try a.resize(&mf, gpa, 64);
	2515	const alignment = ni.alignment(&mf);
1563	2516
1564		try testVerifyContent(&mf, a, 0xaa, a_init_size);
1565		try testVerifyContent(&mf, c, 0xcc, c_init_size);
1566		try testVerifyContent(&mf, d, 0xdd, d_init_size);
1567		for (children, child_init, 0..) \|ni, opts, i\| {
1568		try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1");
1569		}
1570		}
	2517	if (ni.first(&mf) == .none and smith.value(bool)) {
	2518	// Since this is a leaf node, we can use `resizeLeaf`.
	2519	const new_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights));
	2520	try ni.resizeLeaf(&mf, gpa, new_size);
	2521	if (new_size == 0) {
	2522	node_info.initialized = false;
	2523	}
	2524	} else {
	2525	const min_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights));
	2526	try ni.ensureMinimumSize(&mf, gpa, min_size);
	2527	}
1571	2528
1572		// Re-align last node into trailing free space within parent
1573		{
1574		try b.resize(&mf, gpa, b.location(&mf).resolve(&mf)[1] + 64);
	2529	if (ni.first(&mf) == .none) {
	2530	// This is a leaf node, so it can contain data.
	2531	if (node_info.initialized) {
	2532	// It's already initialized, so we'll write the expected footer at the new end.
	2533	const slice = ni.slice(&mf);
	2534	std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little);
	2535	} else if (ni.location(&mf).resolve(&mf)[1] > 0) {
	2536	// It was uninitialized, but it has a non-zero size, so maybe we'd like to
	2537	// initialize it now?
	2538	if (smith.value(bool)) {
	2539	node_info.initialized = true;
	2540	const slice = ni.slice(&mf);
	2541	std.mem.writeInt(u32, slice[0..4], @backingInt(ni), .little);
	2542	std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little);
	2543	}
	2544	}
	2545	}
	2546	},
	2547	.realign => {
	2548	const ni = nodes.keys()[smith.index(nodes.count())];
	2549	const new_alignment = smith.valueWeighted(Alignment, alignment_weights);
	2550	if (new_alignment.compare(.gt, ni.alignment(&mf))) {
	2551	_, const old_size = ni.location(&mf).resolve(&mf);
	2552	try ni.realign(&mf, gpa, new_alignment);
	2553	if (ni.first(&mf) == .none and nodes.get(ni).?.initialized) {
	2554	const slice = ni.slice(&mf);
	2555	@memmove(slice[slice.len - 4 ..][0..4], slice[old_size - 4 ..][0..4]);
	2556	}
	2557	}
	2558	},
	2559	};
1575	2560
1576		const last = children[children.len - 2];
1577		try last.realign(&mf, gpa, .@"4", true);
1578		mf.verify();
	2561	mf.verify();
1579	2562
1580		for (children, child_init, 0..) \|ni, opts, i\|
1581		try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1");
1582		try testVerifyContent(&mf, c, 0xcc, c_init_size);
1583		}
	2563	for (nodes.keys(), nodes.values()) \|ni, expected\| {
	2564	try std.testing.expectEqual(expected.parent, ni.parent(&mf));
	2565	if (ni != .root) {
	2566	try std.testing.expectEqual(expected.position, ni.position(&mf));
	2567	}
1584	2568
1585		// Re-align, shifting sibling nodes
1586		{
1587		try children[1].realign(&mf, gpa, .@"8", true);
1588		mf.verify();
	2569	{
	2570	var num_headers: u32 = 0;
	2571	var header_oni = ni.lastHeader(&mf);
	2572	while (header_oni.unwrap()) \|header_ni\| {
	2573	num_headers += 1;
	2574	header_oni = header_ni.prev(&mf);
	2575	}
	2576	try std.testing.expectEqual(expected.num_headers, num_headers);
	2577	}
1589	2578
1590		for (children, child_init, 0..) \|ni, opts, i\|
1591		try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1");
1592		try testVerifyContent(&mf, c, 0xcc, c_init_size);
1593		}
	2579	{
	2580	var num_footers: u32 = 0;
	2581	var footer_oni = ni.firstFooter(&mf);
	2582	while (footer_oni.unwrap()) \|footer_ni\| {
	2583	num_footers += 1;
	2584	footer_oni = footer_ni.next(&mf);
	2585	}
	2586	try std.testing.expectEqual(expected.num_footers, num_footers);
	2587	}
1594	2588
1595		// Shrink and shift start of trailing node into free space
1596		{
1597		try mf.shrinkNode(gpa, a, 16, true);
1598		mf.verify();
1599
1600		const a_loc, const a_size = a.location(&mf).resolve(&mf);
1601		const b_loc, _ = b.location(&mf).resolve(&mf);
1602		try testing.expectEqual(b_loc, a_loc + a_size);
1603
1604		try testVerifyContent(&mf, a, 0xaa, a_init_size);
1605		try testVerifyContent(&mf, c, 0xcc, c_init_size);
1606		try testVerifyContent(&mf, d, 0xdd, d_init_size);
1607		for (children, child_init, 0..) \|ni, opts, i\| {
1608		try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1");
	2589	if (ni.first(&mf) == .none and expected.initialized) {
	2590	const slice = ni.sliceConst(&mf);
	2591	if (slice.len > 0) {
	2592	try std.testing.expect(slice.len >= min_nonzero_size);
	2593	const header = std.mem.readInt(u32, slice[0..4], .little);
	2594	const footer = std.mem.readInt(u32, slice[slice.len - 4 ..][0..4], .little);
	2595	try std.testing.expectEqual(@backingInt(ni), header);
	2596	try std.testing.expectEqual(~@backingInt(ni), footer);
	2597	}
1609	2598	}
1610	2599	}
1611	2600	}

src/main.zig+1

...	...	@@ -36,6 +36,7 @@ const Module = @import("Module.zig");
36	36
37	37	test {
38	38	_ = @import("codegen.zig");
	39	_ = @import("link/MappedFile.zig");
39	40	}
40	41
41	42	const thread_stack_size = 60 << 20;