From ff85396f7a85750cb703460b5b57b9860088c5ef Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sat, 22 Aug 2026 15:47:44 +0100 Subject: [PATCH 1/6] test runner: initialize `std.testing.io_instance` in fuzz tests Because this was left at `undefined`, fuzz tests were exhibiting Illegal Behavior when they used `std.testing.io`. This wasn't noticed sooner probably because an all-zeroes `std.Io.Threaded` happens to be fairly functional, so things would broadly work if the optimizer didn't catch the IB. --- lib/compiler/test_runner.zig | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 9ecfa4d03951427c9b091f6e59d50c120d4ee7f1..0438a8d469f3678cedca64033aa1146886d98556 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -185,7 +185,6 @@ fn mainServer(init: std.process.Init.Minimal) !void { .environ = init.environ, }); defer io_instance.deinit(); - const io = io_instance.io(); const mode: fuzz_abi.LimitKind = @fromBackingInt(@intCast(try server.receiveBody_u8())); const amount_or_instance = try server.receiveBody_u64(); @@ -208,7 +207,7 @@ fn mainServer(init: std.process.Init.Minimal) !void { .indexes = test_indexes, .server = &server, .gpa = gpa, - .io = io, + .threaded_io = &io_instance, .input_poller = undefined, }; @@ -422,7 +421,7 @@ var fuzz_runner: if (builtin.fuzz) struct { indexes: []u32, server: *std.zig.Server, gpa: std.mem.Allocator, - io: Io, + threaded_io: *Io.Threaded, input_poller: Io.Future(Io.Cancelable!void), comptime { @@ -443,6 +442,12 @@ var fuzz_runner: if (builtin.fuzz) struct { defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1); is_fuzz_test = false; + testing.io_instance = .init(testing.allocator, .{ + .argv0 = fuzz_runner.threaded_io.argv0, + .environ = fuzz_runner.threaded_io.environ.process_environ, + }); + defer testing.io_instance.deinit(); + builtin.test_functions[fuzz_runner.indexes[i]].func() catch |err| switch (err) { error.SkipZigTest => return, else => { @@ -473,7 +478,8 @@ var fuzz_runner: if (builtin.fuzz) struct { export fn runner_start_input_poller() void { @disableInstrumentation(); - const future = fuzz_runner.io.concurrent(inputPoller, .{}) catch |e| switch (e) { + const io = fuzz_runner.threaded_io.io(); + const future = io.concurrent(inputPoller, .{}) catch |e| switch (e) { error.ConcurrencyUnavailable => @panic("failed to spawn concurrent fuzz input poller"), }; fuzz_runner.input_poller = future; @@ -481,17 +487,20 @@ var fuzz_runner: if (builtin.fuzz) struct { export fn runner_stop_input_poller() void { @disableInstrumentation(); - assert(fuzz_runner.input_poller.cancel(fuzz_runner.io) == error.Canceled); + const io = fuzz_runner.threaded_io.io(); + assert(fuzz_runner.input_poller.cancel(io) == error.Canceled); } export fn runner_futex_wait(ptr: *const u32, expected: u32) bool { @disableInstrumentation(); - return fuzz_runner.io.futexWait(u32, ptr, expected) == error.Canceled; + const io = fuzz_runner.threaded_io.io(); + return io.futexWait(u32, ptr, expected) == error.Canceled; } export fn runner_futex_wake(ptr: *const u32, waiters: u32) void { @disableInstrumentation(); - fuzz_runner.io.futexWake(u32, ptr, waiters); + const io = fuzz_runner.threaded_io.io(); + io.futexWake(u32, ptr, waiters); } fn inputPoller() Io.Cancelable!void { -- 2.54.0 From d9078dae3b6266767d66d5f2100f321b200cbd6b Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 17 Aug 2026 10:59:01 +0100 Subject: [PATCH 2/6] link.MappedFile: new `Alignment` type and non-optional `Node.Index` There are two refactors here (apologies for putting them in the same commit!). First, I have replaced uses of `std.mem.Alignment` with a new type based on a fixed `u64` address space. While it is technically okay to use `std.mem.Alignment` in `MappedFile` (because memory-mapping limits the file size to the host's address space size), in practice it is somewhat inconvenient. I wanted to use `InternPool.Alignment`, but that type has an annoying problem of its own: for legacy reasons, it is optional (that is, it has a `.none` field), which makes for very ambiguous APIs unless you meticulously assert and comment all uses of the type. I therefore chose to add yet another alignment type to the Zig repository---sorrry! My hope going forward is that at some point, we can rename the existing `InternPool.Alignment` type to `InternPool.Alignment.Optional`, rename this new type to `InternPool.Alignment`, and slowly transition the entire compiler towards correctly distinguishing between "optional" and "non-optional" alignments. Second, I have made `MappedFile.Node.Index` non-optional (i.e. removed its `.none` tag). Notably, the old definition of this type had `.root == .none`, which was pretty awkward (you couldn't represent a node index which could be the root node *and* could be empty) and unsafe (we couldn't get safety checks for trying to use a "null" node index, instead we would just operate on the root node). To fix this, it has been split into `Node.Index` and `Node.Index.Optional`---I'm sure you all know the drill by now, it's just like all of the index types in `InternPool`. Some of the code I've written in this migration is definitely quite ugly, because I did a fairly mechanical replacement (e.g. for the most part I didn't introduce local constants). The code can be neatened up to avoid the mess of `unwrap` calls all over the place! Also, in `link.Coff`, it's possible that I made some fields optional when they shouldn't have been, which would definitely contribute to the `.unwrap().?` mess I wrote in that linker... --- src/InternPool.zig | 11 - src/link/Coff.zig | 286 ++++++++++++----------- src/link/Elf2.zig | 369 +++++++++++++++--------------- src/link/MappedFile.zig | 487 ++++++++++++++++++++++++---------------- 4 files changed, 623 insertions(+), 530 deletions(-) diff --git a/src/InternPool.zig b/src/InternPool.zig index 96af077d99935e56e6290ca47b544b6e6bd076fc..94375c3f0ca7430d890588979d9409415c2054a6 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -5987,17 +5987,6 @@ pub const Alignment = enum(u6) { return n + 1; } - pub fn toStdMem(a: Alignment) std.mem.Alignment { - assert(a != .none); - return @fromBackingInt(@intCast(@backingInt(a))); - } - - pub fn fromStdMem(a: std.mem.Alignment) Alignment { - const r: Alignment = @fromBackingInt(@intCast(@backingInt(a))); - assert(r != .none); - return r; - } - pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment { return @fromBackingInt(@intCast(@backingInt(a))); } diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 7303c030011bd8935595353982ca2458b93ecae8..f83074f558b19e84d803d17251a6d4014abdd2b2 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -21,6 +21,7 @@ const Zcu = @import("../Zcu.zig"); const ModuleDefinition = @import("../libs/mingw/def.zig").ModuleDefinition; const implib = @import("../libs/mingw/implib.zig"); const Path = std.Build.Cache.Path; +const Alignment = MappedFile.Alignment; base: link.File, options: link.File.OpenOptions, @@ -602,7 +603,7 @@ pub const Member = struct { }; pub const LongNamesTable = struct { - ni: MappedFile.Node.Index = .none, + ni: MappedFile.Node.Index.Optional = .none, entries: std.array_hash_map.Auto(void, Entry), pub const Entry = struct { @@ -832,7 +833,7 @@ pub const String = enum(u32) { pub const Section = struct { si: Symbol.Index, - relocation_table_ni: MappedFile.Node.Index, + relocation_table_ni: MappedFile.Node.Index.Optional, pub const RelocationIndex = enum(u16) { none, @@ -855,7 +856,7 @@ pub const Section = struct { sn: Symbol.SectionNumber, ) ?*align(2) std.coff.Relocation { if (sri == .none) return null; - const table_slice = sn.section(coff).relocation_table_ni.slice(&coff.mf); + const table_slice = sn.section(coff).relocation_table_ni.unwrap().?.slice(&coff.mf); return @ptrCast(@alignCast(&table_slice[@as(u32, sri.unwrap().?) * std.coff.Relocation.sizeOf()])); } }; @@ -891,7 +892,7 @@ const SpecialSymbol = enum { }; pub const Symbol = struct { - ni: MappedFile.Node.Index, + ni: MappedFile.Node.Index.Optional, rva: u32, value: std.meta.BareUnion(Symbol.Value), extra: std.meta.BareUnion(Symbol.Extra), @@ -986,7 +987,7 @@ pub const Symbol = struct { pub fn nodeOffset(sym: *const Symbol, coff: *Coff) u32 { return switch (sym.flags.value_tag) { .node_offset => offset: { - assert(switch (coff.getNode(sym.ni)) { + assert(switch (coff.getNode(sym.ni.unwrap().?)) { // Separate nodes are not created for these entries per-symbol .input_section, .import_address_table => true, else => false, @@ -1052,9 +1053,7 @@ pub const Symbol = struct { } pub fn node(si: Symbol.Index, coff: *Coff) MappedFile.Node.Index { - const ni = si.get(coff).ni; - assert(ni != .none); - return ni; + return si.get(coff).ni.unwrap().?; } pub fn sti(si: Symbol.Index, coff: *Coff) SymbolTable.Index { @@ -1075,7 +1074,7 @@ pub const Symbol = struct { pub fn flushMoved(si: Symbol.Index, coff: *Coff) !void { const sym = si.get(coff); - sym.rva = coff.computeNodeRva(sym.ni) + sym.nodeOffset(coff); + sym.rva = coff.computeNodeRva(sym.ni.unwrap().?) + sym.nodeOffset(coff); try si.applyLocationRelocs(coff); try si.applyTargetRelocs(coff, .none); @@ -1199,12 +1198,11 @@ pub const Reloc = extern struct { pub fn apply(reloc: *Reloc, coff: *Coff) !void { const loc_sym = reloc.loc.get(coff); - switch (loc_sym.ni) { - .none => return, - else => |ni| if (ni.hasMoved(&coff.mf)) return, - } - const loc_slice = loc_sym.ni.slice(&coff.mf)[@intCast(reloc.offset)..]; + const loc_sym_ni = loc_sym.ni.unwrap() orelse return; + if (loc_sym_ni.hasMoved(&coff.mf)) return; + + const loc_slice = loc_sym_ni.slice(&coff.mf)[@intCast(reloc.offset)..]; const target_endian = coff.targetEndian(); const target_machine = coff.targetLoad(&coff.headerPtr().machine); @@ -1331,9 +1329,12 @@ pub const Reloc = extern struct { } const target_sym = reloc.target.get(coff); - const is_abs = switch (target_sym.ni) { - .none => if (target_sym.section_number == .ABSOLUTE) true else return, - else => |ni| if (ni.hasMoved(&coff.mf)) return else false, + const is_abs = if (target_sym.ni.unwrap()) |ni| is_abs: { + if (ni.hasMoved(&coff.mf)) return; + break :is_abs false; + } else is_abs: { + if (target_sym.section_number != .ABSOLUTE) return; + break :is_abs true; }; const target_rva = target_sym.rva +% @as(u64, @bitCast(reloc.addend)); @@ -1573,7 +1574,7 @@ fn create( 33...64 => .@"PE32+", else => return error.UnsupportedCOFFArchitecture, }; - const section_align: std.mem.Alignment = switch (machine) { + const section_align: Alignment = switch (machine) { .AMD64, .I386 => @fromBackingInt(@intCast(12)), .SH3, .SH3DSP, .SH4, .SH5 => @fromBackingInt(@intCast(12)), .MIPS16, .MIPSFPU, .MIPSFPU16, .WCEMIPSV2 => @fromBackingInt(@intCast(12)), @@ -1617,22 +1618,22 @@ fn create( .entries = .empty, }, .import_table = .{ - .ni = .none, + .ni = undefined, .entries = .empty, .iat_symbol_indices = .empty, }, .export_table = .{ - .ni = .none, - .export_directory_table_ni = .none, + .ni = undefined, + .export_directory_table_ni = undefined, .export_address_table_si = .null, - .name_pointer_table_ni = .none, - .ordinal_table_ni = .none, - .name_table_ni = .none, + .name_pointer_table_ni = undefined, + .ordinal_table_ni = undefined, + .name_table_ni = undefined, .entries = .empty, }, .symbol_table = .{ - .ni = .none, - .strings_ni = .none, + .ni = undefined, + .strings_ni = undefined, .strings = .empty, .symbols = .empty, .pending_symbol_index = 0, @@ -1794,13 +1795,13 @@ fn initHeaders( minor_subsystem_version: u16, magic: std.coff.OptionalHeader.Magic, subsystem: std.coff.Subsystem, - section_align: std.mem.Alignment, + section_align: Alignment, file_name: []const u8, ) !void { const comp = coff.base.comp; const gpa = comp.gpa; const target_endian = coff.targetEndian(); - const file_align: std.mem.Alignment = comptime .fromByteUnits(default_file_alignment); + const file_align: Alignment = comptime .fromByteUnits(default_file_alignment); const is_image = coff.isImage(); const is_archive = coff.isArchive(); const target = &comp.root_mod.resolved_target.result; @@ -2191,7 +2192,7 @@ fn initHeaders( coff.export_table.export_address_table_si = coff.addSymbolAssumeCapacity(); const export_address_table_sym = coff.export_table.export_address_table_si.get(coff); - export_address_table_sym.ni = export_address_table_ni; + export_address_table_sym.ni = .wrap(export_address_table_ni); assert(export_address_table_sym.loc_relocs == .none); export_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); export_address_table_sym.section_number = @@ -2260,7 +2261,7 @@ pub fn initBuiltins(coff: *Coff) !void { if (coff.isImage()) { const si = try coff.globalSymbol(.{ .name = "__ImageBase", .type = .data }); const sym = si.get(coff); - sym.ni = Node.known.header; + sym.ni = .wrap(Node.known.header); } defer coff.flushSectionMerges() catch unreachable; @@ -2302,14 +2303,14 @@ pub fn initBuiltins(coff: *Coff) !void { const list_len_si = try coff.globalSymbol(.{ .name = list.global, .type = .data }); const list_len_sym = list_len_si.get(coff); list_len_sym.setExtra(.{ .size = addr_info.size }); - list_len_sym.ni = try coff.mf.addFirstChildNode(gpa, start_sym.ni, .{ + list_len_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, start_sym.ni.unwrap().?, .{ .size = addr_info.size, .fixed = true, - }); + })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_len_si }); list_len_sym.section_number = start_sym.section_number; - const start_slice = list_len_sym.ni.slice(&coff.mf); + const start_slice = list_len_sym.ni.unwrap().?.slice(&coff.mf); switch (addr_info.magic) { _ => unreachable, inline .PE32, .@"PE32+" => |t| { @@ -2324,14 +2325,14 @@ pub fn initBuiltins(coff: *Coff) !void { const list_end_si = coff.addSymbolAssumeCapacity(); const list_end_sym = list_end_si.get(coff); list_end_sym.setExtra(.{ .size = addr_info.size }); - list_end_sym.ni = try coff.mf.addFirstChildNode(gpa, end_sym.ni, .{ + list_end_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, end_sym.ni.unwrap().?, .{ .size = addr_info.size, .fixed = true, - }); + })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_end_si }); list_end_sym.section_number = start_sym.section_number; - @memset(list_end_sym.ni.slice(&coff.mf), 0); + @memset(list_end_sym.ni.unwrap().?.slice(&coff.mf), 0); try list_len_si.flushMoved(coff); try list_end_si.flushMoved(coff); @@ -2387,7 +2388,7 @@ fn getNode(coff: *const Coff, ni: MappedFile.Node.Index) Node { } fn computeNodeRva(coff: *Coff, ni: MappedFile.Node.Index) u32 { const parent_rva = parent_rva: { - const parent_si = switch (coff.getNode(ni.parent(&coff.mf))) { + const parent_si = switch (coff.getNode(ni.parent(&coff.mf).unwrap().?)) { .file, .header, .signature, @@ -2452,11 +2453,11 @@ fn computeSymbolSectionOffset( relative_to: enum { image, pseudo }, ) u32 { var section_offset: u32 = sym.nodeOffset(coff); - var parent_ni = sym.ni; + var parent_ni = sym.ni.unwrap().?; while (true) { const offset, _ = parent_ni.location(&coff.mf).resolve(&coff.mf); section_offset += @intCast(offset); - parent_ni = parent_ni.parent(&coff.mf); + parent_ni = parent_ni.parent(&coff.mf).unwrap().?; switch (coff.getNode(parent_ni)) { else => unreachable, .image_section => break, @@ -2475,7 +2476,7 @@ pub inline fn targetEndian(_: *const Coff) std.lang.Endian { fn targetAddrInfo(coff: *Coff) struct { size: u8, - alignment: std.mem.Alignment, + alignment: Alignment, magic: std.coff.OptionalHeader.Magic, } { const magic = coff.targetLoad(&coff.optionalHeaderStandardPtr().magic); @@ -2875,9 +2876,9 @@ fn navSection( switch (nav_resolved.@"linksection") { .none => coff.mf.flags.block_size, else => switch (nav_resolved.@"align") { - .none => Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu), - else => |alignment| alignment, - }.toStdMem(), + .none => .fromIp(Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu)), + else => |a| .fromIp(a), + }, }, attributes, )).symbol(coff); @@ -3151,7 +3152,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { else .NULL, }; - } else blk: switch (coff.getNode(sym.ni)) { + } else blk: switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => .{ try coff.getOrPutSymbolName(&sym.section_number.header(coff).name, null), 1, @@ -3192,7 +3193,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { }; }, else => { - log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni)), si }); + log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni.unwrap().?)), si }); unreachable; }, }; @@ -3255,13 +3256,13 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { std.mem.byteSwapAllFieldsAligned(std.coff.WeakExternalDefinition, .@"2", aux_ptr); break :aux_init; - } else switch (coff.getNode(sym.ni)) { + } else switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => |sec_si| { assert(si == sec_si); const header = sym.section_number.header(coff); const aux_ptr = coff.symbolTableSectionAuxEntryPtr(sti.*).?; aux_ptr.* = .{ - .length = @intCast(sym.ni.location(&coff.mf).resolve(&coff.mf)[1]), + .length = @intCast(sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[1]), .number_of_relocations = header.number_of_relocations, .number_of_linenumbers = header.number_of_linenumbers, .checksum = 0, @@ -3288,7 +3289,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { .ABSOLUTE, .DEBUG, => unreachable, - else => switch (coff.getNode(sym.ni)) { + else => switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => 0, else => coff.computeSymbolSectionOffset(sym, .image), }, @@ -3397,7 +3398,7 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S { const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = rva; sym.section_number = @fromBackingInt(@intCast(section_table_len)); } @@ -3481,7 +3482,7 @@ const ObjectSectionAttributes = packed struct { fn pseudoSectionMapIndex( coff: *Coff, name: String, - alignment: std.mem.Alignment, + alignment: Alignment, attributes: ObjectSectionAttributes, ) !Node.PseudoSectionMapIndex { const gpa = coff.base.comp.gpa; @@ -3510,7 +3511,7 @@ fn pseudoSectionMapIndex( const si = coff.addSymbolAssumeCapacity(); pseudo_section_gop.value_ptr.* = si; const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = coff.computeNodeRva(ni); sym.section_number = parent.get(coff).section_number; assert(sym.loc_relocs == .none); @@ -3543,7 +3544,7 @@ fn objectSectionParentName(coff: *Coff, name: []const u8) []const u8 { fn objectSectionMapIndex( coff: *Coff, name: String, - alignment: std.mem.Alignment, + alignment: Alignment, attributes: ObjectSectionAttributes, ) !Node.ObjectSectionMapIndex { const gpa = coff.base.comp.gpa; @@ -3565,7 +3566,7 @@ fn objectSectionMapIndex( try coff.nodes.ensureUnusedCapacity(gpa, 1); try coff.symbols.ensureUnusedCapacity(gpa, 1); const parent_ni = parent.node(coff); - var prev_ni: MappedFile.Node.Index = .none; + var prev_oni: MappedFile.Node.Index.Optional = .none; var next_it = parent_ni.children(&coff.mf); while (next_it.next()) |next_ni| switch (std.mem.order( u8, @@ -3574,22 +3575,19 @@ fn objectSectionMapIndex( )) { .lt => break, .eq => unreachable, - .gt => prev_ni = next_ni, - }; - const ni = switch (prev_ni) { - .none => try coff.mf.addFirstChildNode(gpa, parent_ni, .{ - .alignment = alignment, - .fixed = true, - }), - else => try coff.mf.addNodeAfter(gpa, prev_ni, .{ - .alignment = alignment, - .fixed = true, - }), + .gt => prev_oni = .wrap(next_ni), }; + const ni = if (prev_oni.unwrap()) |prev_ni| try coff.mf.addNodeAfter(gpa, prev_ni, .{ + .alignment = alignment, + .fixed = true, + }) else try coff.mf.addFirstChildNode(gpa, parent_ni, .{ + .alignment = alignment, + .fixed = true, + }); const si = coff.addSymbolAssumeCapacity(); object_section_gop.value_ptr.* = si; const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = coff.computeNodeRva(ni); sym.section_number = parent.get(coff).section_number; assert(sym.loc_relocs == .none); @@ -3598,17 +3596,17 @@ fn objectSectionMapIndex( break :sym sym; } else object_section_gop.value_ptr.get(coff); - const parent_ni = sym.ni.parent(&coff.mf); + const parent_ni = sym.ni.unwrap().?.parent(&coff.mf).unwrap().?; const parent_alignment = parent_ni.alignment(&coff.mf); if (alignment.compare(.gt, parent_alignment)) { log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment }); try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); } - const old_alignment = sym.ni.alignment(&coff.mf); + const old_alignment = sym.ni.unwrap().?.alignment(&coff.mf); if (alignment.compare(.gt, old_alignment)) { log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment }); - try sym.ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); } try coff.verifyParentSectionAttributes( @@ -3764,8 +3762,10 @@ fn addRelocAssumeCapacity( if (coff.symbolTableSectionAuxEntryPtr(loc_sn.symbol(coff).sti(coff))) |aux_ptr| coff.targetStore(&aux_ptr.number_of_relocations, new_num_relocations); - if (section.relocation_table_ni == .none) { - section.relocation_table_ni = try coff.mf.addLastChildNode( + if (section.relocation_table_ni.unwrap()) |relocation_table_ni| { + try relocation_table_ni.resize(&coff.mf, gpa, new_size); + } else { + section.relocation_table_ni = .wrap(try coff.mf.addLastChildNode( gpa, coff.sectionParent(), .{ @@ -3774,10 +3774,8 @@ fn addRelocAssumeCapacity( .moved = true, .resized = true, }, - ); + )); coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn }); - } else { - try section.relocation_table_ni.resize(&coff.mf, gpa, new_size); } // TODO: These need to allocate from a free list, once deleting relocs from the table is supported @@ -4581,7 +4579,7 @@ fn loadObject( }, .SAME_SIZE => { // TODO: Verify that this node isn't resized after creation - _, const size = si.get(coff).ni.location(&coff.mf).resolve(&coff.mf); + _, const size = si.get(coff).ni.unwrap().?.location(&coff.mf).resolve(&coff.mf); if (size == section.header.size_of_raw_data) { symbol.si = si; break :comdat .skip; @@ -4598,9 +4596,9 @@ fn loadObject( }, .EXACT_MATCH => { const sym = si.get(coff); - const existing_crc = switch (coff.getNode(sym.ni)) { + const existing_crc = switch (coff.getNode(sym.ni.unwrap().?)) { .input_section => |isi| isi.inputSection(coff).crc, - else => Crc32.hash(sym.ni.sliceConst(&coff.mf)), + else => Crc32.hash(sym.ni.unwrap().?.sliceConst(&coff.mf)), }; if (existing_crc == section.comdat_crc) { @@ -4666,7 +4664,7 @@ fn loadObject( section.parent_si = (try coff.objectSectionMapIndex( section.name, - section.header.flags.ALIGN.alignment() orelse .@"1", + .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), .fromFlags(section.header.flags), )).symbol(coff); } @@ -4681,7 +4679,7 @@ fn loadObject( const ni = try coff.mf.addLastChildNode(gpa, section.parent_si.node(coff), .{ .size = section.header.size_of_raw_data, - .alignment = section.header.flags.ALIGN.alignment() orelse .@"1", + .alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(coff.input_sections.items.len)) }); @@ -4691,7 +4689,7 @@ fn loadObject( pending_symbols.values()[psi].si = section.si; const sym = section.si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = section.parent_si.get(coff).section_number; coff.input_sections.addOneAssumeCapacity().* = .{ @@ -4852,7 +4850,7 @@ fn loadObject( } if (section.comdat_psi.unwrap() == @as(u32, @intCast(i))) - coff.getNode(section.si.get(coff).ni).input_section.inputSection(coff).comdat_si = symbol.si; + coff.getNode(section.si.get(coff).ni.unwrap().?).input_section.inputSection(coff).comdat_si = symbol.si; } if (symbol.weak_external_psi.unwrap()) |weak_external_i| { @@ -4967,14 +4965,14 @@ fn loadObject( const section = §ions[symbol.section_number.toIndex()]; include_section = section.comdat_result == .include; if (include_section) { - const isi = coff.getNode(section.si.get(coff).ni).input_section; + const isi = coff.getNode(section.si.get(coff).ni.unwrap().?).input_section; isi.inputSection(coff).first_li = @fromBackingInt(@intCast(coff.input_symbols.items.len)); } } } if (include_section) { - assert(coff.getNode(symbol.si.get(coff).ni) == .input_section); + assert(coff.getNode(symbol.si.get(coff).ni.unwrap().?) == .input_section); symbol.si.get(coff).setExtra(.{ .isli = @fromBackingInt(@intCast(coff.input_symbols.items.len)) }); coff.input_symbols.addOneAssumeCapacity().* = .{ .si = symbol.si, @@ -5002,7 +5000,7 @@ fn failMultipleDefinitions( var err = try coff.base.comp.link_diags.addErrorWithNotes(num_notes); try err.addMsg("multiple definitions of '{s}'", .{name.toSlice(coff)}); - switch (coff.getNode(existing_si.get(coff).ni)) { + switch (coff.getNode(existing_si.get(coff).ni.unwrap().?)) { .input_section => |isi| { const other_ioi = isi.input(coff); err.addNote("first seen in input '{f}{f}'", .{ @@ -5474,12 +5472,12 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ - .alignment = zcu.navAlignment(nav_index).toStdMem(), + .alignment = .fromIp(zcu.navAlignment(nav_index)), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .nav = nmi }); const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -5490,7 +5488,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; { @@ -5515,7 +5513,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde try ni.resize(&coff.mf, gpa, si.get(coff).extra.size); var parent_ni = ni; while (true) { - parent_ni = parent_ni.parent(&coff.mf); + parent_ni = parent_ni.parent(&coff.mf).unwrap().?; switch (coff.getNode(parent_ni)) { else => unreachable, .image_section, .pseudo_section => break, @@ -5542,10 +5540,11 @@ pub fn lowerUav( try coff.pending_uavs.ensureUnusedCapacity(gpa, 1); const umi = try coff.uavMapIndex(uav_val); const si = umi.symbol(coff); - if (switch (si.get(coff).ni) { - .none => true, - else => |ni| uav_align.toStdMem().order(ni.alignment(&coff.mf)).compare(.gt), - }) { + const need_update: bool = update: { + const existing_ni = si.get(coff).ni.unwrap() orelse break :update true; + break :update Alignment.compare(.fromIp(uav_align), .gt, existing_ni.alignment(&coff.mf)); + }; + if (need_update) { const gop = coff.pending_uavs.getOrPutAssumeCapacity(umi); if (gop.found_existing) { gop.value_ptr.alignment = gop.value_ptr.alignment.max(uav_align); @@ -5603,16 +5602,16 @@ fn updateFuncInner( .debug, .safe, .fast, - => target_util.defaultFunctionAlignment(target), - .small => target_util.minFunctionAlignment(target), + => .fromIp(target_util.defaultFunctionAlignment(target)), + .small => .fromIp(target_util.minFunctionAlignment(target)), }, - else => |a| a.maxStrict(target_util.minFunctionAlignment(target)), - }.toStdMem(), + else => |a| .fromIp(a.maxStrict(target_util.minFunctionAlignment(target))), + }, .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .nav = nmi }); const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -5622,7 +5621,7 @@ fn updateFuncInner( sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var nw: MappedFile.Node.Writer = undefined; @@ -5662,7 +5661,6 @@ fn flushImplib( implib_file: []const u8, ) !void { // Emitting implibs is only valid for images - assert(coff.export_table.ni != .none); const comp = coff.base.comp; const gpa = comp.gpa; @@ -5797,7 +5795,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void { const loc_sym = loc_si.get(coff); // TODO: Make this a helper for anything that needs to report "referenced by" notes - switch (coff.getNode(loc_sym.ni)) { + switch (coff.getNode(loc_sym.ni.unwrap().?)) { .data_directories => { const dir: std.coff.IMAGE.DIRECTORY_ENTRY = @fromBackingInt(@intCast(reloc.offset / @sizeOf(std.coff.ImageDataDirectory))); @@ -5808,7 +5806,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void { const other_ioi = isi.input(coff); if (loc_sym.gmi == .none) { const section = isi.inputSection(coff); - const section_name = coff.getNode(loc_sym.ni.parent(&coff.mf)) + const section_name = coff.getNode(loc_sym.ni.unwrap().?.parent(&coff.mf).unwrap().?) .object_section.name(coff).toSlice(coff); if (section.comdat_si != .null) { @@ -6055,8 +6053,8 @@ fn resolve(coff: *Coff, tid: Zcu.PerThread.Id) !bool { const sub_prog_node = coff.idleProgNode( tid, coff.symbol_prog_node, - if (sym.ni != .none) - coff.getNode(sym.ni) + if (sym.ni.unwrap()) |sym_ni| + coff.getNode(sym_ni) else .{ .import_thunk = sym.gmi }, ); @@ -6173,7 +6171,7 @@ fn idleProgNode( break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{ ioi.path(coff).fmtEscapeString(), fmtMemberNameString(ioi.memberName(coff)), - coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff), + coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff), }) catch &name; }, .import_thunk => |gmi| gmi.name(coff).toSlice(coff), @@ -6214,16 +6212,21 @@ fn flushUav( if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const sym = si.get(coff); const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ - .alignment = uav_align.toStdMem(), + .alignment = .fromIp(uav_align), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .uav = umi }); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => { - if (si.get(coff).ni.alignment(&coff.mf).order(uav_align.toStdMem()).compare(.gte)) + if (Alignment.compare( + si.get(coff).ni.unwrap().?.alignment(&coff.mf), + .gte, + .fromIp(uav_align), + )) { return; + } si.deleteLocationRelocs(coff); }, } @@ -6233,7 +6236,7 @@ fn flushUav( if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var nw: MappedFile.Node.Writer = undefined; @@ -6497,7 +6500,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { lib_name, ImportTable.Adapter{ .coff = coff }, ); - const import_hint_name_align: std.mem.Alignment = .@"2"; + const import_hint_name_align: Alignment = .@"2"; if (!gop.found_existing) { errdefer _ = coff.import_table.entries.pop(); try coff.import_table.ni.resize( @@ -6507,7 +6510,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { ); const import_hint_name_table_len = import_hint_name_align.forward(lib_name.len + ".dll".len + 1); - const idata_section_ni = coff.import_table.ni.parent(&coff.mf); + const idata_section_ni = coff.import_table.ni.parent(&coff.mf).unwrap().?; const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, @@ -6521,7 +6524,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_address_table_si = coff.addSymbolAssumeCapacity(); { const import_address_table_sym = import_address_table_si.get(coff); - import_address_table_sym.ni = import_address_table_ni; + import_address_table_sym.ni = .wrap(import_address_table_ni); assert(import_address_table_sym.loc_relocs == .none); import_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); import_address_table_sym.section_number = @@ -6648,13 +6651,13 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); const target = &comp.root_mod.resolved_target.result; - const alignment = switch (comp.root_mod.optimize_mode) { + const alignment: Alignment = switch (comp.root_mod.optimize_mode) { .debug, .safe, .fast, - => target_util.defaultFunctionAlignment(target), - .small => target_util.minFunctionAlignment(target), - }.toStdMem(); + => .fromIp(target_util.defaultFunctionAlignment(target)), + .small => .fromIp(target_util.minFunctionAlignment(target)), + }; const parent_si = (try coff.pseudoSectionMapIndex( .@".thunks", alignment, @@ -6668,12 +6671,12 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { else => |tag| @panic(@tagName(tag)), .AMD64 => { const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 }; - const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni, .{ + const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni.unwrap().?, .{ .alignment = alignment, .size = init.len, }); @memcpy(ni.slice(&coff.mf)[0..init.len], &init); - sym.ni = ni; + sym.ni = .wrap(ni); sym.extra.size = init.len; try coff.addReloc( si, @@ -6736,7 +6739,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol { try coff.symbols.ensureUnusedCapacity(gpa, 1); const optional_hdr_si = coff.addSymbolAssumeCapacity(); const optional_hdr_sym = optional_hdr_si.get(coff); - optional_hdr_sym.ni = Node.known.optional_header; + optional_hdr_sym.ni = .wrap(Node.known.optional_header); assert(optional_hdr_sym.loc_relocs == .none); optional_hdr_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); @@ -6783,7 +6786,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol { try coff.symbols.ensureUnusedCapacity(gpa, 1); const data_dir_si = coff.addSymbolAssumeCapacity(); const data_dir_sym = data_dir_si.get(coff); - data_dir_sym.ni = Node.known.data_directories; + data_dir_sym.ni = .wrap(Node.known.data_directories); assert(data_dir_sym.loc_relocs == .none); data_dir_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); @@ -6826,7 +6829,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { .code => .{ .lazy_code = @fromBackingInt(@intCast(lmr.index)) }, .const_data => .{ .lazy_const_data = @fromBackingInt(@intCast(lmr.index)) }, }); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -6836,7 +6839,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var required_alignment: InternPool.Alignment = .none; @@ -6914,7 +6917,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void { const flags = coff.targetLoad(&sym.section_number.header(coff).flags); if (!flags.CNT_UNINITIALIZED_DATA) { const file_offset = if (isArchive(coff)) - sym.ni.location(&coff.mf).resolve(&coff.mf)[0] + sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[0] else ni.fileLocation(&coff.mf, false).offset; @@ -6927,7 +6930,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void { .input_section => |isi| { try isi.symbol(coff).flushMoved(coff); for (coff.input_symbols.items[@backingInt(isi.firstSymbol(coff))..]) |input_symbol| { - if (input_symbol.si.get(coff).ni != ni) break; + if (input_symbol.si.get(coff).ni != ni.toOptional()) break; try input_symbol.si.flushMoved(coff); } }, @@ -7062,7 +7065,7 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { if (coff.isArchive() and coff.members.items.len > 0) { const last_member = coff.members.items[coff.members.items.len - 1]; // See .archive_member branch for reasoning - assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni); + assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni.toOptional()); try coff.flushResized(last_member.content_ni); } }, @@ -7090,19 +7093,15 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { => unreachable, .archive_member => |mi| { const content_ni = mi.get(coff).content_ni; - const next_ni = content_ni.next(&coff.mf); const content_offset, _ = content_ni.location(&coff.mf).resolve(&coff.mf); - const next_offset = switch (next_ni) { - .none => offset: { - assert(content_ni.parent(&coff.mf) == Node.known.file); - // This must take into account the final file size. If there are trailing - // bytes, they will be expected to contain another valid member header - break :offset coff.mf.memory_map.memory.len; - }, - else => offset: { - assert(coff.getNode(next_ni) == .archive_member_header); - break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0]; - }, + const next_offset = if (content_ni.next(&coff.mf).unwrap()) |next_ni| offset: { + assert(coff.getNode(next_ni) == .archive_member_header); + break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0]; + } else offset: { + assert(content_ni.parent(&coff.mf) == Node.known.file.toOptional()); + // This must take into account the final file size. If there are trailing + // bytes, they will be expected to contain another valid member header + break :offset coff.mf.memory_map.memory.len; }; // Not inserting IMAGE_ARCHIVE_PAD `\n` byte here, because we are expanding to full size @@ -7356,7 +7355,7 @@ fn virtualSlide(coff: *Coff, start_section_index: usize, start_rva: u32) !void { const section_sym = section.si.get(coff); section_sym.rva = rva; coff.targetStore(&header.virtual_address, rva); - try section_sym.ni.childrenMoved(coff.base.comp.gpa, &coff.mf); + try section_sym.ni.unwrap().?.childrenMoved(coff.base.comp.gpa, &coff.mf); rva += coff.targetLoad(&header.virtual_size); } switch (coff.optionalHeaderPtr()) { @@ -7430,7 +7429,7 @@ fn updateExportInner( // TODO: add an errMsg if this conflicts with an existing symbol const export_si = try coff.globalSymbol(.{ .name = name }); const export_sym = export_si.get(coff); - export_sym.ni = exported_ni; + export_sym.ni = .wrap(exported_ni); export_sym.rva = exported_sym.rva; export_sym.section_number = exported_sym.section_number; if (@"export".opts.linkage == .weak and !coff.isImage()) { @@ -7599,14 +7598,13 @@ fn printSymbol( si: Symbol.Index, ) !void { const sym = si.get(coff); - const node = coff.getNode(sym.ni); - try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{t: <26} | {x:08} ", .{ + try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{s: <26} | {x:08} ", .{ si, sym.section_number, if (sym.flags.extra_tag == .size) @as(u64, sym.extra.size) - else if (sym.ni != .none) - sym.ni.location(&coff.mf).resolve(&coff.mf)[1] + else if (sym.ni.unwrap()) |ni| + ni.location(&coff.mf).resolve(&coff.mf)[1] else 0, switch (sym.flags.value_tag) { @@ -7627,7 +7625,7 @@ fn printSymbol( }, sym.ni, if (sym.flags.value_tag == .node_offset) sym.value.node_offset else 0, - node, + if (sym.ni.unwrap()) |ni| @tagName(coff.getNode(ni)) else "", sym.rva, }); @@ -7635,7 +7633,7 @@ fn printSymbol( try w.print("G {f}\n", .{fmtGlobalName(coff, sym.gmi)}); } else { try w.writeAll("| "); - try coff.printNodeName(w, tid, node); + try coff.printNodeName(w, tid, coff.getNode(sym.ni.unwrap().?)); if (sym.flags.extra_tag == .isli) try w.print(" | {s}", .{sym.extra.isli.name(coff).toSlice(coff)}); try w.writeByte('\n'); @@ -7672,7 +7670,7 @@ fn printNodeName( try w.print("({f}{f}, {s}", .{ ioi.path(coff).fmtEscapeString(), fmtMemberNameString(ioi.memberName(coff)), - coff.getNode(is.si.node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff), + coff.getNode(is.si.node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff), }); if (is.comdat_si != .null) { const comdat_sym = is.comdat_si.get(coff); diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 1400d0174815615d4f853ebb53ff2bc73dcd7915..7f23494f4fe45d74c749f6b65b255b50a20781c7 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -18,14 +18,16 @@ const tracy = @import("../tracy.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const Zcu = @import("../Zcu.zig"); +const Alignment = MappedFile.Alignment; base: link.File, options: link.File.OpenOptions, mf: MappedFile, ni: Node.Known, nodes: std.MultiArrayList(Node), +/// Does not contain an item for `SHN_UNDEF`. shdrs: std.ArrayList(Section), -phdrs: std.ArrayList(MappedFile.Node.Index), +phdrs: std.ArrayList(MappedFile.Node.Index.Optional), shndx: struct { got: Section.Index, /// Always `.UNDEF` on some targets (e.g. SPARC). @@ -99,7 +101,7 @@ dso_globals: std.array_hash_map.Auto(String(.strtab), struct { /// the section containing the symbol, and the symbol's offset within the section. I know this /// sounds like a terrible hack, but it is *genuinely* how you're supposed to do this. Copy /// relocations suck. - alignment: std.mem.Alignment, + alignment: Alignment, }), shstrtab: StringTable, strtab: StringTable, @@ -175,7 +177,7 @@ symbol_relocs: std.ArrayList(SymbolReloc), got_relocs: std.ArrayList(GotReloc), /// Set of relocations which must be re-applied if the size of the TLS segment changes. tls_size_symbol_relocs: std.array_hash_map.Auto(SymbolReloc.Index, void), -/// Index matches the index into `shdrs`. +/// Index matches the index into `shdrs`. Like `shdrs`, this map excludes `SHN_UNDEF`. section_by_name: std.array_hash_map.Auto(String(.shstrtab), void), /// Key is the name of a global symbol which has been moved to a new symtab index. Any relocation /// entries which target that symbol must be updated to reference the correct symbol index. @@ -339,8 +341,6 @@ const Node = union(enum) { }; pub const Known = struct { - archive: MappedFile.Node.Index, - archive_header: MappedFile.Node.Index, elf: MappedFile.Node.Index, ehdr: MappedFile.Node.Index, shdr: MappedFile.Node.Index, @@ -349,7 +349,7 @@ const Node = union(enum) { text: MappedFile.Node.Index, data: MappedFile.Node.Index, data_rel_ro: MappedFile.Node.Index, - tls: MappedFile.Node.Index, + tls: MappedFile.Node.Index.Optional, }; comptime { @@ -505,7 +505,7 @@ const Section = struct { } fn get(s: Index, elf: *Elf) *Section { - return &elf.shdrs.items[@backingInt(s)]; + return &elf.shdrs.items[@backingInt(s) - 1]; // overflow means you tried to get the `.UNDEF` section } fn name(s: Index, elf: *Elf) String(.shstrtab) { @@ -539,7 +539,7 @@ const Section = struct { } } - fn ensureAligned(shndx: Index, elf: *Elf, min_align: std.mem.Alignment) Error!void { + fn ensureAligned(shndx: Index, elf: *Elf, min_align: Alignment) Error!void { switch (elf.shdrPtr(shndx)) { inline else => |shdr| { if (elf.targetLoad(&shdr.addralign) >= min_align.toByteUnits()) { @@ -552,7 +552,7 @@ const Section = struct { if (min_align.compare(.gt, ni.alignment(&elf.mf))) { try ni.realign(&elf.mf, elf.base.comp.gpa, min_align, .{}); } - switch (elf.getNode(ni.parent(&elf.mf))) { + switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .elf => {}, .segment => |phndx| try elf.ensureSegmentAligned(phndx, min_align), else => unreachable, @@ -818,7 +818,7 @@ const GotReloc = struct { /// * A section /// * A NAV, UAV, or lazy code/data /// * `.none`, if this relocation was deleted (in which case it should be ignored) - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, /// The offset of the relocation inside of `node`. offset: u64, target: GotKey, @@ -942,8 +942,10 @@ const GotReloc = struct { fn apply(reloc: *GotReloc, elf: *Elf) void { assert(elf.ehdrType() != .REL); - if (reloc.node == .none) return; // deleted - if (reloc.node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) { + const node = reloc.node.unwrap() orelse { + return; // deleted + }; + if (node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) { // There's no point applying the relocation now, because it will be re-applied by // `flushMoved` at some point anyway. return; @@ -968,8 +970,9 @@ const GotReloc = struct { } } fn applyInner(reloc: *const GotReloc, elf: *Elf) error{ RelocationOverflow, RelocationMisaligned }!void { - const dest_vaddr = elf.getNodeVAddr(reloc.node) + reloc.offset; - const dest_slice = reloc.node.slice(&elf.mf)[@intCast(reloc.offset)..]; + const node = reloc.node.unwrap().?; + const dest_vaddr = elf.getNodeVAddr(node) + reloc.offset; + const dest_slice = node.slice(&elf.mf)[@intCast(reloc.offset)..]; const got_vaddr = elf.shndx.got.vaddr(elf); const got_index: u64 = elf.got.getIndex(reloc.target).?; @@ -1587,7 +1590,7 @@ const SymbolReloc = struct { } }, .sparc_le_hix22 => { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -1646,7 +1649,6 @@ const SymbolReloc = struct { fn apply(reloc: *SymbolReloc, elf: *Elf) void { assert(elf.ehdrType() != .REL); - assert(reloc.node != .none); if (reloc.node.hasMoved(&elf.mf) or reloc.target.hasMoved(elf)) { // There's no point applying the relocation now, because it will be re-applied by // `flushMoved` at some point anyway. @@ -1692,7 +1694,7 @@ const SymbolReloc = struct { .I_original => |tls| tls.tcb_size +% reloc.target.value(elf) +% addend, .I_modified => |tls| 0 -% tls.tp_off +% reloc.target.value(elf) +% addend, .II => { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -2044,7 +2046,7 @@ fn pltEntryIsDead(elf: *Elf, plt_index: usize) bool { } const AddLocalSymbolOptions = struct { - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, name: String(.strtab), value: u64, size: u64, @@ -2126,7 +2128,7 @@ const AddGlobalSymbolOptions = struct { } }; - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, name: Name, lib_name: ?[]const u8 = null, value: u64, @@ -2294,8 +2296,8 @@ fn addGlobalSymbolAssumeCapacity(elf: *Elf, opts: AddGlobalSymbolOptions) error{ } const old_head: String(.strtab) = old_head: { - if (opts.node == .none) break :old_head .empty; - const gop = elf.node_global_symbols.getOrPutAssumeCapacity(opts.node); + const node = opts.node.unwrap() orelse break :old_head .empty; + const gop = elf.node_global_symbols.getOrPutAssumeCapacity(node); const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty; gop.value_ptr.* = opts.name.strtab; break :old_head old_head; @@ -2363,7 +2365,7 @@ fn setGlobalSymbolValue( global_name: String(.strtab), global_ptr: *Symbol.Global, new: struct { - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, value: u64, size: u64, type: std.elf.STT, @@ -2371,18 +2373,17 @@ fn setGlobalSymbolValue( }, ) void { assert(new.shndx != .UNDEF); - const old_node = global_ptr.symtab_index.ptr(elf).node; - if (old_node != .none) { + if (global_ptr.symtab_index.ptr(elf).node.unwrap()) |old_node| { if (global_ptr.next_in_node != .empty) { const next = elf.globalByName(global_ptr.next_in_node).?; assert(next.prev_in_node == global_name); - assert(next.symtab_index.ptr(elf).node == old_node); + assert(next.symtab_index.ptr(elf).node.unwrap().? == old_node); next.prev_in_node = global_ptr.prev_in_node; } if (global_ptr.prev_in_node != .empty) { const prev = elf.globalByName(global_ptr.prev_in_node).?; assert(prev.next_in_node == global_name); - assert(prev.symtab_index.ptr(elf).node == old_node); + assert(prev.symtab_index.ptr(elf).node.unwrap().? == old_node); prev.next_in_node = global_ptr.next_in_node; } else { // We're the start of the linked list, so we need to change the head. @@ -2417,8 +2418,8 @@ fn setGlobalSymbolValue( global_ptr.symtab_index.ptr(elf).node = new.node; const old_head: String(.strtab) = old_head: { - if (new.node == .none) break :old_head .empty; - const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new.node); + const new_node = new.node.unwrap() orelse break :old_head .empty; + const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new_node); const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty; gop.value_ptr.* = global_name; break :old_head old_head; @@ -2644,7 +2645,7 @@ const Symbol = struct { /// * A section (the symbol's value is some vaddr in that section) /// * An input section (the symbol's value is some vaddr in that input section) /// * A NAV, UAV, or lazy code/data (the symbol's value is exactly the vaddr of that node) - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, /// The head of a linked list of relocations targeting this symbol. first_target_reloc: SymbolReloc.Index, @@ -2852,8 +2853,7 @@ const Symbol = struct { /// Returns `true` if the target of `s` has moved, meaning the symbol's value will change at /// some point due to a call to `flushMoved`. fn hasMoved(s: Symbol.Id, elf: *Elf) bool { - const node = s.index(elf).ptr(elf).node; - if (node != .none) { + if (s.index(elf).ptr(elf).node.unwrap()) |node| { return node.hasMoved(&elf.mf); } switch (s.unwrap()) { @@ -2998,7 +2998,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol ) catch unreachable; gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, name), .value = 0, .size = 0, @@ -3349,16 +3349,14 @@ fn create( .options = options, .mf = try .init(file, comp.gpa, io), .ni = .{ - .archive = .root, - .archive_header = .none, - .elf = .root, - .ehdr = .none, - .shdr = .none, - .rodata = .none, - .phdr = .none, - .text = .none, - .data = .none, - .data_rel_ro = .none, + .elf = undefined, + .ehdr = undefined, + .shdr = undefined, + .rodata = undefined, + .phdr = undefined, + .text = undefined, + .data = undefined, + .data_rel_ro = undefined, .tls = .none, }, .nodes = .empty, @@ -3489,7 +3487,7 @@ fn initHeaders( .EXEC => comp.config.link_mode == .dynamic, .DYN => true, }; - const addr_align: std.mem.Alignment = switch (class) { + const addr_align: Alignment = switch (class) { .NONE, _ => unreachable, .@"32" => .@"4", .@"64" => .@"8", @@ -3503,7 +3501,7 @@ fn initHeaders( // // It can be handy to temporarily set this to `.@"1"` when working on the linker, because it // prevents alignment bugs from being hidden by your filesystem's block alignment. - const node_block_align: std.mem.Alignment = elf.mf.flags.block_size; + const node_block_align: Alignment = elf.mf.flags.block_size; const plt: PltInfo = .fromMachine(machine); @@ -3601,18 +3599,19 @@ fn initHeaders( const expected_nodes_len = @as(usize, if (is_archive) 2 else 0) + // .archive, .archive_header 3 + // `.file`, `.ehdr`, and `.shdr` nodes - (shnum - 1) + // -1 because the null shdr does not have a `.section` node + (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node try elf.nodes.ensureTotalCapacity(gpa, expected_nodes_len); - try elf.shdrs.ensureTotalCapacity(gpa, shnum); - try elf.section_by_name.ensureUnusedCapacity(gpa, shnum); + try elf.shdrs.ensureTotalCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF + try elf.section_by_name.ensureUnusedCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF try elf.phdrs.resize(gpa, phnum); try elf.symtab.ensureTotalCapacity(gpa, 1); if (is_archive) { elf.nodes.appendAssumeCapacity(.archive); - elf.ni.archive_header = try elf.mf.addOnlyChildNode(gpa, elf.ni.archive, .{ + + const archive_header_ni = try elf.mf.addOnlyChildNode(gpa, .root, .{ .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, .alignment = .@"2", .fixed = true, @@ -3620,7 +3619,8 @@ fn initHeaders( .bubbles_moved = false, .enable_next_moved = true, }); - const archive_header_slice = elf.ni.archive_header.slice(&elf.mf); + elf.nodes.appendAssumeCapacity(.archive_header); + const archive_header_slice = archive_header_ni.slice(&elf.mf); @memcpy(archive_header_slice[0..std.elf.ARMAG.len], std.elf.ARMAG); const strtab_ar_hdr: *std.elf.ar_hdr = @ptrCast(archive_header_slice[std.elf.ARMAG.len..]); strtab_ar_hdr.* = .{ @@ -3633,15 +3633,17 @@ fn initHeaders( .ar_fmag = std.elf.ARFMAG.*, }; - elf.nodes.appendAssumeCapacity(.archive_header); - elf.ni.elf = try elf.mf.addLastChildNode(gpa, elf.ni.archive, .{ + elf.ni.elf = try elf.mf.addLastChildNode(gpa, .root, .{ .alignment = node_block_align.max(.@"2"), .next_moved = true, .bubbles_moved = false, .enable_next_moved = true, }); + elf.nodes.appendAssumeCapacity(.elf); + } else { + elf.ni.elf = .root; + elf.nodes.appendAssumeCapacity(.elf); } - elf.nodes.appendAssumeCapacity(.elf); const entsize: struct { ph: u32, sh: u32 } = switch (class) { .NONE, _ => unreachable, @@ -3665,7 +3667,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.rodata }); - elf.phdrs.items[phndx.rodata] = elf.ni.rodata; + elf.phdrs.items[phndx.rodata] = .wrap(elf.ni.rodata); elf.ni.phdr = try elf.mf.addOnlyChildNode(gpa, elf.ni.rodata, .{ .size = @as(u64, phnum) * entsize.ph, @@ -3675,7 +3677,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.phdr }); - elf.phdrs.items[phndx.phdr] = elf.ni.phdr; + elf.phdrs.items[phndx.phdr] = .wrap(elf.ni.phdr); elf.ni.text = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ .alignment = node_block_align, @@ -3683,7 +3685,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.text }); - elf.phdrs.items[phndx.text] = elf.ni.text; + elf.phdrs.items[phndx.text] = .wrap(elf.ni.text); elf.ni.data = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ // Must be at least `addr_align` for `elf.ni.data_rel_ro` to be placed inside this node @@ -3692,7 +3694,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.data }); - elf.phdrs.items[phndx.data] = elf.ni.data; + elf.phdrs.items[phndx.data] = .wrap(elf.ni.data); if (plt.got_plt == null) { const plt_ni = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ @@ -3701,7 +3703,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.plt }); - elf.phdrs.items[phndx.plt] = plt_ni; + elf.phdrs.items[phndx.plt] = .wrap(plt_ni); } elf.ni.data_rel_ro = try elf.mf.addOnlyChildNode(gpa, elf.ni.data, .{ @@ -3712,14 +3714,14 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.relro }); - elf.phdrs.items[phndx.relro] = elf.ni.data_rel_ro; + elf.phdrs.items[phndx.relro] = .wrap(elf.ni.data_rel_ro); if (comp.config.any_non_single_threaded) { - elf.ni.tls = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + elf.ni.tls = .wrap(try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, - }); + })); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.tls }); elf.phdrs.items[phndx.tls] = elf.ni.tls; } @@ -3785,14 +3787,14 @@ fn initHeaders( ehdr.phentsize = @sizeOf(ElfN.Phdr); ehdr.phnum = @min(phnum, std.elf.PN_XNUM); ehdr.shentsize = @sizeOf(ElfN.Shdr); - ehdr.shnum = 1; // Only the null shdr initially---will be incremented by `addSection` + ehdr.shnum = 1; // Only the SHN_UNDEF shdr initially---will be incremented by `addSection` ehdr.shstrndx = std.elf.SHN_UNDEF; if (elf.targetEndian() != native_endian) std.mem.byteSwapAllFields(ElfN.Ehdr, ehdr); }, } elf.ni.shdr = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ - .size = 1 * entsize.sh, // as above, only the null shdr initially + .size = 1 * entsize.sh, // as above, only the SHN_UNDEF initially .alignment = addr_align.max(node_block_align), .moved = true, .resized = true, @@ -3916,7 +3918,7 @@ fn initHeaders( }; } - if (comp.config.any_non_single_threaded) { + if (elf.ni.tls.unwrap()) |tls_segment_ni| { const ph_tls = &phdr[phndx.tls]; ph_tls.* = .{ .type = .TLS, @@ -3926,7 +3928,7 @@ fn initHeaders( .filesz = 0, .memsz = 0, .flags = .{ .R = true }, - .@"align" = @intCast(elf.ni.tls.alignment(&elf.mf).toByteUnits()), + .@"align" = @intCast(tls_segment_ni.alignment(&elf.mf).toByteUnits()), }; } @@ -3987,7 +3989,6 @@ fn initHeaders( .entsize = 0, }; if (target_endian != native_endian) std.mem.byteSwapAllFields(ElfN.Shdr, sh_undef); - elf.shdrs.appendAssumeCapacity(.{ .lsi = .null, .ni = .none, .rela = .{ .shndx = .UNDEF } }); elf.symtab.addOneAssumeCapacity().* = .{ .node = .none, @@ -4092,7 +4093,7 @@ fn initHeaders( .node_align = node_block_align, }); } else { - elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt], .{ + elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ .name = ".plt", .type = .PROGBITS, .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, @@ -4115,7 +4116,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.interp }); - elf.phdrs.items[phndx.interp] = interp_ni; + elf.phdrs.items[phndx.interp] = .wrap(interp_ni); const sec_interp_shndx = try elf.addSection(interp_ni, .{ .name = ".interp", @@ -4135,7 +4136,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.dynamic }); - elf.phdrs.items[phndx.dynamic] = dynamic_ni; + elf.phdrs.items[phndx.dynamic] = .wrap(dynamic_ni); const dynstr_shndx = try elf.addSection(elf.ni.rodata, .{ .name = ".dynstr", @@ -4347,7 +4348,7 @@ fn initHeaders( try elf.ensureUnusedSymbolCapacity(10, .maybe_global); // Despite the name, `__dso_handle` is necessary even in static binaries. _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = Section.Index.text.get(elf).ni, + .node = .wrap(Section.Index.text.get(elf).ni), .name = try .string(elf, "__dso_handle"), .value = Section.Index.text.vaddr(elf), .size = 0, @@ -4359,7 +4360,7 @@ fn initHeaders( error.MultipleDefinitions => unreachable, // no inputs are processed yet }; _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.plt.get(elf).ni, + .node = .wrap(elf.shndx.plt.get(elf).ni), .name = try .string(elf, "_PROCEDURE_LINKAGE_TABLE_"), .value = elf.shndx.plt.vaddr(elf), .size = 0, @@ -4371,7 +4372,7 @@ fn initHeaders( error.MultipleDefinitions => unreachable, // no inputs are processed yet }; _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.got.get(elf).ni, + .node = .wrap(elf.shndx.got.get(elf).ni), .name = try .string(elf, "_GLOBAL_OFFSET_TABLE_"), .value = switch (machine) { .AARCH64, @@ -4468,7 +4469,7 @@ fn initHeaders( }; if (have_dynamic_section) { _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.dynamic.get(elf).ni, + .node = .wrap(elf.shndx.dynamic.get(elf).ni), .name = try .string(elf, "_DYNAMIC"), .value = elf.shndx.dynamic.vaddr(elf), .size = 0, @@ -4484,16 +4485,16 @@ fn initHeaders( assert(maybe_interp == null); assert(!have_dynamic_section); } - if (comp.config.any_non_single_threaded) elf.shndx.tdata = try elf.addSection(elf.ni.tls, .{ + if (elf.ni.tls.unwrap()) |tls_segment_ni| elf.shndx.tdata = try elf.addSection(tls_segment_ni, .{ .name = ".tdata", .flags = .{ .WRITE = true, .ALLOC = true, .TLS = true }, .node_align = node_block_align, }); assert(elf.nodes.len == expected_nodes_len); - assert(elf.shdrs.items.len == shnum); + assert(elf.shdrs.items.len == shnum - 1); // -1 to exclude SHN_UNDEF - for (0..shnum) |shndx_raw| { + for (1..shnum) |shndx_raw| { // start at 1 to exclude SHN_UNDEF const shndx: Section.Index = @fromBackingInt(@intCast(shndx_raw)); elf.section_by_name.putAssumeCapacityNoClobber(shndx.name(elf), {}); } @@ -4569,7 +4570,7 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { .uav, .lazy_code, .lazy_const_data, - => elf.getNode(ni.parent(&elf.mf)).section, + => elf.getNode(ni.parent(&elf.mf).unwrap().?).section, }; } fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { @@ -4593,7 +4594,7 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { }; } fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { - const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf))) { + const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .archive, .archive_header => unreachable, .elf => return 0, .ehdr, .shdr => unreachable, @@ -4660,7 +4661,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { if (got_relocs) |ptr| { if (ptr.* != .none) { for (elf.got_relocs.items[@backingInt(ptr.*)..]) |*reloc| { - if (reloc.node != ni) break; + if (reloc.node != ni.toOptional()) break; reloc.delete(elf); } } @@ -4691,7 +4692,7 @@ fn flushMovedNodeRelocs( if (first_got_reloc != .none) { for (elf.got_relocs.items[@backingInt(first_got_reloc)..]) |*reloc| { - if (reloc.node != node) break; + if (reloc.node != node.toOptional()) break; reloc.apply(elf); } } @@ -4756,7 +4757,7 @@ fn targetPtrSize(elf: *const Elf) u8 { /// Page alignment for the target platform. /// Usually this returns the maximum page size supported on the /// target to maximize compatibility but there can be exceptions. -fn targetPageAlign(elf: *const Elf) std.mem.Alignment { +fn targetPageAlign(elf: *const Elf) Alignment { return .fromByteUnits(switch (elf.ehdrMachine()) { .AARCH64 => 0x10000, .LOONGARCH => 0x10000, @@ -4810,7 +4811,7 @@ const PltInfo = struct { /// entry, not the `.plt` entry. The `.plt.sec` section has no header entries, and is aligned to /// the same boundary as the `.plt` section. plt_sec: ?struct { entry_size: u8 }, - @"align": std.mem.Alignment, + @"align": Alignment, entry_size: u8, header_entries: u8, @@ -4941,8 +4942,9 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { switch (elf.identClass()) { .NONE, _ => unreachable, inline else => |class| { + const shdrs_len = elf.shdrs.items.len + 1; // +1 for SHN_UNDEF const shdr_slice: []class.ElfN().Shdr = @ptrCast(@alignCast( - raw_slice[0 .. elf.shdrs.items.len * @sizeOf(class.ElfN().Shdr)], + raw_slice[0 .. shdrs_len * @sizeOf(class.ElfN().Shdr)], )); const shdr_ptr = &shdr_slice[@backingInt(shndx)]; return @unionInit(ShdrPtr, @tagName(class), shdr_ptr); @@ -4951,7 +4953,7 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { } fn arHdrPtr(elf: *Elf, ni: MappedFile.Node.Index) *align(2) std.elf.ar_hdr { - assert(elf.ni.elf != MappedFile.Node.Index.root); + assert(elf.ni.elf != .root); const file_offset = ni.fileLocation(&elf.mf, false).offset; return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) { else => unreachable, @@ -5055,7 +5057,7 @@ fn mapInputSection(elf: *Elf, opts: struct { const parent_node: MappedFile.Node.Index = parent: { if (!opts.flags.ALLOC) break :parent elf.ni.elf; if (opts.flags.EXECINSTR) break :parent elf.ni.text; - if (opts.flags.TLS) break :parent elf.ni.tls; + if (opts.flags.TLS) break :parent elf.ni.tls.unwrap().?; if (opts.flags.WRITE) break :parent elf.ni.data; break :parent elf.ni.rodata; }; @@ -5148,12 +5150,12 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node break :section .data_rel_ro; // TODO: it would be better to use `.rodata` if the NAV value doesn't have relocs } }; - const alignment: InternPool.Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) { + const alignment: Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) { .@"fn" => a: { const mod = zcu.navFileScope(nav_index).mod.?; const target = &mod.resolved_target.result; const min = target_util.minFunctionAlignment(target); - break :a switch (nav.resolved.?.@"align") { + break :a .fromIp(switch (nav.resolved.?.@"align") { else => |a| a.maxStrict(min), .none => switch (mod.optimize_mode) { .debug, @@ -5162,20 +5164,20 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node => target_util.defaultFunctionAlignment(target), .small => min, }.maxStrict(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), - }; + }); }, else => switch (nav.resolved.?.@"align") { - .none => Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu), - else => |a| a, + .none => .fromIp(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), + else => |a| .fromIp(a), }, }; - try shndx.ensureAligned(elf, alignment.toStdMem()); + try shndx.ensureAligned(elf, alignment); const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ - .alignment = alignment.toStdMem(), + .alignment = alignment, }); nav_gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, nav.fqn.toSlice(ip)), .value = 0, .size = 0, @@ -5204,19 +5206,19 @@ fn uavMapIndex( try elf.pending_uavs.ensureUnusedCapacity(gpa, 1); const abi_align = Value.fromInterned(uav_val).typeOf(zcu).abiAlignment(zcu); - const resolved_align: InternPool.Alignment = switch (uav_align) { - .none => abi_align, - else => |a| a.minStrict(abi_align), + const resolved_align: Alignment = switch (uav_align) { + .none => .fromIp(abi_align), + else => |a| .fromIp(a.minStrict(abi_align)), }; const uav_gop = elf.uavs.getOrPutAssumeCapacity(uav_val); const umi: Node.UavMapIndex = @fromBackingInt(@intCast(uav_gop.index)); if (!uav_gop.found_existing) { const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs - try shndx.ensureAligned(elf, resolved_align.toStdMem()); + try shndx.ensureAligned(elf, resolved_align); const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ .moved = true, // see assert at end of `genUav` - .alignment = resolved_align.toStdMem(), + .alignment = resolved_align, }); var name_buf: [32]u8 = undefined; const name = std.fmt.bufPrint( @@ -5226,7 +5228,7 @@ fn uavMapIndex( ) catch unreachable; uav_gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, name), .value = 0, .size = 0, @@ -5239,11 +5241,11 @@ fn uavMapIndex( elf.const_prog_node.increaseEstimatedTotalItems(1); elf.pending_uavs.appendAssumeCapacity(umi); } else { - const node = uav_gop.value_ptr.lsi.index().ptr(elf).node; - const shndx = elf.getNode(node.parent(&elf.mf)).section; - try shndx.ensureAligned(elf, resolved_align.toStdMem()); - if (resolved_align.toStdMem().order(node.alignment(&elf.mf)).compare(.gt)) { - try node.realign(&elf.mf, gpa, resolved_align.toStdMem(), .{}); + const node = uav_gop.value_ptr.lsi.index().ptr(elf).node.unwrap().?; + const shndx = elf.getNode(node.parent(&elf.mf).unwrap().?).section; + try shndx.ensureAligned(elf, resolved_align); + if (resolved_align.order(node.alignment(&elf.mf)).compare(.gt)) { + try node.realign(&elf.mf, gpa, resolved_align, .{}); } } return umi; @@ -5459,7 +5461,7 @@ fn loadObject( .member = if (member) |m| try gpa.dupe(u8, m) else null, .extra = undefined, }; - if (elf.ni.elf != MappedFile.Node.Index.root) { + if (elf.ni.elf != .root) { try elf.nodes.ensureUnusedCapacity(gpa, 1); input.extra = .{ .node = try elf.mf.addLastChildNode(gpa, .root, .{ .size = fl.size + @sizeOf(std.elf.ar_hdr), @@ -5640,7 +5642,7 @@ fn loadObject( .node_fixed = true, }, }; - const need_align: std.mem.Alignment = .fromByteUnits( + const need_align: Alignment = .fromByteUnits( std.math.ceilPowerOfTwoAssert(usize, @intCast(@max(section.shdr.addralign, 1))), ); try opts.shndx.ensureAligned(elf, need_align); @@ -5754,7 +5756,7 @@ fn loadObject( ), .LOCAL => { const lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = input_section_node, + .node = .wrap(input_section_node), .name = try elf.string(.strtab, name), .value = input_sym.value, .size = input_sym.size, @@ -5765,7 +5767,7 @@ fn loadObject( }, .GLOBAL, .WEAK, .GNU_UNIQUE => |bind| { si.* = elf.addGlobalSymbolAssumeCapacity(.{ - .node = input_section_node, + .node = .wrap(input_section_node), .name = try .string(elf, name), .value = input_sym.value, .size = input_sym.size, @@ -5893,7 +5895,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars return diags.failParse(path, "bad machine", .{}); if (ehdr.shnum > 0) try fr.seekTo(ehdr.shoff); // We're going to need to know the alignment of every section later. - const section_aligns = try gpa.alloc(std.mem.Alignment, ehdr.shnum); + const section_aligns = try gpa.alloc(Alignment, ehdr.shnum); defer gpa.free(section_aligns); const dynamic_sh: ElfN.Shdr, const dynsym_sh: ElfN.Shdr = sh: { var dynamic_sh: ?ElfN.Shdr = null; @@ -5999,7 +6001,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We need to guess the worst-case alignment of the symbol. Yes, I know this seems // insane---refer to the doc comment on `alignment` in `Elf.dso_globals`. - const sym_align: std.mem.Alignment = switch (sym.value) { + const sym_align: Alignment = switch (sym.value) { 0 => section_aligns[sym.shndx], else => section_aligns[sym.shndx].min(@fromBackingInt(@intCast(@ctz(sym.value)))), }; @@ -6158,7 +6160,7 @@ fn createInitFiniArraySection( ) Error!void { assert(shndx.* == .UNDEF); const gpa = elf.base.comp.gpa; - const addr_align: std.mem.Alignment = switch (elf.identClass()) { + const addr_align: Alignment = switch (elf.identClass()) { .NONE, _ => unreachable, .@"32" => .@"4", .@"64" => .@"8", @@ -6178,14 +6180,14 @@ fn createInitFiniArraySection( const start_sym_name = try elf.string(.strtab, "__" ++ name ++ "_start"); const end_sym_name = try elf.string(.strtab, "__" ++ name ++ "_end"); elf.setGlobalSymbolValue(start_sym_name, elf.globals.strong_def.getPtr(start_sym_name).?, .{ - .node = shndx.get(elf).ni, + .node = .wrap(shndx.get(elf).ni), .value = shndx.vaddr(elf), .size = 0, .type = .NOTYPE, .shndx = shndx.*, }); elf.setGlobalSymbolValue(end_sym_name, elf.globals.strong_def.getPtr(end_sym_name).?, .{ - .node = shndx.get(elf).ni, + .node = .wrap(shndx.get(elf).ni), .value = shndx.vaddr(elf), .size = 0, .type = .NOTYPE, @@ -6218,7 +6220,7 @@ fn prelinkInner(elf: *Elf) Error!void { const comp = elf.base.comp; const gpa = comp.gpa; - if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == MappedFile.Node.Index.root) { + if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == .root) { // We're using self-hosted codegen---add an input representing the Zig "object". try elf.ensureUnusedSymbolCapacity(1, .all_local); try elf.inputs.ensureUnusedCapacity(gpa, 1); @@ -6388,9 +6390,9 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { size: std.elf.Xword = 0, link: std.elf.Word = 0, info: std.elf.Word = 0, - addralign: std.mem.Alignment = .@"1", + addralign: Alignment = .@"1", entsize: std.elf.Word = 0, - node_align: std.mem.Alignment = .@"1", + node_align: Alignment = .@"1", fixed: bool = false, }) Error!Section.Index { switch (opts.type) { @@ -6447,7 +6449,7 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { }); const addr = elf.computeNodeVAddr(ni); const lsi: Symbol.LocalIndex = if (opts.flags.ALLOC) elf.addLocalSymbolAssumeCapacity(.{ - .node = ni, + .node = .wrap(ni), .name = .empty, .value = addr, .size = 0, @@ -6499,7 +6501,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize) assert(elf.section_by_name.count() == elf.shdrs.items.len); try elf.section_by_name.ensureUnusedCapacity(gpa, 1); - const rela_shndx = try elf.addSection(.none, .{ + const rela_shndx = try elf.addSection(elf.ni.elf, .{ .name = rela_name, .type = .RELA, .link = @backingInt(Section.Index.symtab), @@ -6546,7 +6548,6 @@ fn addRelocAssumeCapacity( addend: i64, @"type": MachineRelocType, ) (Error || error{ UnknownRelocation, NonStaticRelocation, UnimplementedRelocation })!void { - assert(node != .none); switch (elf.ehdrType()) { .REL => { const rela_shndx = elf.getNodeShndx(node).get(elf).rela.shndx; @@ -6894,7 +6895,6 @@ fn addSymbolRelocAssumeCapacity( @"type": SymbolReloc.Type, ) Error!void { assert(elf.ehdrType() != .REL); - assert(node != .none); const rela_index: Section.RelaIndex.Optional = r: { if (elf.shndx.dynamic == .UNDEF) break :r .none; @@ -7089,7 +7089,7 @@ fn addGotRelocAssumeCapacity( } elf.got_relocs.appendAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .offset = offset, .target = target, .addend = addend, @@ -7111,7 +7111,7 @@ fn updateGotEntry(elf: *Elf, got_index: usize) void { .tpoff => |sym_id| val: { // Only the executable's per-module TLS block is at a known offset from the TLS pointer. if (elf.base.comp.config.output_mode == .Exe and elf.classifySymbolValue(sym_id) != .dynamic) { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -7336,7 +7336,7 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) if (!Type.fromInterned(nav.resolved.?.type).hasRuntimeBits(zcu)) return; const nmi = try elf.navMapIndex(zcu, nav_index); - const ni = nmi.symbol(elf).index().ptr(elf).node; + const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be @@ -7392,7 +7392,7 @@ fn updateFuncInner( const nmi = try elf.navMapIndex(zcu, func.owner_nav); log.debug("updateFunc({f}) = {d}", .{ nav.fqn.fmt(ip), nmi.symbol(elf) }); - const ni = nmi.symbol(elf).index().ptr(elf).node; + const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be @@ -7677,7 +7677,7 @@ fn idleProgNode( break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), }) catch &name; }, .nav => |nmi| { @@ -7737,7 +7737,7 @@ fn genUav( const gpa = comp.gpa; const uav_val = umi.uavValue(elf); - const ni = umi.symbol(elf).index().ptr(elf).node; + const ni = umi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); var nw: MappedFile.Node.Writer = undefined; @@ -7766,7 +7766,7 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { const gpa = zcu.gpa; const lazy = lmr.lazySymbol(elf); - const ni = lmr.symbol(elf).index().ptr(elf).node; + const ni = lmr.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the lazy node is marked as moved so that once we're done, `flushMoved` will eventually @@ -7842,7 +7842,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { fr.seekTo(file_loc.offset) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), e, @@ -7853,7 +7853,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { defer nw.deinit(); const n_bytes = nw.interface.sendFileAll(&fr, .limited(@intCast(file_loc.size))) catch |err| switch (err) { error.ReadFailed => return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), fr.err orelse (fr.seek_err orelse fr.size_err.?), @@ -7861,7 +7861,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { error.WriteFailed => return nw.err.?, }; if (n_bytes != file_loc.size) return diags.fail("failed to read input section '{s}' from \"{f}{f}\": unexpected eof", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), }); @@ -7994,7 +7994,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void const ii = isi.input(elf); var lsi, const end_lsi = ii.localSymbolRange(elf); while (lsi != end_lsi) : (lsi = @fromBackingInt(@backingInt(lsi) + 1)) { - if (lsi.index().ptr(elf).node != ni) continue; + if (lsi.index().ptr(elf).node != ni.toOptional()) continue; const visibility: std.elf.STV = switch (elf.symPtr(lsi.index())) { inline else => |sym| elf.targetLoad(&sym.other).visibility, }; @@ -8079,7 +8079,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void /// moving or resizing of a segment could reorder them and thereby affect how we handle *future* /// changes to segments. fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Error!void { - const segment_ni = elf.phdrs.items[orig_phndx]; + const segment_ni = elf.phdrs.items[orig_phndx].unwrap().?; assert(elf.getNode(segment_ni).segment == orig_phndx); const page_align = elf.targetPageAlign(); const node_align = segment_ni.alignment(&elf.mf); @@ -8165,7 +8165,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro const next_ni = elf.phdrs.items[next_phndx]; elf.phdrs.items[phndx] = next_ni; elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; - elf.phdrs.items[next_phndx] = segment_ni; + elf.phdrs.items[next_phndx] = .wrap(segment_ni); elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; phndx = @intCast(next_phndx); } @@ -8203,7 +8203,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo .shdr => {}, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| { - assert(elf.phdrs.items[phndx] == ni); + assert(elf.phdrs.items[phndx].unwrap().? == ni); const ph = &phdr[phndx]; elf.targetStore(&ph.filesz, @intCast(size)); switch (elf.targetLoad(&ph.type)) { @@ -8301,51 +8301,45 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! break :member_offset switch (tag) { else => unreachable, .archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true }, - .elf, .input_member => .{ offset, switch (ni.prev(&elf.mf)) { - .none => unreachable, - else => |prev_ni| !prev_ni.hasNextMoved(&elf.mf), - } }, + .elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) }, }; }; - const member_size = member_end: switch (ni.next(&elf.mf)) { - else => |next_ni| { - const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_size = next_member_end: switch (next_ni.next(&elf.mf)) { - else => |next_next_ni| { - const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); - break :next_member_end next_next_offset - @sizeOf(std.elf.ar_hdr); - }, - .none => { - _, const parent_size = - ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf); - break :next_member_end parent_size; - }, - } - next_offset; - const ar_hdr = elf.arHdrPtr(next_ni); - var name_buf: [16]u8 = undefined; - _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ - switch (elf.getNode(next_ni)) { - else => unreachable, - .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), - .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ - std.fs.path.basename(ii.path(elf).sub_path), - }), - } catch @panic("TODO: long archive member names"), - }) catch @panic("TODO: long archive member names"); - ar_hdr.ar_date = "0 ".*; - ar_hdr.ar_uid = "0 ".*; - ar_hdr.ar_gid = "0 ".*; - ar_hdr.ar_mode = "644 ".*; - _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch - @panic("archive member too large"); - ar_hdr.ar_fmag = std.elf.ARFMAG.*; - break :member_end next_offset - @sizeOf(std.elf.ar_hdr); - }, - .none => { - _, const parent_size = ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf); - break :member_end parent_size; - }, - } - member_offset; + const member_size = if (ni.next(&elf.mf).unwrap()) |next_ni| member_size: { + const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); + const next_member_size = if (next_ni.next(&elf.mf).unwrap()) |next_next_ni| next_member_size: { + const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); + const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr); + break :next_member_size next_member_end - next_offset; + } else next_member_size: { + _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); + const next_member_end = parent_size; + break :next_member_size next_member_end - next_offset; + }; + const ar_hdr = elf.arHdrPtr(next_ni); + var name_buf: [16]u8 = undefined; + _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ + switch (elf.getNode(next_ni)) { + else => unreachable, + .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), + .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ + std.fs.path.basename(ii.path(elf).sub_path), + }), + } catch @panic("TODO: long archive member names"), + }) catch @panic("TODO: long archive member names"); + ar_hdr.ar_date = "0 ".*; + ar_hdr.ar_uid = "0 ".*; + ar_hdr.ar_gid = "0 ".*; + ar_hdr.ar_mode = "644 ".*; + _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch + @panic("archive member too large"); + ar_hdr.ar_fmag = std.elf.ARFMAG.*; + const member_end = next_offset - @sizeOf(std.elf.ar_hdr); + break :member_size member_end - member_offset; + } else member_size: { + _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); + const member_end = parent_size; + break :member_size member_end - member_offset; + }; if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{ member_size, }) catch @panic("archive member too large"); @@ -8775,12 +8769,13 @@ fn updateExportInner( // only emitting this error if the symbol we're conflicting with comes from an input // section (as opposed to the ZCU). const conflicting_global = elf.globalByName(try elf.string(.strtab, name)).?; - const conflicting_node = conflicting_global.symtab_index.ptr(elf).node; - if (elf.getNode(conflicting_node) == .input_section) { - return elf.base.comp.link_diags.fail( - "multiple definitions of '{s}'", - .{name}, - ); + if (conflicting_global.symtab_index.ptr(elf).node.unwrap()) |conflicting_node| { + if (elf.getNode(conflicting_node) == .input_section) { + return elf.base.comp.link_diags.fail( + "multiple definitions of '{s}'", + .{name}, + ); + } } }, }; @@ -8842,7 +8837,7 @@ pub fn printNode( try w.print("({f}{f}, {s})", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), }); }, .copied_global => |name| try w.print("(copy:{s})", .{name.slice(elf)}), @@ -8916,14 +8911,14 @@ pub fn printNode( } } -fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignment) Error!void { +fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error!void { const gpa = elf.base.comp.gpa; // We need to loop through parent nodes because segments may be nested (e.g. a PT_TLS segment // inside a PT_LOAD segment). var phndx = start_phndx; while (true) { // Align the actual node - const seg_ni = elf.phdrs.items[phndx]; + const seg_ni = elf.phdrs.items[phndx].unwrap().?; if (min_align.compare(.gt, seg_ni.alignment(&elf.mf))) { try seg_ni.realign(&elf.mf, gpa, min_align, .{}); } @@ -8948,7 +8943,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen }, } // Continue on to the parent segment, if any - switch (elf.getNode(seg_ni.parent(&elf.mf))) { + switch (elf.getNode(seg_ni.parent(&elf.mf).unwrap().?)) { .segment => |parent_phndx| phndx = parent_phndx, .elf => return, else => unreachable, @@ -8959,7 +8954,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen /// Must be called deterministically after any call to `MappedFile.Node.Index.resize` /// (of `elf.ni.elf` or one of its children) before any possible calls to `idle`. fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void { - if (elf.ni.elf == MappedFile.Node.Index.root) return; + if (elf.ni.elf == .root) return; var child_it = elf.ni.elf.reverseChildren(&elf.mf); const last_end = if (child_it.next()) |last_ni| last_end: { const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf); diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index a6b86e2fc7ac35353027705400fc95e63e229587..b1da80973c756c2aab7b30d88b3605171b06362f 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -13,14 +13,14 @@ const windows = std.os.windows; io: Io, flags: packed struct { - block_size: std.mem.Alignment, + block_size: Alignment, copy_file_range_unsupported: bool, fallocate_punch_hole_unsupported: bool, fallocate_insert_range_unsupported: bool, }, memory_map: Io.File.MemoryMap, nodes: std.ArrayList(Node), -free_ni: Node.Index, +free_ni: Node.Index.Optional, large: std.ArrayList(u64), updates: std.ArrayList(Node.Index), /// This progress node's estimated total items is increased once for each node appended to `updates`. @@ -62,6 +62,94 @@ pub const Error = Allocator.Error || Io.Cancelable || error{ MappedFileIo, }; +/// This separate `Alignment` type exists because neither of the other options is really suitable: +/// +/// * `std.mem.Alignment` is based on `usize`, which---while technically okay since the file is +/// memory-mapped---is in practice very annoying to work with in linker implementations +/// +/// * `InternPool.Alignment` is based on `u64`, which is better, but it has the value `.none`, which +/// is also really annoying to handle, because no alignment is ever nullable in this API +/// +/// At some point we should probably just change `InternPool.Alignment` to be non-optional, and add +/// a new `InternPool.Alignment.Optional` type for the case where it can actually be `.none`. At +/// that point we can transition this code to using `InternPool.Alignment` (although it should +/// probably be namespaced elsewhere, it has nothing to do with the `InternPool`!). +pub const Alignment = enum(u6) { + @"1" = 0, + @"2" = 1, + @"4" = 2, + @"8" = 3, + @"16" = 4, + @"32" = 5, + @"64" = 6, + _, + + pub fn fromIp(a: @import("../InternPool.zig").Alignment) Alignment { + assert(a != .none); + return @bitCast(a); + } + + pub fn toLog2Units(a: Alignment) u6 { + return @backingInt(a); + } + + pub fn fromLog2Units(a: u6) Alignment { + return @fromBackingInt(a); + } + + pub fn toByteUnits(a: Alignment) u64 { + return @as(u64, 1) << @backingInt(a); + } + + pub fn fromByteUnits(n: u64) Alignment { + assert(std.math.isPowerOfTwo(n)); + return @fromBackingInt(@intCast(@ctz(n))); + } + + pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order { + return std.math.order(@backingInt(lhs), @backingInt(rhs)); + } + + pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool { + return std.math.compare(@backingInt(lhs), op, @backingInt(rhs)); + } + + pub fn max(lhs: Alignment, rhs: Alignment) Alignment { + return @fromBackingInt(@max(@backingInt(lhs), @backingInt(rhs))); + } + + pub fn min(lhs: Alignment, rhs: Alignment) Alignment { + return @fromBackingInt(@min(@backingInt(lhs), @backingInt(rhs))); + } + + pub inline fn of(comptime T: type) Alignment { + return comptime .fromByteUnits(@alignOf(T)); + } + + /// Given that a base address is known to be aligned to `a`, computes the known alignment of + /// that base address plus `off`. + pub fn offset(a: Alignment, off: u64) Alignment { + return .fromLog2Units(@min(a.toLog2Units(), @ctz(off))); + } + + /// Align an address forwards to this alignment. + pub fn forward(a: Alignment, addr: u64) u64 { + const x = (@as(u64, 1) << @backingInt(a)) - 1; + return (addr + x) & ~x; + } + + /// Align an address backwards to this alignment. + pub fn backward(a: Alignment, addr: u64) u64 { + const x = (@as(u64, 1) << @backingInt(a)) - 1; + return addr & ~x; + } + + /// Check if an address is aligned to this amount. + pub fn check(a: Alignment, addr: u64) bool { + return @ctz(addr) >= @backingInt(a); + } +}; + pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancelable || IoError)!MappedFile { var mf: MappedFile = .{ .io = io, @@ -101,7 +189,7 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel .alignment = mf.flags.block_size, .fixed = true, } }); - assert(root_ni == Node.Index.root); + assert(root_ni == .root); try mf.ensureTotalCapacityInner(@intCast(size)); return mf; } @@ -117,17 +205,17 @@ pub fn deinit(mf: *MappedFile, gpa: Allocator) void { } pub const Node = extern struct { - parent: Node.Index, - prev: Node.Index, - next: Node.Index, - first: Node.Index, - last: Node.Index, + parent: Node.Index.Optional, + prev: Node.Index.Optional, + next: Node.Index.Optional, + first: Node.Index.Optional, + last: Node.Index.Optional, flags: Flags, location_payload: Location.Payload, pub const Flags = packed struct(u32) { location_tag: Location.Tag, - alignment: std.mem.Alignment, + alignment: Alignment, /// Whether this node can be moved. fixed: bool, /// Whether this node has been moved. @@ -142,7 +230,7 @@ pub const Node = extern struct { bubbles_moved: bool, /// Whether `next_moved` events are reported in `updates`. enable_next_moved: bool, - unused: @Int(.unsigned, 32 - @bitSizeOf(std.mem.Alignment) - 8) = 0, + unused: u18 = 0, }; pub const Location = union(enum(u1)) { @@ -180,46 +268,62 @@ pub const Node = extern struct { }; pub const Index = enum(u32) { - none, + root, _, - pub const root: Node.Index = .none; + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn unwrap(oi: Optional) ?Index { + return switch (oi) { + _ => @fromBackingInt(@backingInt(oi)), + .none => null, + }; + } + pub fn wrap(i: Index) Optional { + const oi: Optional = @bitCast(i); + assert(oi != .none); + return oi; + } + }; fn get(ni: Node.Index, mf: *const MappedFile) *Node { return &mf.nodes.items[@backingInt(ni)]; } - pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index { + /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. + pub const toOptional = Optional.wrap; + + pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).parent; } - pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index { + pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).next; } fn setNext( prev_ni: Node.Index, gpa: Allocator, - next_ni: Node.Index, + next_ni: Node.Index.Optional, mf: *MappedFile, ) Allocator.Error!void { - assert(prev_ni != .none); const prev_next = &prev_ni.get(mf).next; if (prev_next.* == next_ni) return; prev_next.* = next_ni; try prev_ni.nextMoved(gpa, mf); } - pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index { + pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).prev; } pub fn ChildIterator(comptime direction: enum { prev, next }) type { return struct { mf: *const MappedFile, - ni: Node.Index, + ni: Node.Index.Optional, pub fn next(it: *@This()) ?Node.Index { - const ni = it.ni; - if (ni == .none) return null; + const ni = it.ni.unwrap() orelse return null; it.ni = @field(ni.get(it.mf), @tagName(direction)); return ni; } @@ -233,20 +337,20 @@ pub const Node = extern struct { } pub fn childrenMoved(ni: Node.Index, gpa: Allocator, mf: *MappedFile) Allocator.Error!void { - var child_ni = ni.get(mf).last; - while (child_ni != .none) { + var child_oni = ni.get(mf).last; + while (child_oni.unwrap()) |child_ni| { try child_ni.moved(gpa, mf); - child_ni = child_ni.get(mf).prev; + child_oni = child_ni.get(mf).prev; } } pub fn hasMoved(ni: Node.Index, mf: *const MappedFile) bool { var parent_ni = ni; - while (parent_ni != Node.Index.root) { + while (parent_ni != .root) { const parent_node = parent_ni.get(mf); if (!parent_node.flags.bubbles_moved) break; if (parent_node.flags.moved) return true; - parent_ni = parent_node.parent; + parent_ni = parent_node.parent.unwrap().?; } return false; } @@ -263,9 +367,8 @@ pub const Node = extern struct { if (ni.hasMoved(mf)) return; const node = ni.get(mf); node.flags.moved = true; - switch (node.prev) { - .none => {}, - else => |prev_ni| prev_ni.nextMovedAssumeCapacity(mf), + if (node.prev.unwrap()) |prev_ni| { + prev_ni.nextMovedAssumeCapacity(mf); } if (node.flags.resized or node.flags.next_moved) return; mf.updates.appendAssumeCapacity(ni); @@ -314,7 +417,7 @@ pub const Node = extern struct { mf.update_prog_node.increaseEstimatedTotalItems(1); } - pub fn alignment(ni: Node.Index, mf: *const MappedFile) std.mem.Alignment { + pub fn alignment(ni: Node.Index, mf: *const MappedFile) Alignment { return ni.get(mf).flags.alignment; } @@ -361,8 +464,11 @@ pub const Node = extern struct { while (true) { const parent_node = parent_ni.get(mf); if (set_has_content) parent_node.flags.has_content = true; - if (parent_ni == .none) break; - parent_ni = parent_node.parent; + if (parent_ni == .root) { + assert(parent_node.parent == .none); + break; + } + parent_ni = parent_node.parent.unwrap().?; const parent_offset, _ = parent_ni.location(mf).resolve(mf); offset += parent_offset; } @@ -402,12 +508,12 @@ pub const Node = extern struct { }; /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. - /// Asserts that `ni` is not `Node.Index.root`. + /// Asserts that `ni` is not `.root`. pub fn realign( ni: Node.Index, mf: *MappedFile, gpa: Allocator, - new_alignment: std.mem.Alignment, + new_alignment: Alignment, opts: RealignNodeOptions, ) Error!void { mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { @@ -590,9 +696,9 @@ pub const Node = extern struct { }; fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { - parent: Node.Index = .none, - prev: Node.Index = .none, - next: Node.Index = .none, + parent: Node.Index.Optional = .none, + prev: Node.Index.Optional = .none, + next: Node.Index.Optional = .none, offset: u64 = 0, add_node: AddNodeOptions, }) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { @@ -605,22 +711,32 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; }; - const free_ni: Node.Index, const free_node = free: switch (mf.free_ni) { - .none => .{ @fromBackingInt(@intCast(mf.nodes.items.len)), mf.nodes.addOneAssumeCapacity() }, - else => |free_ni| { - const free_node = free_ni.get(mf); - mf.free_ni = free_node.next; - break :free .{ free_ni, free_node }; - }, + + const free_ni: Node.Index, const free_node: *Node = if (mf.free_ni.unwrap()) |free_ni| free: { + const free_node = free_ni.get(mf); + mf.free_ni = free_node.next; + break :free .{ free_ni, free_node }; + } else .{ + @fromBackingInt(@intCast(mf.nodes.items.len)), + mf.nodes.addOneAssumeCapacity(), }; - switch (opts.prev) { - .none => opts.parent.get(mf).first = free_ni, - else => |prev_ni| try prev_ni.setNext(gpa, free_ni, mf), + + if (opts.prev.unwrap()) |prev_ni| { + try prev_ni.setNext(gpa, .wrap(free_ni), mf); + } else if (opts.parent.unwrap()) |parent_ni| { + parent_ni.get(mf).first = .wrap(free_ni); + } else { + assert(free_ni == .root); } - switch (opts.next) { - .none => opts.parent.get(mf).last = free_ni, - else => |next_ni| next_ni.get(mf).prev = free_ni, + + if (opts.next.unwrap()) |next_ni| { + next_ni.get(mf).prev = .wrap(free_ni); + } else if (opts.parent.unwrap()) |parent_ni| { + parent_ni.get(mf).last = .wrap(free_ni); + } else { + assert(free_ni == .root); } + free_node.* = .{ .parent = opts.parent, .prev = opts.prev, @@ -659,7 +775,7 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { pub const AddNodeOptions = struct { size: u64 = 0, - alignment: std.mem.Alignment = .@"1", + alignment: Alignment = .@"1", fixed: bool = false, moved: bool = false, resized: bool = false, @@ -678,7 +794,7 @@ pub fn addOnlyChildNode( const parent = parent_ni.get(mf); assert(parent.first == .none and parent.last == .none); return mf.addNode(gpa, .{ - .parent = parent_ni, + .parent = .wrap(parent_ni), .add_node = opts, }) catch |err| switch (err) { error.OutOfMemory, @@ -700,7 +816,7 @@ pub fn addFirstChildNode( try mf.nodes.ensureUnusedCapacity(gpa, 1); const parent = parent_ni.get(mf); return mf.addNode(gpa, .{ - .parent = parent_ni, + .parent = .wrap(parent_ni), .next = parent.first, .add_node = opts, }) catch |err| switch (err) { @@ -723,14 +839,12 @@ pub fn addLastChildNode( try mf.nodes.ensureUnusedCapacity(gpa, 1); const parent = parent_ni.get(mf); return mf.addNode(gpa, .{ - .parent = parent_ni, + .parent = .wrap(parent_ni), .prev = parent.last, - .offset = offset: switch (parent.last) { - .none => 0, - else => |last_ni| { - const last_offset, const last_size = last_ni.location(mf).resolve(mf); - break :offset last_offset + last_size; - }, + .offset = offset: { + const last_ni = parent.last.unwrap() orelse break :offset 0; + const last_offset, const last_size = last_ni.location(mf).resolve(mf); + break :offset last_offset + last_size; }, .add_node = opts, }) catch |err| switch (err) { @@ -750,13 +864,12 @@ pub fn addNodeAfter( prev_ni: Node.Index, opts: AddNodeOptions, ) Error!Node.Index { - assert(prev_ni != .none); try mf.nodes.ensureUnusedCapacity(gpa, 1); const prev = prev_ni.get(mf); const prev_offset, const prev_size = prev.location().resolve(mf); return mf.addNode(gpa, .{ .parent = prev.parent, - .prev = prev_ni, + .prev = .wrap(prev_ni), .next = prev.next, .offset = prev_offset + prev_size, .add_node = opts, @@ -783,10 +896,10 @@ fn shrinkNode( const old_offset, _ = node.location().resolve(mf); // This would require unmapping first - assert(ni != Node.Index.root); + assert(ni != .root); - if (node.last != .none) { - const last = node.last.get(mf); + if (node.last.unwrap()) |last_ni| { + const last = last_ni.get(mf); const last_offset, const last_size = last.location().resolve(mf); assert(last_offset + last_size > size); } @@ -795,15 +908,16 @@ fn shrinkNode( try mf.updates.ensureUnusedCapacity(gpa, 4); ni.setLocationAssumeCapacity(mf, old_offset, size); - if (!shift_next or node.next == .none) return; + if (!shift_next) return; + const next_ni = node.next.unwrap() orelse return; - const next = node.next.get(mf); + const next = next_ni.get(mf); const old_next_offset, const next_size = next.location().resolve(mf); const padding = old_next_offset - (old_offset + size); const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); if (next.flags.has_content and new_next_offset < old_next_offset) { - const old_file_offset = node.next.fileLocation(mf, false).offset; + const old_file_offset = next_ni.fileLocation(mf, false).offset; const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; @memmove( mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], @@ -812,7 +926,7 @@ fn shrinkNode( @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); } - node.next.setLocationAssumeCapacity(mf, new_next_offset, next_size); + next_ni.setLocationAssumeCapacity(mf, new_next_offset, next_size); } fn resizeNode( @@ -828,7 +942,8 @@ fn resizeNode( const new_size = node.flags.alignment.forward(@intCast(requested_size)); // Resize the entire file - if (ni == Node.Index.root) { + const parent_ni = node.parent.unwrap() orelse { + assert(ni == .root); try mf.ensureCapacityForSetLocation(gpa); mf.memory_map.write(io) catch |err| switch (err) { error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking @@ -839,15 +954,13 @@ fn resizeNode( try mf.ensureTotalCapacityInner(@intCast(new_size)); ni.setLocationAssumeCapacity(mf, old_offset, new_size); return; - } - const parent = node.parent.get(mf); + }; + const parent = parent_ni.get(mf); _, var old_parent_size = parent.location().resolve(mf); - const trailing_end = trailing_end: switch (node.next) { - .none => old_parent_size, - else => |next_ni| { - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }, + const trailing_end = trailing_end: { + const next_ni = node.next.unwrap() orelse break :trailing_end old_parent_size; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :trailing_end next_offset; }; assert(old_offset + old_size <= trailing_end); if (old_offset + new_size <= trailing_end) { @@ -877,7 +990,7 @@ fn resizeNode( else => |e| return e, }; // Ask the filesystem driver to insert extents into the file without copying any data - const last_offset, const last_size = parent.last.location(mf).resolve(mf); + const last_offset, const last_size = parent.last.unwrap().?.location(mf).resolve(mf); const last_end = last_offset + last_size; assert(last_end <= old_parent_size); _, const file_size = Node.Index.root.location(mf).resolve(mf); @@ -900,13 +1013,13 @@ fn resizeNode( enclosing.location().resolve(mf); const new_enclosing_size = old_enclosing_size + range_size; enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); - if (enclosing_ni == Node.Index.root) { + if (enclosing_ni == .root) { assert(enclosing_offset == 0); try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); break; } - var after_ni = enclosing.next; - while (after_ni != .none) { + var after_oni = enclosing.next; + while (after_oni.unwrap()) |after_ni| { try mf.ensureCapacityForSetLocation(gpa); const after = after_ni.get(mf); const after_offset, const after_size = after.location().resolve(mf); @@ -915,9 +1028,9 @@ fn resizeNode( range_size + after_offset, after_size, ); - after_ni = after.next; + after_oni = after.next; } - enclosing_ni = enclosing.parent; + enclosing_ni = enclosing.parent.unwrap().?; } return; }, @@ -939,32 +1052,33 @@ fn resizeNode( if (node.next == .none) { // As this is the last node, we simply need more space in the parent const new_parent_size = old_offset + new_size; - try mf.resizeNode(gpa, node.parent, new_parent_size +| new_parent_size / growth_factor); + try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); try mf.ensureCapacityForSetLocation(gpa); ni.setLocationAssumeCapacity(mf, old_offset, new_size); return; } if (!node.flags.fixed) { // Make space at the end of the parent for this floating node - const last = parent.last.get(mf); + const last = parent.last.unwrap().?.get(mf); const last_offset, const last_size = last.location().resolve(mf); const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); const new_parent_size = new_offset + new_size; if (new_parent_size > old_parent_size) - try mf.resizeNode(gpa, node.parent, new_parent_size +| new_parent_size / growth_factor); + try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); try mf.ensureCapacityForSetLocation(gpa); - const next_ni = node.next; + const next_ni = node.next.unwrap().?; next_ni.get(mf).prev = node.prev; - switch (node.prev) { - .none => parent.first = next_ni, - else => |prev_ni| try prev_ni.setNext(gpa, next_ni, mf), + if (node.prev.unwrap()) |prev_ni| { + try prev_ni.setNext(gpa, .wrap(next_ni), mf); + } else { + parent.first = .wrap(next_ni); } - try parent.last.setNext(gpa, ni, mf); + try parent.last.unwrap().?.setNext(gpa, .wrap(ni), mf); node.prev = parent.last; try ni.setNext(gpa, .none, mf); - parent.last = ni; + parent.last = .wrap(ni); if (node.flags.has_content) { - const parent_file_offset = node.parent.fileLocation(mf, false).offset; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; try mf.moveRange( parent_file_offset + old_offset, parent_file_offset + new_offset, @@ -976,94 +1090,89 @@ fn resizeNode( } // Search for the first floating node following this fixed node var last_fixed_ni = ni; - var first_floating_ni = node.next; + var first_floating_oni = node.next; var shift = new_size - old_size; - var max_shift_align: std.mem.Alignment = .@"1"; + var max_shift_align: Alignment = .@"1"; var direction: enum { forward, reverse } = .forward; while (true) { - assert(last_fixed_ni != .none); const last_fixed = last_fixed_ni.get(mf); assert(last_fixed.flags.fixed); const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); const new_last_fixed_offset = old_last_fixed_offset + shift; - make_space: switch (first_floating_ni) { - else => { - const first_floating = first_floating_ni.get(mf); - const old_first_floating_offset, const first_floating_size = - first_floating.location().resolve(mf); - assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); - if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) - break :make_space; - assert(direction == .forward); - max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); - if (first_floating.flags.fixed) { - shift = max_shift_align.forward(@intCast( - @max(shift, first_floating_size), - )); + if (first_floating_oni.unwrap()) |first_floating_ni| make_space: { + const first_floating = first_floating_ni.get(mf); + const old_first_floating_offset, const first_floating_size = + first_floating.location().resolve(mf); + assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); + if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) + break :make_space; + assert(direction == .forward); + max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); + if (first_floating.flags.fixed) { + shift = max_shift_align.forward(@intCast( + @max(shift, first_floating_size), + )); - // Not enough space, try the next node - last_fixed_ni = first_floating_ni; - first_floating_ni = first_floating.next; - continue; - } - // Move the found floating node to make space for preceding fixed nodes - const last = parent.last.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_first_floating_offset = max_shift_align.forward( - @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), - ); - const new_parent_size = new_first_floating_offset + first_floating_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - node.parent, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); - } - try mf.ensureCapacityForSetLocation(gpa); - if (parent.last != first_floating_ni) { - const old_last = parent.last; - first_floating.prev = old_last; - parent.last = first_floating_ni; - try old_last.setNext(gpa, first_floating_ni, mf); - try last_fixed_ni.setNext(gpa, first_floating.next, mf); - switch (first_floating.next) { - .none => {}, - else => |next_ni| next_ni.get(mf).prev = last_fixed_ni, - } - try first_floating_ni.setNext(gpa, .none, mf); - } - if (first_floating.flags.has_content) { - const parent_file_offset = - node.parent.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_first_floating_offset, - parent_file_offset + new_first_floating_offset, - first_floating_size, - ); - } - first_floating_ni.setLocationAssumeCapacity( - mf, - new_first_floating_offset, - first_floating_size, - ); - // Continue the search after the just-moved floating node - first_floating_ni = last_fixed.next; + // Not enough space, try the next node + last_fixed_ni = first_floating_ni; + first_floating_oni = first_floating.next; continue; - }, - .none => { - assert(direction == .forward); - const new_parent_size = new_last_fixed_offset + last_fixed_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - node.parent, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); + } + // Move the found floating node to make space for preceding fixed nodes + const last = parent.last.unwrap().?.get(mf); + const last_offset, const last_size = last.location().resolve(mf); + const new_first_floating_offset = max_shift_align.forward( + @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), + ); + const new_parent_size = new_first_floating_offset + first_floating_size; + if (new_parent_size > old_parent_size) { + try mf.resizeNode( + gpa, + parent_ni, + new_parent_size +| new_parent_size / growth_factor, + ); + _, old_parent_size = parent.location().resolve(mf); + } + try mf.ensureCapacityForSetLocation(gpa); + if (parent.last.unwrap().? != first_floating_ni) { + const old_last = parent.last.unwrap().?; + first_floating.prev = .wrap(old_last); + parent.last = .wrap(first_floating_ni); + try old_last.setNext(gpa, .wrap(first_floating_ni), mf); + try last_fixed_ni.setNext(gpa, first_floating.next, mf); + if (first_floating.next.unwrap()) |next_ni| { + next_ni.get(mf).prev = .wrap(last_fixed_ni); } - }, + try first_floating_ni.setNext(gpa, .none, mf); + } + if (first_floating.flags.has_content) { + const parent_file_offset = + parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_first_floating_offset, + parent_file_offset + new_first_floating_offset, + first_floating_size, + ); + } + first_floating_ni.setLocationAssumeCapacity( + mf, + new_first_floating_offset, + first_floating_size, + ); + // Continue the search after the just-moved floating node + first_floating_oni = last_fixed.next; + continue; + } else { + assert(direction == .forward); + const new_parent_size = new_last_fixed_offset + last_fixed_size; + if (new_parent_size > old_parent_size) { + try mf.resizeNode( + gpa, + parent_ni, + new_parent_size +| new_parent_size / growth_factor, + ); + _, old_parent_size = parent.location().resolve(mf); + } } try mf.ensureCapacityForSetLocation(gpa); if (last_fixed_ni == ni) { @@ -1077,7 +1186,7 @@ fn resizeNode( } // Move a fixed node into trailing free space if (last_fixed.flags.has_content) { - const parent_file_offset = node.parent.fileLocation(mf, false).offset; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; try mf.moveRange( parent_file_offset + old_last_fixed_offset, parent_file_offset + new_last_fixed_offset, @@ -1086,8 +1195,8 @@ fn resizeNode( } last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); // Retry the previous nodes now that there is enough space - first_floating_ni = last_fixed_ni; - last_fixed_ni = last_fixed.prev; + first_floating_oni = .wrap(last_fixed_ni); + last_fixed_ni = last_fixed.prev.unwrap().?; direction = .reverse; } } @@ -1096,7 +1205,7 @@ fn realignNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - new_alignment: std.mem.Alignment, + new_alignment: Alignment, opts: Node.Index.RealignNodeOptions, ) (Allocator.Error || Io.Cancelable || IoError)!void { mf.nodes_lock.assertUnlocked(); @@ -1109,25 +1218,27 @@ fn realignNode( } const old_offset, const size = node.location().resolve(mf); - if (ni == Node.Index.root) return mf.resizeNode(gpa, ni, size); + const parent_ni = node.parent.unwrap() orelse { + assert(ni == .root); + return mf.resizeNode(gpa, ni, size); + }; const new_size = new_alignment.forward(@intCast(size)); if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); - _, const parent_size = node.parent.location(mf).resolve(mf); - const trailing_end = trailing_end: switch (node.next) { - .none => parent_size, - else => |next_ni| { - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }, + _, const parent_size = parent_ni.location(mf).resolve(mf); + const trailing_end = trailing_end: { + const next_ni = node.next.unwrap() orelse break :trailing_end parent_size; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :trailing_end next_offset; }; if (opts.try_backwards) { const backward_offset = new_alignment.backward(@intCast(old_offset)); - const prev_end = if (node.prev == .none) 0 else prev: { - const prev_offset, const prev_size = node.prev.location(mf).resolve(mf); - break :prev prev_offset + prev_size; + const prev_end = prev_end: { + const prev_ni = node.prev.unwrap() orelse break :prev_end 0; + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :prev_end prev_offset + prev_size; }; if (backward_offset >= prev_end) { @@ -1399,7 +1510,7 @@ fn verify(mf: *MappedFile) void { assert(root.parent == .none); assert(root.prev == .none); assert(root.next == .none); - mf.verifyNode(Node.Index.root); + mf.verifyNode(.root); } fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { @@ -1517,7 +1628,7 @@ test { try testVerifyContent(&mf, d, 0xdd, d_init_size); } - const child_init: []const struct { std.mem.Alignment, usize } = &.{ + const child_init: []const struct { Alignment, usize } = &.{ .{ .@"16", 16 }, .{ .@"1", 1 }, .{ .@"1", 19 }, -- 2.54.0 From 96c9ff1c93532e7c2764449e0cd6f39d4dce2285 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sun, 23 Aug 2026 11:21:31 +0100 Subject: [PATCH 3/6] link.MappedFile: rework node operations This commit is a refactor of the public `MappedFile` API, and a near-total rewrite of its implementation (or at least, the implementation of the node moving and resizing logic). Nodes can be "header", "footer", or "floating" nodes, which dictates how they are positioned relative to their parent; "header" nodes (similar to the old "fixed" nodes) are placed at the start of the parent node, "footer" nodes are placed at the end of the parent node, and "floating" nodes may appear anywhere in the parent. There are separate functions for adding each of these types of node. Notably, when adding a floating node, the API no longer permits the caller to specify *where* these nodes are placed, because floating nodes give the implementation the freedom to make this choice for itself. Another important property is that for header and footer nodes, only their size is aligned to the node's alignment. Their offsets are not necessarily aligned, because they are required to be placed at the start/end of the parent with no additional padding: this constraint already dictates their offset. The `Node.Index.resize` function is replaced with two functions. The first, `ensureMinimumSize`, is permitted for any node, and guarantees only that the node's size is *at least* a particular value, applying exponential growth (`growth_factor`) if necessary---it is essentially equivalent to the helper function `Elf2.ensureNodeSize` which the `Elf2` linker was already making frequent use of. The other, `resizeLeaf`, sets the size of a node *exactly*, but may only be used on leaf nodes. The logic for actually placing nodes in the file, as well as becoming slightly more involved due to handling the new semantics of headers and footers, has also been made more efficient. In particular, the amount of unused "padding" space is, broadly speaking, lower with this implementation than it previously was: empirically, binaries emitted by `Elf2` are about half the size as they were before (in terms of `stat` size, not size on disk), although binaries emitted by `Coff` are around the same size as before (perhaps slightly smaller). I have spoken with Casey about some potential enhancements to `Coff` which, as well as making it more performant, could also slightly improve its file sizes. At least one alignment-related bug, wherein the Linux-specific `FALLOCATE_FL_INSERT_RANGE` path did not respect neighbors' alignment requirements, has been fixed. In order to verify correctness of this new implementation (particularly since neither linker uses footer nodes yet), I wrote a small fuzz test which performs a random sequence of node operations (add, resize, realign), while writing content into (some) leaf nodes. After all operations are complete, it validates that the node structure is valid (headers are tightly packed against the start of the parent node, no two sibling nodes overlap, etc), and ensures that all leaves contain the expected content. Even with our alpha-quality fuzzer implementation, this fuzz test was surprisingly helpful in identifying bugs during development. I suspect `MappedFile` is unusually easy to fuzz, because most code paths can be hit with relatively few nodes, so even purely random fuzzing (as opposed to coverage-guided) is likely to discover any bugs fairly quickly. This fuzz test is in `src/link/MappedFile.zig`, and is referenced by the standard compiler unit tests, so `MappedFile` can be fuzzed at any time by running `zig build test-unit --fuzz`. --- src/link/Coff.zig | 290 ++--- src/link/Elf2.zig | 138 +- src/link/MappedFile.zig | 2688 ++++++++++++++++++++++++++------------- src/main.zig | 1 + 4 files changed, 1990 insertions(+), 1127 deletions(-) diff --git a/src/link/Coff.zig b/src/link/Coff.zig index f83074f558b19e84d803d17251a6d4014abdd2b2..09ea7b6a25e8dcba3cec1d1996266de05863b809 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -533,10 +533,10 @@ pub const Member = struct { errdefer _ = coff.export_table.entries.pop(); _, const old_size = Node.known.longnames_member.location(&coff.mf).resolve(&coff.mf); - const new_size = old_size + name.len + 1; + const new_size = Alignment.@"4".forward(old_size + name.len + 1); assert(new_size < comptime try std.math.powi(u64, 10, max_name_len - 1)); - try Node.known.longnames_member.resize(&coff.mf, gpa, new_size); + try Node.known.longnames_member.resizeLeaf(&coff.mf, gpa, new_size); const name_table_slice = Node.known.longnames_member.slice(&coff.mf); const name_slice = name_table_slice[@intCast(old_size)..][0 .. name.len + 1]; @memcpy(name_slice[0..name.len], name); @@ -1840,34 +1840,20 @@ fn initHeaders( coff.nodes.appendAssumeCapacity(.file); const header_ni = Node.known.header; - assert(header_ni == try coff.mf.addOnlyChildNode(gpa, Node.known.file, .{ + assert(header_ni == try Node.known.file.addOnlyHeaderChild(&coff.mf, gpa, .{ .alignment = coff.mf.flags.block_size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.header); - const signature_ni = Node.known.signature; - assert(signature_ni == try coff.mf.addLastChildNode(gpa, if (is_image or !is_archive) header_ni else Node.known.file, .{ - .size = if (is_image) - msdos_stub.len + std.coff.pe_signature.len - else if (is_archive) - std.coff.archive_signature.len - else - 0, - .alignment = .@"4", - .fixed = true, - })); - coff.nodes.appendAssumeCapacity(.signature); - - const signature_slice = signature_ni.slice(&coff.mf); - if (is_image) { - @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub); - @memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature); - } else if (is_archive) { + const coff_parent_ni: MappedFile.Node.Index = if (is_archive) parent: { + assert(try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{ + .size = std.coff.archive_signature.len, + .alignment = .@"4", + }) == Node.known.signature); + coff.nodes.appendAssumeCapacity(.signature); + const signature_slice = Node.known.signature.slice(&coff.mf); @memcpy(signature_slice, std.coff.archive_signature); - } - const opt_coff_parent_ni = if (is_archive) parent: { const initial_member_count = Member.Index.known_count + @intFromBool(comp.zcu != null); try coff.members.ensureTotalCapacity(gpa, initial_member_count); @@ -1893,46 +1879,54 @@ fn initHeaders( const zcu_member = zcu_mi.get(coff); try zcu_member.initHeader(coff, zcu.main_mod.fully_qualified_name, timestamp); + assert(try zcu_member.content_ni.addOnlyHeaderChild(&coff.mf, gpa, .{ + .size = @sizeOf(std.coff.Header), + .alignment = .@"4", + }) == Node.known.coff_header); + coff.nodes.appendAssumeCapacity(.coff_header); + break :parent zcu_member.content_ni; } + // If we're not generating any code, no more known nodes are used + // These placeholder nodes are placed before the first member - if there are // no other members then the last linker member (longnames) needs to expand // to fill the padding at the end of the file. - assert(Node.known.zcu_member_header == try coff.mf.addNodeAfter(gpa, Node.known.header, .{})); - assert(Node.known.zcu_member == try coff.mf.addNodeAfter(gpa, Node.known.header, .{})); - coff.nodes.appendAssumeCapacity(.placeholder); - coff.nodes.appendAssumeCapacity(.placeholder); + while (coff.nodes.len < Node.known_count) { + _ = try Node.known.header.addHeaderChildAfter(&coff.mf, gpa, .none, .{}); + coff.nodes.appendAssumeCapacity(.placeholder); + } - break :parent null; + return; } else parent: { + assert(try header_ni.addOnlyHeaderChild(&coff.mf, gpa, .{ + .size = if (is_image) msdos_stub.len + std.coff.pe_signature.len else 0, + .alignment = .@"4", + }) == Node.known.signature); + coff.nodes.appendAssumeCapacity(.signature); + if (is_image) { + const signature_slice = Node.known.signature.slice(&coff.mf); + @memcpy(signature_slice[0..msdos_stub.len], &msdos_stub); + @memcpy(signature_slice[signature_slice.len - std.coff.pe_signature.len ..], std.coff.pe_signature); + } + // TODO: Not ideal to have this many placeholder nodes - use two distinct `Node.known` types? while (true) { - const placeholder_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{}); + const placeholder_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .none, .{}); coff.nodes.appendAssumeCapacity(.placeholder); if (placeholder_ni == Node.known.zcu_member) break; } - break :parent Node.known.header; - }; - - const coff_parent_ni = opt_coff_parent_ni orelse { - // If we're not generating any code, no more known nodes are used - while (coff.nodes.len < Node.known_count) { - _ = try coff.mf.addNodeAfter(gpa, Node.known.header, .{}); - coff.nodes.appendAssumeCapacity(.placeholder); - } + assert(try header_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.signature), .{ + .size = @sizeOf(std.coff.Header), + .alignment = .@"4", + }) == Node.known.coff_header); + coff.nodes.appendAssumeCapacity(.coff_header); - return; + break :parent header_ni; }; - const coff_header_ni = Node.known.coff_header; - assert(coff_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ - .size = @sizeOf(std.coff.Header), - .alignment = .@"4", - .fixed = true, - })); - coff.nodes.appendAssumeCapacity(.coff_header); { const coff_header = coff.headerPtr(); coff_header.* = .{ @@ -1955,10 +1949,9 @@ fn initHeaders( } const optional_header_ni = Node.known.optional_header; - assert(optional_header_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(optional_header_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(Node.known.coff_header), .{ .size = optional_header_size, .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.optional_header); if (is_image) { @@ -2067,10 +2060,9 @@ fn initHeaders( } const data_directories_ni = Node.known.data_directories; - assert(data_directories_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(data_directories_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(optional_header_ni), .{ .size = data_directories_size, .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.data_directories); if (is_image) { @@ -2083,9 +2075,8 @@ fn initHeaders( } const section_table_ni = Node.known.section_table; - assert(section_table_ni == try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + assert(section_table_ni == try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(data_directories_ni), .{ .alignment = .@"4", - .fixed = true, })); coff.nodes.appendAssumeCapacity(.section_table); @@ -2093,16 +2084,14 @@ fn initHeaders( if (!is_image) { // TODO: These two nodes could be inside one movable node? - coff.symbol_table.ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + coff.symbol_table.ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(section_table_ni), .{ .alignment = .@"2", - .fixed = true, .moved = true, }); coff.nodes.appendAssumeCapacity(.symbol_table); - coff.symbol_table.strings_ni = try coff.mf.addLastChildNode(gpa, coff_parent_ni, .{ + coff.symbol_table.strings_ni = try coff_parent_ni.addHeaderChildAfter(&coff.mf, gpa, .wrap(coff.symbol_table.ni), .{ .size = @sizeOf(u32), - .fixed = true, .resized = true, }); coff.nodes.appendAssumeCapacity(.string_table); @@ -2149,15 +2138,14 @@ fn initHeaders( } // TODO: Lazily initialize this instead, avoid the extra logic for this in flushMoved / flushResized - coff.import_table.ni = try coff.mf.addLastChildNode( - gpa, - (try coff.objectSectionMapIndex( - .@".idata", - coff.mf.flags.block_size, - .{ .read = true, .initialized = true }, - )).symbol(coff).node(coff), - .{ .alignment = .@"4" }, - ); + const import_table_parent_ni = (try coff.objectSectionMapIndex( + .@".idata", + coff.mf.flags.block_size, + .{ .read = true, .initialized = true }, + )).symbol(coff).node(coff); + coff.import_table.ni = try import_table_parent_ni.addFloatingChild(&coff.mf, gpa, .{ + .alignment = .@"4", + }); coff.nodes.appendAssumeCapacity(.import_directory_table); coff.export_table.ni = (try coff.pseudoSectionMapIndex( @@ -2166,15 +2154,10 @@ fn initHeaders( .{ .read = true, .initialized = true }, )).symbol(coff).node(coff); - coff.export_table.export_directory_table_ni = try coff.mf.addLastChildNode( - gpa, - coff.export_table.ni, - .{ - .size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1, - .moved = true, - .fixed = true, - }, - ); + coff.export_table.export_directory_table_ni = try coff.export_table.ni.addHeaderChildAfter(&coff.mf, gpa, coff.export_table.ni.last(&coff.mf), .{ + .size = @sizeOf(std.coff.ExportDirectoryTable) + file_name.len + 1, + .moved = true, + }); coff.nodes.appendAssumeCapacity(.export_directory_table); const name_index = @sizeOf(std.coff.ExportDirectoryTable); @@ -2182,7 +2165,7 @@ fn initHeaders( @memcpy(table_slice[name_index..][0..file_name.len], file_name[0..file_name.len]); @memset(table_slice[name_index + file_name.len ..], 0); - const export_address_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + const export_address_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportAddressTableEntry), .moved = true, }); @@ -2198,19 +2181,19 @@ fn initHeaders( export_address_table_sym.section_number = coff.getNode(coff.export_table.ni).pseudo_section.symbol(coff).get(coff).section_number; - coff.export_table.name_pointer_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.name_pointer_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportNamePointerTableEntry), .moved = true, }); coff.nodes.appendAssumeCapacity(.export_name_pointer_table); - coff.export_table.ordinal_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.ordinal_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(std.coff.ExportOrdinalTableEntry), .moved = true, }); coff.nodes.appendAssumeCapacity(.export_ordinal_table); - coff.export_table.name_table_ni = try coff.mf.addLastChildNode(gpa, coff.export_table.ni, .{ + coff.export_table.name_table_ni = try coff.export_table.ni.addFloatingChild(&coff.mf, gpa, .{ .alignment = .of(u8), .moved = true, }); @@ -2303,9 +2286,8 @@ pub fn initBuiltins(coff: *Coff) !void { const list_len_si = try coff.globalSymbol(.{ .name = list.global, .type = .data }); const list_len_sym = list_len_si.get(coff); list_len_sym.setExtra(.{ .size = addr_info.size }); - list_len_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, start_sym.ni.unwrap().?, .{ + list_len_sym.ni = .wrap(try start_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{ .size = addr_info.size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_len_si }); list_len_sym.section_number = start_sym.section_number; @@ -2325,9 +2307,8 @@ pub fn initBuiltins(coff: *Coff) !void { const list_end_si = coff.addSymbolAssumeCapacity(); const list_end_sym = list_end_si.get(coff); list_end_sym.setExtra(.{ .size = addr_info.size }); - list_end_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, end_sym.ni.unwrap().?, .{ + list_end_sym.ni = .wrap(try end_sym.ni.unwrap().?.addHeaderChildAfter(&coff.mf, gpa, .none, .{ .size = addr_info.size, - .fixed = true, })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_end_si }); list_end_sym.section_number = start_sym.section_number; @@ -2742,7 +2723,7 @@ fn getOrPutSymbolName(coff: *Coff, name: []const u8, opt_string: ?String) !Symbo const string_index = coff.symbol_table.strings_ni.location(&coff.mf).resolve(&coff.mf)[1]; string_gop.value_ptr.* = @fromBackingInt(@intCast(string_index)); - try coff.symbol_table.strings_ni.resize(&coff.mf, gpa, string_index + name.len + 1); + try coff.symbol_table.strings_ni.resizeLeaf(&coff.mf, gpa, string_index + name.len + 1); const slice = coff.symbol_table.strings_ni.slice(&coff.mf); @memcpy(slice[@intCast(string_index)..][0..name.len], name); slice[@intCast(string_index + name.len)] = 0; @@ -2967,24 +2948,22 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind, const comp = coff.base.comp; const gpa = comp.gpa; - // TODO: These two nodes could to be inside a movable node if kind == .coff|.import - const header_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{ + const header_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, Node.known.file.last(&coff.mf), .{ .size = @sizeOf(std.coff.ArchiveMemberHeader), .alignment = .@"2", - .fixed = true, .moved = true, }); - const content_ni = try coff.mf.addLastChildNode(gpa, Node.known.file, .{ - // The actual alignment required by the spec is 2, but to allow aligned access to - // the various COFF data structures in-place during linking we overalign - .alignment = switch (kind) { - .coff => .@"4", - else => .@"2", - }, - .size = size, + // The actual alignment required by the spec is 2, but to allow aligned access to + // the various COFF data structures in-place during linking we overalign + const content_align: Alignment = switch (kind) { + .first_linker, .second_linker, .longnames, .coff => .@"4", + else => .@"2", + }; + const content_ni = try Node.known.file.addHeaderChildAfter(&coff.mf, gpa, .wrap(header_ni), .{ + .alignment = content_align, + .size = content_align.forward(size), .resized = size > 0, - .fixed = true, }); const mi: Member.Index = @fromBackingInt(@intCast(coff.members.items.len)); @@ -3010,7 +2989,7 @@ fn addMemberAssumeCapacity(coff: *Coff, kind: std.coff.ArchiveMemberHeader.Kind, const old_size = Node.known.second_linker_member.location(&coff.mf).resolve(&coff.mf)[1]; const old_header_size = new_num_members * @sizeOf(u32); const trailing_size: usize = @intCast(old_size - old_header_size); - try Node.known.second_linker_member.resize(&coff.mf, gpa, old_size + @sizeOf(u32)); + try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, old_size + @sizeOf(u32)); const slice = Node.known.second_linker_member.slice(&coff.mf); @memmove( @@ -3048,7 +3027,7 @@ fn appendMemberSymbolString( name: []const u8, offset: u64, ) !void { - try strings_ni.resize(&coff.mf, coff.base.comp.gpa, new_size); + try strings_ni.resizeLeaf(&coff.mf, coff.base.comp.gpa, new_size); const name_slice = strings_ni.slice(&coff.mf)[offset..][0 .. name.len + 1]; @memcpy(name_slice[0..name.len], name); name_slice[name.len] = 0; @@ -3081,7 +3060,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void { { const old_header_size: usize = @intCast(@sizeOf(u32) + @backingInt(mfli) * @sizeOf(u32)); const new_header_size: usize = @intCast(old_header_size + @sizeOf(u32)); - try Node.known.first_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size); + try Node.known.first_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size)); const slice = Node.known.first_linker_member.slice(&coff.mf); @memmove(slice[new_header_size..][0..coff.lib_string_len], slice[old_header_size..][0..coff.lib_string_len]); @@ -3095,7 +3074,7 @@ fn ensureMemberSymbol(coff: *Coff, mi: Member.Index, name: String) !void { const num_members = coff.targetLoad(coff.secondLinkerMemberNumMembersPtr()); const old_header_size = 2 * @sizeOf(u32) + num_members * @sizeOf(u32) + @backingInt(mfli) * @sizeOf(u16); const new_header_size = old_header_size + @sizeOf(u16); - try Node.known.second_linker_member.resize(&coff.mf, gpa, new_header_size + new_string_table_size); + try Node.known.second_linker_member.resizeLeaf(&coff.mf, gpa, Alignment.@"4".forward(new_header_size + new_string_table_size)); const old_needs_sort = coff.pending_members.get(Member.Index.second) != null; const needs_sort = old_needs_sort or (if (coff.lib_string_table.items.len > 0) @@ -3202,7 +3181,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { const new_num_symbols = old_num_symbols + 1 + num_aux_symbols; coff.targetStore(&coff.headerPtr().number_of_symbols, new_num_symbols); - try coff.symbol_table.ni.resize(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf()); + try coff.symbol_table.ni.resizeLeaf(&coff.mf, gpa, new_num_symbols * std.coff.Symbol.sizeOf()); sti.* = .wrap(old_num_symbols); si.flushSymbolTableIndex(coff); @@ -3365,13 +3344,13 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S const section_index = coff.targetLoad(&coff_header.number_of_sections); const section_table_len = section_index + 1; coff.targetStore(&coff_header.number_of_sections, section_table_len); - try Node.known.section_table.resize( + try Node.known.section_table.resizeLeaf( &coff.mf, gpa, @sizeOf(std.coff.SectionHeader) * section_table_len, ); - const ni = try coff.mf.addLastChildNode(gpa, coff.sectionParent(), .{ + const ni = try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{ .alignment = coff.mf.flags.block_size, .moved = true, .bubbles_moved = false, @@ -3507,7 +3486,7 @@ fn pseudoSectionMapIndex( try coff.nodes.ensureUnusedCapacity(gpa, 1); try coff.symbols.ensureUnusedCapacity(gpa, 1); - const ni = try coff.mf.addLastChildNode(gpa, parent.node(coff), .{ .alignment = alignment }); + const ni = try parent.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment }); const si = coff.addSymbolAssumeCapacity(); pseudo_section_gop.value_ptr.* = si; const sym = si.get(coff); @@ -3577,12 +3556,8 @@ fn objectSectionMapIndex( .eq => unreachable, .gt => prev_oni = .wrap(next_ni), }; - const ni = if (prev_oni.unwrap()) |prev_ni| try coff.mf.addNodeAfter(gpa, prev_ni, .{ + const ni = try parent_ni.addHeaderChildAfter(&coff.mf, gpa, prev_oni, .{ .alignment = alignment, - .fixed = true, - }) else try coff.mf.addFirstChildNode(gpa, parent_ni, .{ - .alignment = alignment, - .fixed = true, }); const si = coff.addSymbolAssumeCapacity(); object_section_gop.value_ptr.* = si; @@ -3600,13 +3575,13 @@ fn objectSectionMapIndex( const parent_alignment = parent_ni.alignment(&coff.mf); if (alignment.compare(.gt, parent_alignment)) { log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment }); - try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try parent_ni.realign(&coff.mf, gpa, alignment); } const old_alignment = sym.ni.unwrap().?.alignment(&coff.mf); if (alignment.compare(.gt, old_alignment)) { log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment }); - try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment); } try coff.verifyParentSectionAttributes( @@ -3763,18 +3738,14 @@ fn addRelocAssumeCapacity( coff.targetStore(&aux_ptr.number_of_relocations, new_num_relocations); if (section.relocation_table_ni.unwrap()) |relocation_table_ni| { - try relocation_table_ni.resize(&coff.mf, gpa, new_size); + try relocation_table_ni.resizeLeaf(&coff.mf, gpa, new_size); } else { - section.relocation_table_ni = .wrap(try coff.mf.addLastChildNode( - gpa, - coff.sectionParent(), - .{ - .size = new_size, - .alignment = .@"2", - .moved = true, - .resized = true, - }, - )); + section.relocation_table_ni = .wrap(try coff.sectionParent().addFloatingChild(&coff.mf, gpa, .{ + .size = new_size, + .alignment = .@"2", + .moved = true, + .resized = true, + })); coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn }); } @@ -4677,9 +4648,10 @@ fn loadObject( for (sections) |*section| { if (section.parent_si == .null) continue; - const ni = try coff.mf.addLastChildNode(gpa, section.parent_si.node(coff), .{ - .size = section.header.size_of_raw_data, - .alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), + const alignment: Alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1); + const ni = try section.parent_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ + .size = alignment.forward(section.header.size_of_raw_data), + .alignment = alignment, .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(coff.input_sections.items.len)) }); @@ -5471,7 +5443,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde const sec_si = try coff.navSection(zcu, nav.resolved.?); try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = .fromIp(zcu.navAlignment(nav_index)), .moved = true, }); @@ -5510,21 +5482,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde } if (nav.resolved.?.@"linksection".unwrap()) |_| { - try ni.resize(&coff.mf, gpa, si.get(coff).extra.size); - var parent_ni = ni; - while (true) { - parent_ni = parent_ni.parent(&coff.mf).unwrap().?; - switch (coff.getNode(parent_ni)) { - else => unreachable, - .image_section, .pseudo_section => break, - .object_section => { - var child_it = parent_ni.reverseChildren(&coff.mf); - const last_offset, const last_size = - child_it.next().?.location(&coff.mf).resolve(&coff.mf); - try parent_ni.resize(&coff.mf, gpa, last_offset + last_size); - }, - } - } + try ni.resizeLeaf(&coff.mf, gpa, si.get(coff).extra.size); } } @@ -5596,7 +5554,7 @@ fn updateFuncInner( if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const mod = zcu.navFileScope(func.owner_nav).mod.?; const target = &mod.resolved_target.result; - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = switch (nav.resolved.?.@"align") { .none => switch (mod.optimize_mode) { .debug, @@ -5900,15 +5858,17 @@ pub fn flush( coff.symbol_table.pending_shrink = false; const number_of_symbols = coff.targetLoad(&coff.headerPtr().number_of_symbols); - coff.symbol_table.ni.shrink( + coff.symbol_table.ni.resizeLeaf( &coff.mf, comp.gpa, number_of_symbols * std.coff.Symbol.sizeOf(), - true, - ) catch |err| return comp.link_diags.fail( - "linker failed to compact symbol table: {t}", - .{err}, - ); + ) catch |err| switch (err) { + else => |e| return e, + error.MappedFileIo => return comp.link_diags.fail( + "linker failed to compact symbol table: {t}", + .{coff.mf.io_err.?}, + ), + }; } while (try coff.idle(tid)) {} @@ -6211,7 +6171,7 @@ fn flushUav( try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const sym = si.get(coff); - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .alignment = .fromIp(uav_align), .moved = true, }); @@ -6503,7 +6463,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_hint_name_align: Alignment = .@"2"; if (!gop.found_existing) { errdefer _ = coff.import_table.entries.pop(); - try coff.import_table.ni.resize( + try coff.import_table.ni.resizeLeaf( &coff.mf, gpa, @sizeOf(std.coff.ImportDirectoryEntry) * (gop.index + 2), @@ -6511,12 +6471,12 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_hint_name_table_len = import_hint_name_align.forward(lib_name.len + ".dll".len + 1); const idata_section_ni = coff.import_table.ni.parent(&coff.mf).unwrap().?; - const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_lookup_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, .moved = true, }); - const import_address_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_address_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, .moved = true, @@ -6530,7 +6490,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { import_address_table_sym.section_number = coff.getNode(idata_section_ni).object_section.symbol(coff).get(coff).section_number; } - const import_hint_name_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ + const import_hint_name_table_ni = try idata_section_ni.addFloatingChild(&coff.mf, gpa, .{ .size = import_hint_name_table_len, .alignment = import_hint_name_align, .moved = true, @@ -6586,9 +6546,9 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { gop.value_ptr.len = import_symbol_index + 1; const new_symbol_table_size = addr_info.size * (import_symbol_index + 2); - try gop.value_ptr.import_lookup_table_ni.resize(&coff.mf, gpa, new_symbol_table_size); + try gop.value_ptr.import_lookup_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size); const import_address_table_ni = gop.value_ptr.import_address_table_si.node(coff); - try import_address_table_ni.resize(&coff.mf, gpa, new_symbol_table_size); + try import_address_table_ni.resizeLeaf(&coff.mf, gpa, new_symbol_table_size); const opt_imp_name = import.name.toSlice(coff); const opt_import_hint_name_index = if (opt_imp_name) |imp_name| blk: { @@ -6596,7 +6556,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { gop.value_ptr.hint_name_len = @intCast( import_hint_name_align.forward(import_hint_name_index + 2 + imp_name.len + 1), ); - try gop.value_ptr.import_hint_name_table_ni.resize(&coff.mf, gpa, gop.value_ptr.hint_name_len); + try gop.value_ptr.import_hint_name_table_ni.resizeLeaf(&coff.mf, gpa, gop.value_ptr.hint_name_len); break :blk import_hint_name_index; } else null; @@ -6671,9 +6631,9 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { else => |tag| @panic(@tagName(tag)), .AMD64 => { const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 }; - const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni.unwrap().?, .{ + const ni = try parent_sym.ni.unwrap().?.addFloatingChild(&coff.mf, gpa, .{ .alignment = alignment, - .size = init.len, + .size = alignment.forward(init.len), }); @memcpy(ni.slice(&coff.mf)[0..init.len], &init); sym.ni = .wrap(ni); @@ -6824,7 +6784,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { .code => .text, .const_data => .rdata, }; - const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ .moved = true }); + const ni = try sec_si.node(coff).addFloatingChild(&coff.mf, gpa, .{ .moved = true }); coff.nodes.appendAssumeCapacity(switch (lazy.kind) { .code => .{ .lazy_code = @fromBackingInt(@intCast(lmr.index)) }, .const_data => .{ .lazy_const_data = @fromBackingInt(@intCast(lmr.index)) }, @@ -7480,7 +7440,7 @@ fn updateExportInner( if (new_name_table_size > std.math.maxInt(@FieldType(ExportTable.Entry, "name_index"))) return coff.base.comp.link_diags.fail("exports name table limit reached", .{}); - try coff.export_table.name_table_ni.resize(&coff.mf, gpa, new_name_table_size); + try coff.export_table.name_table_ni.resizeLeaf(&coff.mf, gpa, new_name_table_size); const name_table_slice = coff.export_table.name_table_ni.slice(&coff.mf); @memcpy(name_table_slice[name_index..][0 .. name.len + 1], name[0 .. name.len + 1]); @@ -7503,19 +7463,19 @@ fn updateExportInner( // TODO: These should all be resized ahead of time to fit all exports // after https://github.com/ziglang/zig/issues/23616 - try coff.export_table.export_address_table_si.node(coff).resize( + try coff.export_table.export_address_table_si.node(coff).resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportAddressTableEntry), ); - try coff.export_table.name_pointer_table_ni.resize( + try coff.export_table.name_pointer_table_ni.resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportNamePointerTableEntry), ); - try coff.export_table.ordinal_table_ni.resize( + try coff.export_table.ordinal_table_ni.resizeLeaf( &coff.mf, gpa, export_count * @sizeOf(std.coff.ExportOrdinalTableEntry), @@ -7746,12 +7706,12 @@ pub fn printNode( { const mf_node = &coff.mf.nodes.items[@backingInt(ni)]; const off, const size = mf_node.location().resolve(&coff.mf); - try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}\n", .{ + try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}\n", .{ @backingInt(ni), off, size, mf_node.flags.alignment.toByteUnits(), - if (mf_node.flags.fixed) " fixed" else "", + mf_node.flags.position, if (mf_node.flags.moved) " moved" else "", if (mf_node.flags.resized) " resized" else "", if (mf_node.flags.has_content) " has_content" else "", diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 7f23494f4fe45d74c749f6b65b255b50a20781c7..0acc9a6fe86b17f887eb41a55f35ea199bb1dca7 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -550,7 +550,7 @@ const Section = struct { } const ni = shndx.get(elf).ni; if (min_align.compare(.gt, ni.alignment(&elf.mf))) { - try ni.realign(&elf.mf, elf.base.comp.gpa, min_align, .{}); + try ni.realign(&elf.mf, elf.base.comp.gpa, min_align); } switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .elf => {}, @@ -583,7 +583,7 @@ const Section = struct { break :need_size cur_size + need_additional * ent_size; }, }; - try elf.ensureNodeSize(node, need_size); + try node.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, need_size); } /// Asserts that `rela_shndx` is a `SHT_RELA` section and deletes the `ElfN.Rela` entry at @@ -1787,6 +1787,8 @@ const SymbolReloc = struct { }; fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { + const gpa = elf.base.comp.gpa; + const min_buckets = max_dynsym_count / 2; const cur_dynsym_count: u32 = switch (elf.shdrPtr(elf.shndx.dynsym)) { @@ -1807,7 +1809,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { // We don't need to add any buckets, but we still need to make sure the section is large // enough to fit `max_dynsym_count` chains. const need_size = @sizeOf(info.Header()) + (nbucket + max_dynsym_count) * 4; - try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size); + try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); return; } // We need more buckets, so we'll have to rebuild the hash table. @@ -1819,7 +1821,7 @@ fn ensureDynsymHashCapacity(elf: *Elf, max_dynsym_count: u32) Error!void { { const need_size = @sizeOf(info.Header()) + (new_nbucket + max_dynsym_count) * 4; - try elf.ensureNodeSize(elf.shndx.hash.get(elf).ni, need_size); + try elf.shndx.hash.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } elf.mf.nodes_lock.lock(); @@ -1965,7 +1967,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe const need_node_size: u64 = switch (elf.shdrPtr(.symtab)) { inline else => |shdr, class| elf.targetLoad(&shdr.size) + len * @sizeOf(class.ElfN().Sym), }; - try elf.ensureNodeSize(Section.Index.symtab.get(elf).ni, need_node_size); + try Section.Index.symtab.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_node_size); } switch (kind) { @@ -1988,7 +1990,7 @@ fn ensureUnusedSymbolCapacity(elf: *Elf, len: u32, kind: enum { all_local, maybe const dynsym_cur_len: u32 = @intCast(@divExact(dynsym_cur_size, dynsym_ent_size)); const dynsym_need_size: u64 = (dynsym_cur_len + len) * dynsym_ent_size; - try elf.ensureNodeSize(elf.shndx.dynsym.get(elf).ni, dynsym_need_size); + try elf.shndx.dynsym.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, dynsym_need_size); try elf.ensureDynsymHashCapacity(dynsym_cur_len + len); @@ -2010,19 +2012,19 @@ fn ensureUnusedPltCapacity(elf: *Elf, len: u32) Error!void { // Ensure the `.plt` section's node is big enough: { const need_size: usize = plt.entry_size * (1 + need_plt_count); - try elf.ensureNodeSize(elf.shndx.plt.get(elf).ni, need_size); + try elf.shndx.plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } // If there is a `.got.plt` section, ensure its node is big enough if (plt.got_plt) |got_plt| { const need_size: usize = elf.targetPtrSize() * (got_plt.header_entries + need_plt_count); - try elf.ensureNodeSize(elf.shndx.got_plt.get(elf).ni, need_size); + try elf.shndx.got_plt.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } // If there is a `.plt.sec` section, ensure its node is big enough if (plt.plt_sec) |plt_sec| { const need_size: usize = plt_sec.entry_size * need_plt_count; - try elf.ensureNodeSize(elf.shndx.plt_sec.get(elf).ni, need_size); + try elf.shndx.plt_sec.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_size); } } /// Given an index into the PLT, returns whether that PLT entry is dead, meaning it may be reused at @@ -2989,7 +2991,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol .code => .{ .text, .FUNC }, .const_data => .{ .rodata, .OBJECT }, }; - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{}); + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{}); var name_buf: [64]u8 = undefined; const name = std.fmt.bufPrint( &name_buf, @@ -3248,7 +3250,7 @@ const StringTable = struct { break :size .{ old_size, new_size }; }, }; - try elf.ensureNodeSize(ni, new_size); + try ni.ensureMinimumSize(&elf.mf, gpa, new_size); const slice = ni.slice(&elf.mf)[old_size..]; @memcpy(slice[0..key.len], key); slice[key.len] = 0; @@ -3611,10 +3613,11 @@ fn initHeaders( if (is_archive) { elf.nodes.appendAssumeCapacity(.archive); - const archive_header_ni = try elf.mf.addOnlyChildNode(gpa, .root, .{ + const archive_ni: MappedFile.Node.Index = .root; + + const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, .alignment = .@"2", - .fixed = true, .next_moved = true, .bubbles_moved = false, .enable_next_moved = true, @@ -3633,7 +3636,7 @@ fn initHeaders( .ar_fmag = std.elf.ARFMAG.*, }; - elf.ni.elf = try elf.mf.addLastChildNode(gpa, .root, .{ + elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align.max(.@"2"), .next_moved = true, .bubbles_moved = false, @@ -3657,19 +3660,18 @@ fn initHeaders( // the rodata segment. Although to my knowledge neither ELF nor any ELF-based OS strictly // requires this, it is highly conventional and therefore sometimes relied upon. if (@"type" != .REL) { - elf.ni.rodata = try elf.mf.addOnlyChildNode(gpa, elf.ni.elf, .{ + // This node will contain the ehdr, which must be at the start of the ELF file, so this + // node must itself be a header of the `.elf` node. + elf.ni.rodata = try elf.ni.elf.addOnlyHeaderChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for `elf.ni.phdr` to be placed inside this node .alignment = node_block_align.max(addr_align), - // This node will contain the ehdr, which must be at the start of the ELF file, so this - // node must itself be fixed. - .fixed = true, .moved = true, .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.rodata }); elf.phdrs.items[phndx.rodata] = .wrap(elf.ni.rodata); - elf.ni.phdr = try elf.mf.addOnlyChildNode(gpa, elf.ni.rodata, .{ + elf.ni.phdr = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .size = @as(u64, phnum) * entsize.ph, .alignment = addr_align, // keep in sync with `elf.ni.rodata` alignment above .moved = true, @@ -3679,7 +3681,7 @@ fn initHeaders( elf.nodes.appendAssumeCapacity(.{ .segment = phndx.phdr }); elf.phdrs.items[phndx.phdr] = .wrap(elf.ni.phdr); - elf.ni.text = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + elf.ni.text = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, @@ -3687,7 +3689,7 @@ fn initHeaders( elf.nodes.appendAssumeCapacity(.{ .segment = phndx.text }); elf.phdrs.items[phndx.text] = .wrap(elf.ni.text); - elf.ni.data = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + elf.ni.data = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for `elf.ni.data_rel_ro` to be placed inside this node .alignment = node_block_align.max(addr_align), .moved = true, @@ -3697,7 +3699,7 @@ fn initHeaders( elf.phdrs.items[phndx.data] = .wrap(elf.ni.data); if (plt.got_plt == null) { - const plt_ni = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ + const plt_ni = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, @@ -3706,7 +3708,7 @@ fn initHeaders( elf.phdrs.items[phndx.plt] = .wrap(plt_ni); } - elf.ni.data_rel_ro = try elf.mf.addOnlyChildNode(gpa, elf.ni.data, .{ + elf.ni.data_rel_ro = try elf.ni.data.addFloatingChild(&elf.mf, gpa, .{ // Must be at least `addr_align` for the `PT_DYNAMIC` node to be placed inside this one // later (if `have_dynamic_section`). Keep in sync with `elf.ni.data` alignment above. .alignment = node_block_align.max(addr_align), @@ -3717,7 +3719,7 @@ fn initHeaders( elf.phdrs.items[phndx.relro] = .wrap(elf.ni.data_rel_ro); if (comp.config.any_non_single_threaded) { - elf.ni.tls = .wrap(try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + elf.ni.tls = .wrap(try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, @@ -3738,10 +3740,9 @@ fn initHeaders( .REL => elf.ni.elf, .DYN, .EXEC => elf.ni.rodata, }; - elf.ni.ehdr = try elf.mf.addFirstChildNode(gpa, parent_ni, .{ + elf.ni.ehdr = try parent_ni.addOnlyHeaderChild(&elf.mf, gpa, .{ .size = @sizeOf(ElfN.Ehdr), .alignment = addr_align, - .fixed = true, }); elf.nodes.appendAssumeCapacity(.ehdr); @@ -3793,8 +3794,8 @@ fn initHeaders( }, } - elf.ni.shdr = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ - .size = 1 * entsize.sh, // as above, only the SHN_UNDEF initially + elf.ni.shdr = try elf.ni.elf.addFloatingChild(&elf.mf, gpa, .{ + .size = node_block_align.forward(1 * entsize.sh), // as above, only the SHN_UNDEF initially .alignment = addr_align.max(node_block_align), .moved = true, .resized = true, @@ -4109,7 +4110,7 @@ fn initHeaders( .node_align = node_block_align, }); if (maybe_interp) |interp| { - const interp_ni = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + const interp_ni = try elf.ni.rodata.addFloatingChild(&elf.mf, gpa, .{ .size = interp.len + 1, .moved = true, .resized = true, @@ -4130,7 +4131,7 @@ fn initHeaders( } if (have_dynamic_section) { assert(elf.ni.data_rel_ro.alignment(&elf.mf).compare(.gte, addr_align)); - const dynamic_ni = try elf.mf.addLastChildNode(gpa, elf.ni.data_rel_ro, .{ + const dynamic_ni = try elf.ni.data_rel_ro.addFloatingChild(&elf.mf, gpa, .{ .alignment = addr_align, .moved = true, .bubbles_moved = false, @@ -4208,7 +4209,7 @@ fn initHeaders( .flags = .{ .ALLOC = true, .WRITE = true }, .link = dynstr_shndx.toSection().?, .entsize = @intCast(addr_align.toByteUnits() * 2), - .node_align = addr_align, + .addralign = addr_align, }); switch (elf.targetDynsymHashInfo()) { inline else => |info| { @@ -4869,7 +4870,7 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo { // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`. }; } -fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { +pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child { const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer; const Child = pointer_ty.child; const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child); @@ -5051,7 +5052,7 @@ fn mapInputSection(elf: *Elf, opts: struct { const name_shstrtab = try elf.string(.shstrtab, name); const gop = try elf.section_by_name.getOrPut(gpa, name_shstrtab); if (gop.found_existing) { - break :existing @fromBackingInt(@intCast(gop.index)); + break :existing @fromBackingInt(@intCast(gop.index + 1)); // +1 to account for SHN_UDNEF } errdefer assert(elf.section_by_name.pop().?.key == name_shstrtab); const parent_node: MappedFile.Node.Index = parent: { @@ -5172,7 +5173,7 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node }, }; try shndx.ensureAligned(elf, alignment); - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ .alignment = alignment, }); nav_gop.value_ptr.* = .{ @@ -5216,7 +5217,7 @@ fn uavMapIndex( if (!uav_gop.found_existing) { const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs try shndx.ensureAligned(elf, resolved_align); - const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ + const node = try shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ .moved = true, // see assert at end of `genUav` .alignment = resolved_align, }); @@ -5245,7 +5246,7 @@ fn uavMapIndex( const shndx = elf.getNode(node.parent(&elf.mf).unwrap().?).section; try shndx.ensureAligned(elf, resolved_align); if (resolved_align.order(node.alignment(&elf.mf)).compare(.gt)) { - try node.realign(&elf.mf, gpa, resolved_align, .{}); + try node.realign(&elf.mf, gpa, resolved_align); } } return umi; @@ -5462,9 +5463,10 @@ fn loadObject( .extra = undefined, }; if (elf.ni.elf != .root) { + const archive_ni: MappedFile.Node.Index = .root; try elf.nodes.ensureUnusedCapacity(gpa, 1); - input.extra = .{ .node = try elf.mf.addLastChildNode(gpa, .root, .{ - .size = fl.size + @sizeOf(std.elf.ar_hdr), + input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{ + .size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)), .alignment = .@"2", .next_moved = true, .bubbles_moved = false, @@ -5646,12 +5648,24 @@ fn loadObject( std.math.ceilPowerOfTwoAssert(usize, @intCast(@max(section.shdr.addralign, 1))), ); try opts.shndx.ensureAligned(elf, need_align); - const ni = try elf.mf.addLastChildNode(gpa, opts.shndx.get(elf).ni, .{ - .size = section.shdr.size, + const add_node_opts: MappedFile.Node.AddOptions = .{ + .size = need_align.forward(section.shdr.size), .alignment = need_align, .moved = true, // see assert at end of `flushInputSection` - .fixed = opts.node_fixed, - }); + }; + const ni = if (opts.node_fixed) ni: { + const shndx_ni = opts.shndx.get(elf).ni; + const after_oni: MappedFile.Node.Index.Optional = after: { + const last_ni = shndx_ni.last(&elf.mf).unwrap() orelse break :after .none; + break :after switch (last_ni.position(&elf.mf)) { + .header => .wrap(last_ni), + .footer, .floating => .none, + }; + }; + break :ni try shndx_ni.addHeaderChildAfter(&elf.mf, gpa, after_oni, add_node_opts); + } else ni: { + break :ni try opts.shndx.get(elf).ni.addFloatingChild(&elf.mf, gpa, add_node_opts); + }; elf.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(elf.input_sections.items.len)), }); @@ -6019,8 +6033,8 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We have a copy relocation for this global, but the amount of space we // reserved for it could be too small or underaligned! try Section.Index.data.ensureAligned(elf, gop.value_ptr.alignment); - try copied_global.node.resize(&elf.mf, gpa, gop.value_ptr.size); - try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment, .{}); + try copied_global.node.resizeLeaf(&elf.mf, gpa, gop.value_ptr.alignment.forward(gop.value_ptr.size)); + try copied_global.node.realign(&elf.mf, gpa, gop.value_ptr.alignment); const global_ptr = elf.globalByName(name).?; switch (elf.symPtr(global_ptr.symtab_index)) { inline else => |sym_ptr| elf.targetStore(&sym_ptr.size, @intCast(gop.value_ptr.size)), @@ -6267,7 +6281,7 @@ fn prepareDynamic(elf: *Elf) Error!void { const dynamic_size = dynamic_len * 2 * elf.targetPtrSize(); - try elf.shndx.dynamic.get(elf).ni.resize(&elf.mf, comp.gpa, dynamic_size); + try elf.shndx.dynamic.get(elf).ni.resizeLeaf(&elf.mf, comp.gpa, dynamic_size); switch (elf.shdrPtr(elf.shndx.dynamic)) { inline else => |shdr| elf.targetStore(&shdr.size, @intCast(dynamic_size)), } @@ -6393,7 +6407,6 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { addralign: Alignment = .@"1", entsize: std.elf.Word = 0, node_align: Alignment = .@"1", - fixed: bool = false, }) Error!Section.Index { switch (opts.type) { .NULL => assert(opts.size == 0), @@ -6437,14 +6450,15 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { break :shndx .{ @fromBackingInt(shndx), @as(u64, elf.targetLoad(&ehdr.shentsize)) * @as(u64, shnum) }; }, }; - try elf.ensureNodeSize(elf.ni.shdr, new_shdr_size); - const ni = try elf.mf.addLastChildNode(gpa, switch (elf.ehdrType()) { + try elf.ni.shdr.ensureMinimumSize(&elf.mf, gpa, new_shdr_size); + const parent_ni = switch (elf.ehdrType()) { .REL => elf.ni.elf, .EXEC, .DYN => segment_ni, - }, .{ - .size = opts.size, + }; + assert(opts.addralign.check(opts.size)); + const ni = try parent_ni.addFloatingChild(&elf.mf, gpa, .{ + .size = opts.node_align.forward(opts.size), .alignment = opts.addralign.max(opts.node_align), - .fixed = opts.fixed, .resized = opts.size > 0, }); const addr = elf.computeNodeVAddr(ni); @@ -6530,7 +6544,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize) .NONE, _ => unreachable, inline else => |ct_class| (elf.got.count() + new_got_entries) * @sizeOf(ct_class.ElfN().Addr), }; - try elf.ensureNodeSize(elf.shndx.got.get(elf).ni, need_got_size); + try elf.shndx.got.get(elf).ni.ensureMinimumSize(&elf.mf, gpa, need_got_size); if (elf.shndx.dynamic != .UNDEF) { try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, new_got_entries); @@ -7284,8 +7298,8 @@ fn maybeAddCopyRelocation(elf: *Elf, global_name: String(.strtab)) Error!bool { try Section.Index.data.ensureAligned(elf, dso_global.alignment); try elf.nodes.ensureUnusedCapacity(gpa, 1); - const node = try elf.mf.addLastChildNode(gpa, Section.Index.data.get(elf).ni, .{ - .size = dso_global.size, + const node = try Section.Index.data.get(elf).ni.addFloatingChild(&elf.mf, gpa, .{ + .size = dso_global.alignment.forward(dso_global.size), .alignment = dso_global.alignment, }); errdefer comptime unreachable; @@ -8868,12 +8882,12 @@ pub fn printNode( { const mf_node = &elf.mf.nodes.items[@backingInt(ni)]; const off, const size = mf_node.location().resolve(&elf.mf); - try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x}{s}{s}{s}{s}{s}\n", .{ + try w.print(" index={d} offset=0x{x} size=0x{x} align=0x{x} {t}{s}{s}{s}{s}\n", .{ @backingInt(ni), off, size, mf_node.flags.alignment.toByteUnits(), - if (mf_node.flags.fixed) " fixed" else "", + mf_node.flags.position, if (mf_node.flags.moved) " moved" else "", if (mf_node.flags.next_moved) " next_moved" else "", if (mf_node.flags.resized) " resized" else "", @@ -8920,7 +8934,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error // Align the actual node const seg_ni = elf.phdrs.items[phndx].unwrap().?; if (min_align.compare(.gt, seg_ni.alignment(&elf.mf))) { - try seg_ni.realign(&elf.mf, gpa, min_align, .{}); + try seg_ni.realign(&elf.mf, gpa, min_align); } // Update the phdr `@"align"` field if necessary switch (elf.phdrSlice()) { @@ -8960,15 +8974,7 @@ fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void { const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf); break :last_end last_offset + last_size; } else 0; - try elf.ensureNodeSize(elf.ni.elf, last_end + @sizeOf(std.elf.ar_hdr)); -} - -fn ensureNodeSize(elf: *Elf, node: MappedFile.Node.Index, need_size: u64) MappedFile.Error!void { - _, const node_size = node.location(&elf.mf).resolve(&elf.mf); - if (need_size <= node_size) return; - const gpa = elf.base.comp.gpa; - const new_size = need_size + need_size / MappedFile.growth_factor; - try node.resize(&elf.mf, gpa, new_size); + try elf.ni.elf.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, last_end + @sizeOf(std.elf.ar_hdr)); } /// If `sym` has a PLT entry, returns the address of that entry (specifically, the address which a diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index b1da80973c756c2aab7b30d88b3605171b06362f..40af66dbf8c53093c492a79d69064879175cc4ef 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -183,14 +183,42 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel .fallocate_insert_range_unsupported = false, .fallocate_punch_hole_unsupported = false, }; - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const root_ni = try mf.addNode(gpa, .{ .add_node = .{ - .size = size, - .alignment = mf.flags.block_size, - .fixed = true, - } }); - assert(root_ni == .root); - try mf.ensureTotalCapacityInner(@intCast(size)); + + const root_location: Node.Location = l: { + if (std.math.cast(u32, size)) |small_size| { + break :l .{ .small = .{ .offset = 0, .size = small_size } }; + } + try mf.large.appendSlice(gpa, &.{ 0, size }); + break :l .{ .large = .{ .index = 0 } }; + }; + try mf.nodes.append(gpa, .{ + .parent = .none, + .prev = .none, + .next = .none, + .first = .none, + .last = .none, + .flags = .{ + .alignment = mf.flags.block_size, + .position = .floating, + .bubbles_moved = true, + .enable_next_moved = false, + .location_tag = root_location, + .moved = false, + .resized = false, + .next_moved = false, + .has_content = false, + }, + .location_payload = switch (root_location) { + .small => |small| .{ .small = small }, + .large => |large| .{ .large = large }, + }, + }); + + mf.ensureTotalCapacity(@intCast(size)) catch |err| switch (err) { + error.MappedFileIo => return mf.io_err.?, + else => |e| return e, + }; + return mf; } @@ -213,24 +241,53 @@ pub const Node = extern struct { flags: Flags, location_payload: Location.Payload, + /// Any non-leaf node may designate its first N children as "header" nodes. This means that its + /// first N children must be densely packed together and positioned at the start of the parent. + /// The implementation guarantees that it will never re-order these nodes, nor will it introduce + /// padding between them. + /// + /// Likewise, any non-leaf node may designate its *last* M children as "footer" nodes, which are + /// like header nodes except they are positioned at the *end* of the parent rather than the + /// start. + /// + /// Nodes which are neither headers nor footers are called "floating". The implementation is + /// always free to re-order floating nodes relative to one another, and to add or remove padding + /// between them. + pub const Position = enum(u2) { + header, + footer, + floating, + }; + pub const Flags = packed struct(u32) { - location_tag: Location.Tag, + /// While the number of header and footer nodes within a parent node is logically a part of + /// that parent, we actually store this information on the child nodes for efficiency: this + /// field indicates whether each child is a header node, a footer node, or a floating node. + /// + /// This value is meaningless for the root node, so is arbitrarily set to `.floating`. + position: Position, + /// For floating nodes, this node's offset into its parent will always be aligned to this + /// boundary. (This is not the case for header and footer nodes due to the requirement that + /// they be densely packed against the start/end of the parent node.) + /// + /// This node's size will also always be aligned to this boundary. (This applies regardless + /// of whether this is a floating node, a header node, or a footer node.) alignment: Alignment, - /// Whether this node can be moved. - fixed: bool, + /// Whether `moved` events on this node bubble down to children. + bubbles_moved: bool, + /// Whether `next_moved` events are reported in `updates`. + enable_next_moved: bool, + + location_tag: Location.Tag, /// Whether this node has been moved. moved: bool, /// Whether this node has been resized. resized: bool, /// Whether the next sibling has moved or is a different node. next_moved: bool, - /// Whether this node might contain non-zero bytes. + /// Whether this node might contain initialized bytes. has_content: bool, - /// Whether `moved` events on this node bubble down to children. - bubbles_moved: bool, - /// Whether `next_moved` events are reported in `updates`. - enable_next_moved: bool, - unused: u18 = 0, + unused: u17 = 0, }; pub const Location = union(enum(u1)) { @@ -267,6 +324,18 @@ pub const Node = extern struct { } }; + pub const AddOptions = struct { + /// Must be aligned to the given `alignment`. + size: u64 = 0, + alignment: Alignment = .@"1", + bubbles_moved: bool = true, + enable_next_moved: bool = false, + + moved: bool = false, + resized: bool = false, + next_moved: bool = false, + }; + pub const Index = enum(u32) { root, _, @@ -292,6 +361,70 @@ pub const Node = extern struct { return &mf.nodes.items[@backingInt(ni)]; } + /// Adds a floating child node to `parent_ni`. Returns the index of the new child. + pub fn addFloatingChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .floating, + .parent = parent_ni, + .prev = parent_ni.lastHeader(mf), + }); + } + /// Adds a header child node to `parent_ni`. Returns the index of the new child. + /// + /// Asserts that `parent_ni` has no existing header children. + pub fn addOnlyHeaderChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + if (parent_ni.first(mf).unwrap()) |first_ni| { + assert(first_ni.position(mf) != .header); // `parent_ni` already has a header child + } + return parent_ni.addHeaderChildAfter(mf, gpa, .none, opts); + } + /// Adds a header child node to `parent_ni`. Returns the index of the new child. + /// + /// If `prev_oni` is `.none`, the new child is placed at the very start of the parent, + /// before any existing header nodes. + /// + /// Otherwise, asserts that `prev_oni` is a header node and a child of `parent_ni`, and + /// places the new child node immediately after `prev_oni`. + pub fn addHeaderChildAfter(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, prev_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .header, + .parent = parent_ni, + .prev = prev_oni, + }); + } + /// Adds a footer child node to `parent_ni`. Returns the index of the new child. + /// + /// Asserts that `parent_ni` has no existing footer children. + pub fn addOnlyFooterChild(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, opts: AddOptions) Error!Node.Index { + if (parent_ni.last(mf).unwrap()) |last_ni| { + assert(last_ni.position(mf) != .footer); // `parent_ni` already has a footer child + } + return parent_ni.addFooterChildBefore(mf, gpa, .none, opts); + } + /// Adds a footer child node to `parent_ni`. Returns the index of the new child. + /// + /// If `next_oni` is `.none`, the new child is placed at the very end of the parent, after + /// any existing footer nodes. + /// + /// Otherwise, asserts that `next_oni` is a footer node and a child of `parent_ni`, and + /// places the new child node immediately before `next_oni`. + pub fn addFooterChildBefore(parent_ni: Node.Index, mf: *MappedFile, gpa: Allocator, next_oni: Node.Index.Optional, opts: AddOptions) Error!Node.Index { + const prev_oni: Node.Index.Optional = prev: { + const next_ni = next_oni.unwrap() orelse { + break :prev parent_ni.last(mf); + }; + break :prev next_ni.prev(mf); + }; + return mf.addNode(gpa, .{ + .add_options = opts, + .position = .footer, + .parent = parent_ni, + .prev = prev_oni, + }); + } + /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. pub const toOptional = Optional.wrap; @@ -299,19 +432,54 @@ pub const Node = extern struct { return ni.get(mf).parent; } + pub fn first(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + return ni.get(mf).first; + } + + pub fn last(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + return ni.get(mf).last; + } + + fn lastHeader(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + var header_ni = ni.first(mf).unwrap() orelse return .none; + if (header_ni.position(mf) != .header) return .none; + while (true) { + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + return .wrap(header_ni); + } + fn firstFooter(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { + var footer_ni = ni.last(mf).unwrap() orelse return .none; + if (footer_ni.position(mf) != .footer) return .none; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + return .wrap(footer_ni); + } + + /// Asserts that `ni` is not `.root`, because `Position` is meaningless for the root node. + pub fn position(ni: Node.Index, mf: *const MappedFile) Node.Position { + assert(ni != .root); + return ni.get(mf).flags.position; + } + pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).next; } fn setNext( - prev_ni: Node.Index, + ni: Node.Index, gpa: Allocator, next_ni: Node.Index.Optional, mf: *MappedFile, ) Allocator.Error!void { - const prev_next = &prev_ni.get(mf).next; - if (prev_next.* == next_ni) return; - prev_next.* = next_ni; - try prev_ni.nextMoved(gpa, mf); + const next_ptr = &ni.get(mf).next; + if (next_ptr.* == next_ni) return; + next_ptr.* = next_ni; + try ni.nextMoved(gpa, mf); } pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { @@ -421,8 +589,14 @@ pub const Node = extern struct { return ni.get(mf).flags.alignment; } - fn setLocationAssumeCapacity(ni: Node.Index, mf: *MappedFile, offset: u64, size: u64) void { + fn setLocation(ni: Node.Index, mf: *MappedFile, gpa: Allocator, offset: u64, size: u64) Allocator.Error!void { + try mf.large.ensureUnusedCapacity(gpa, 2); + try mf.updates.ensureUnusedCapacity(gpa, 2); const node = ni.get(mf); + if (node.flags.position == .floating) { + assert(node.flags.alignment.check(offset)); + } + assert(node.flags.alignment.check(size)); if (size == 0) node.flags.has_content = false; switch (node.location()) { .small => |small| { @@ -485,62 +659,46 @@ pub const Node = extern struct { return mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)]; } - pub fn resize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { - mf.resizeNode(gpa, ni, size) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; - var writers_it = mf.writers.first; - while (writers_it) |writer_node| : (writers_it = writer_node.next) { - const w: *Node.Writer = @fieldParentPtr("writer_node", writer_node); - w.interface.buffer = w.ni.slice(mf); + /// Ensures that the size of `ni` is at least `min_size`. Valid for any node. + /// + /// Applies `growth_factor` if necessary (so the caller should *not* apply `growth_factor`). + pub fn ensureMinimumSize(ni: Node.Index, mf: *MappedFile, gpa: Allocator, min_size: u64) Error!void { + _, const current_size = ni.location(mf).resolve(mf); + if (current_size >= min_size) return; + const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor); + try mf.growNode(gpa, ni, new_size, .minimum); + mf.updateWriters(); + } + + /// Sets the size of `ni` to exactly `size`. + /// + /// Asserts that `ni` is a leaf node, i.e. has no children. + /// + /// Asserts that `size` is aligned to `ni.alignment(mf)`. + pub fn resizeLeaf(ni: Node.Index, mf: *MappedFile, gpa: Allocator, size: u64) Error!void { + assert(ni.first(mf) == .none); + // The alignment of `size` is asserted by `shrinkLeafNode` and `growNode`. + _, const old_size = ni.location(mf).resolve(mf); + switch (std.math.order(size, old_size)) { + .lt => try mf.shrinkLeafNode(gpa, ni, size), + .eq => {}, // `old_size` must be well-aligned, so `size` is too + .gt => try mf.growNode(gpa, ni, size, .exact), } + mf.updateWriters(); } - pub const RealignNodeOptions = struct { - /// Shift the node backwards if possible - try_backwards: bool = false, - }; - - /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. - /// Asserts that `ni` is not `.root`. + /// Updates a node's alignment to exactly `new_alignment`. Valid for any node. + /// + /// If the node's current offset or size is not sufficiently aligned, it will be moved + /// and/or resized to match the new alignment. The node's size may be increased by any + /// amount, as if `ensureMinimumSize` were used. pub fn realign( ni: Node.Index, mf: *MappedFile, gpa: Allocator, new_alignment: Alignment, - opts: RealignNodeOptions, ) Error!void { - mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; - mf.updateWriters(); - } - - /// Shrink a node to `size`, exactly. - /// Asserts that the new size can contain all the children. - /// If `shift_next` is set, then the following node is shifted backwards into - /// the free space as much as alignment allows. - /// Asserts that `size` is >= the end of the last child node. - pub fn shrink( - ni: Node.Index, - mf: *MappedFile, - gpa: Allocator, - size: u64, - shift_next: bool, - ) Error!void { - try mf.shrinkNode(gpa, ni, size, shift_next); + try mf.realignNode(gpa, ni, new_alignment); mf.updateWriters(); } @@ -644,16 +802,9 @@ pub const Node = extern struct { file_reader.pos, w.ni.fileLocation(w.mf, true).offset + interface.end, limit.minInt(interface.unusedCapacityLen()), - ) catch |err| switch (err) { - error.Canceled => |e| { - w.err = e; - return error.WriteFailed; - }, - else => |e| { - w.mf.io_err = e; - w.err = error.MappedFileIo; - return error.WriteFailed; - }, + ) catch |err| { + w.err = err; + return error.WriteFailed; }); if (n == 0) return error.Unimplemented; file_reader.pos += n; @@ -680,10 +831,8 @@ pub const Node = extern struct { unused_capacity: usize, ) Io.Writer.Error!void { _ = preserve; - const total_capacity = interface.end + unused_capacity; - if (interface.buffer.len >= total_capacity) return; const w: *Writer = @fieldParentPtr("interface", interface); - w.ni.resize(w.mf, w.gpa, total_capacity +| total_capacity / growth_factor) catch |err| { + w.ni.ensureMinimumSize(w.mf, w.gpa, interface.end + unused_capacity) catch |err| { w.err = err; return error.WriteFailed; }; @@ -691,514 +840,1227 @@ pub const Node = extern struct { }; comptime { - if (!std.debug.runtime_safety) std.debug.assert(@sizeOf(Node) == 32); + if (!std.debug.runtime_safety) assert(@sizeOf(Node) == 32); } }; +/// Asserts that `opts.position` is compatible with `opts.prev` (i.e. that this addition will not +/// violate the requirement that header nodes come before floating nodes come before footer nodes). fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { - parent: Node.Index.Optional = .none, - prev: Node.Index.Optional = .none, - next: Node.Index.Optional = .none, - offset: u64 = 0, - add_node: AddNodeOptions, -}) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { + add_options: Node.AddOptions, + position: Node.Position, + parent: Node.Index, + /// If `position == .floating`, this is just used as an initial value, and may be immediately + /// replaced when finding a location for this node. In this case, it is still necessary that + /// `prev` be compatible with `position` (so `prev` must be either a floating node or the last + /// header node in `parent`). + prev: Node.Index.Optional, +}) Error!Node.Index { mf.nodes_lock.assertUnlocked(); - const location_tag: Node.Location.Tag, const location_payload: Node.Location.Payload = location: { - if (std.math.cast(u32, opts.offset)) |small_offset| break :location .{ .small, .{ - .small = .{ .offset = small_offset, .size = 0 }, - } }; - try mf.large.ensureUnusedCapacity(gpa, 2); - defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); - break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; - }; - const free_ni: Node.Index, const free_node: *Node = if (mf.free_ni.unwrap()) |free_ni| free: { - const free_node = free_ni.get(mf); - mf.free_ni = free_node.next; - break :free .{ free_ni, free_node }; - } else .{ - @fromBackingInt(@intCast(mf.nodes.items.len)), - mf.nodes.addOneAssumeCapacity(), + try mf.nodes.ensureUnusedCapacity(gpa, 1); + try mf.large.ensureUnusedCapacity(gpa, 2); + + const new_ni: Node.Index = new: { + if (mf.free_ni.unwrap()) |free_ni| { + mf.free_ni = free_ni.get(mf).next; + break :new free_ni; + } + const new_ni: Node.Index = @fromBackingInt(@intCast(mf.nodes.items.len)); + _ = mf.nodes.addOneAssumeCapacity(); + break :new new_ni; }; - if (opts.prev.unwrap()) |prev_ni| { - try prev_ni.setNext(gpa, .wrap(free_ni), mf); - } else if (opts.parent.unwrap()) |parent_ni| { - parent_ni.get(mf).first = .wrap(free_ni); - } else { - assert(free_ni == .root); - } + const next_oni: Node.Index.Optional = if (opts.prev.unwrap()) |prev_ni| next: { + assert(prev_ni.parent(mf) == opts.parent.toOptional()); // `prev` is not a child of `parent` + break :next prev_ni.get(mf).next; + } else opts.parent.first(mf); - if (opts.next.unwrap()) |next_ni| { - next_ni.get(mf).prev = .wrap(free_ni); - } else if (opts.parent.unwrap()) |parent_ni| { - parent_ni.get(mf).last = .wrap(free_ni); - } else { - assert(free_ni == .root); + // Validate node ordering + switch (opts.position) { + .floating => { + if (opts.prev.unwrap()) |prev_ni| { + assert(prev_ni.position(mf) != .footer); // tried to add floating node after footer node + } + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.position(mf) != .header); // tried to add floating node before header node + } + }, + .header => if (opts.prev.unwrap()) |prev_ni| { + switch (prev_ni.position(mf)) { + .header => {}, + .floating => unreachable, // tried to add header node after floating node + .footer => unreachable, // tried to add header node after footer node + } + }, + .footer => if (next_oni.unwrap()) |next_ni| { + switch (next_ni.position(mf)) { + .header => unreachable, // tried to add footer node before header node + .floating => unreachable, // tried to add footer node before floating node + .footer => {}, + } + }, } - free_node.* = .{ - .parent = opts.parent, - .prev = opts.prev, - .next = opts.next, + // Initialize the node as empty with alignment 1 + const location: Node.Location = loc: { + const offset: u64 = switch (opts.position) { + .header, .floating => offset: { + const prev_ni = opts.prev.unwrap() orelse break :offset 0; + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :offset prev_offset + prev_size; + }, + .footer => offset: { + const next_ni = next_oni.unwrap() orelse { + _, const parent_size = opts.parent.location(mf).resolve(mf); + break :offset parent_size; + }; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :offset next_offset; + }, + }; + if (std.math.cast(u32, offset)) |small_offset| { + break :loc .{ .small = .{ .offset = small_offset, .size = 0 } }; + } + const large_index = mf.large.items.len; + mf.large.appendSliceAssumeCapacity(&.{ offset, 0 }); + break :loc .{ .large = .{ .index = large_index } }; + }; + new_ni.get(mf).* = .{ + .parent = .wrap(opts.parent), + .prev = .none, + .next = .none, .first = .none, .last = .none, .flags = .{ - .location_tag = location_tag, + .position = opts.position, .alignment = .@"1", - .fixed = opts.add_node.fixed, - .moved = true, - .resized = true, - .next_moved = true, + .bubbles_moved = opts.add_options.bubbles_moved, + .enable_next_moved = opts.add_options.enable_next_moved, + .location_tag = location, + .moved = false, + .resized = false, + .next_moved = false, .has_content = false, - .bubbles_moved = opts.add_node.bubbles_moved, - .enable_next_moved = opts.add_node.enable_next_moved, }, - .location_payload = location_payload, + .location_payload = switch (location) { + .small => |small| .{ .small = small }, + .large => |large| .{ .large = large }, + }, }; - { - defer { - free_node.flags.moved = false; - free_node.flags.resized = false; - free_node.flags.next_moved = false; - } - try mf.realignNode(gpa, free_ni, opts.add_node.alignment, .{}); - try mf.resizeNode(gpa, free_ni, opts.add_node.size); + try mf.addNodesToChildListBefore(gpa, next_oni, new_ni, new_ni); + + try mf.realignNode(gpa, new_ni, opts.add_options.alignment); + if (opts.add_options.size > 0) { + try mf.growNode(gpa, new_ni, opts.add_options.size, .exact); } mf.updateWriters(); - if (opts.add_node.moved) try free_ni.moved(gpa, mf); - if (opts.add_node.resized) try free_ni.resized(gpa, mf); - if (opts.add_node.next_moved) try free_ni.nextMoved(gpa, mf); - return free_ni; -} -pub const AddNodeOptions = struct { - size: u64 = 0, - alignment: Alignment = .@"1", - fixed: bool = false, - moved: bool = false, - resized: bool = false, - next_moved: bool = false, - bubbles_moved: bool = true, - enable_next_moved: bool = false, -}; - -pub fn addOnlyChildNode( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - assert(parent.first == .none and parent.last == .none); - return mf.addNode(gpa, .{ - .parent = .wrap(parent_ni), - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; + new_ni.get(mf).flags.moved = false; + new_ni.get(mf).flags.resized = false; + new_ni.get(mf).flags.next_moved = false; + + if (opts.add_options.moved) try new_ni.moved(gpa, mf); + if (opts.add_options.resized) try new_ni.resized(gpa, mf); + if (opts.add_options.next_moved) try new_ni.nextMoved(gpa, mf); + + return new_ni; } -pub fn addFirstChildNode( +fn shrinkLeafNode( mf: *MappedFile, gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - return mf.addNode(gpa, .{ - .parent = .wrap(parent_ni), - .next = parent.first, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; + ni: Node.Index, + new_size: u64, +) Error!void { + mf.nodes_lock.assertUnlocked(); + + const old_offset, const old_size = ni.location(mf).resolve(mf); + + assert(new_size < old_size); + assert(ni.alignment(mf).check(new_size)); + assert(ni.first(mf) == .none); // `ni` must be a leaf node + + const parent_ni = ni.parent(mf).unwrap() orelse { + assert(ni == .root); + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; return error.MappedFileIo; - }, + }; + mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; + try mf.ensureTotalCapacityPrecise(@intCast(new_size)); + try ni.setLocation(mf, gpa, old_offset, new_size); + return; }; -} -pub fn addLastChildNode( - mf: *MappedFile, - gpa: Allocator, - parent_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const parent = parent_ni.get(mf); - return mf.addNode(gpa, .{ - .parent = .wrap(parent_ni), - .prev = parent.last, - .offset = offset: { - const last_ni = parent.last.unwrap() orelse break :offset 0; - const last_offset, const last_size = last_ni.location(mf).resolve(mf); - break :offset last_offset + last_size; + switch (ni.position(mf)) { + .header => { + const shift = old_size - new_size; + + try ni.setLocation(mf, gpa, old_offset, new_size); + + // We need to shift backwards all header nodes following us. + const next_header_ni = ni.next(mf).unwrap() orelse return; + if (next_header_ni.position(mf) != .header) return; + + var header_ni = next_header_ni; + while (true) { + const old_header_off, const old_header_size = header_ni.location(mf).resolve(mf); + try header_ni.setLocation(mf, gpa, old_header_off - shift, old_header_size); + + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; + } + + // Now we must shift the actual header bytes of those nodes backwards. + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + const move_src_off = old_offset + old_size; + const move_dest_off = old_offset + new_size; + assert(next_header_ni.location(mf).resolve(mf)[0] == move_dest_off); // `move_dest_off` because we already updated the location + const move_size = size: { + // `header_ni` is the last header in the parent. + const last_off, const last_size = header_ni.location(mf).resolve(mf); + const move_end = last_off + last_size; + break :size move_end - move_dest_off; // `move_dest_off` because we already updated the location + }; + try mf.moveRange( + parent_file_off + move_src_off, + parent_file_off + move_dest_off, + move_size, + ); }, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; + .floating => { + try ni.setLocation(mf, gpa, old_offset, new_size); }, - }; -} + .footer => { + const shift = old_size - new_size; -pub fn addNodeAfter( - mf: *MappedFile, - gpa: Allocator, - prev_ni: Node.Index, - opts: AddNodeOptions, -) Error!Node.Index { - try mf.nodes.ensureUnusedCapacity(gpa, 1); - const prev = prev_ni.get(mf); - const prev_offset, const prev_size = prev.location().resolve(mf); - return mf.addNode(gpa, .{ - .parent = prev.parent, - .prev = .wrap(prev_ni), - .next = prev.next, - .offset = prev_offset + prev_size, - .add_node = opts, - }) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} + const new_offset = old_offset + shift; + try ni.setLocation(mf, gpa, new_offset, new_size); -fn shrinkNode( - mf: *MappedFile, - gpa: Allocator, - ni: Node.Index, - size: u64, - shift_next: bool, -) !void { - mf.nodes_lock.assertUnlocked(); - const node = ni.get(mf); - const old_offset, _ = node.location().resolve(mf); + const prev_footers_size = prev_footers_size: { + // We need to shift forwards all footer nodes preceding us. + const prev_footer_ni = ni.prev(mf).unwrap() orelse { + break :prev_footers_size 0; + }; + if (prev_footer_ni.position(mf) != .footer) { + break :prev_footers_size 0; + } - // This would require unmapping first - assert(ni != .root); + var footer_ni = prev_footer_ni; + while (true) { + const old_footer_off, const old_footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_off + shift, old_footer_size); - if (node.last.unwrap()) |last_ni| { - const last = last_ni.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - assert(last_offset + last_size > size); - } + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } - try mf.large.ensureUnusedCapacity(gpa, 4); - try mf.updates.ensureUnusedCapacity(gpa, 4); - - ni.setLocationAssumeCapacity(mf, old_offset, size); - if (!shift_next) return; - const next_ni = node.next.unwrap() orelse return; - - const next = next_ni.get(mf); - const old_next_offset, const next_size = next.location().resolve(mf); - const padding = old_next_offset - (old_offset + size); - const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); - - if (next.flags.has_content and new_next_offset < old_next_offset) { - const old_file_offset = next_ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(next_size)], - ); - @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); - } + // `footer_ni` is the first footer in the parent. This expression gets its *new* + // offset because we already did the `setLocation` calls. + const first_footer_new_offset = footer_ni.location(mf).resolve(mf)[0]; + + break :prev_footers_size new_offset - first_footer_new_offset; + }; - next_ni.setLocationAssumeCapacity(mf, new_next_offset, next_size); + // Now we must shift the actual footer bytes forwards, including our own. + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_offset - prev_footers_size, + parent_file_offset + new_offset - prev_footers_size, + prev_footers_size + new_size, + ); + }, + } } -fn resizeNode( +const GrowMode = enum { exact, minimum }; + +/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`. +/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`. +/// +/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!). +/// +/// Asserts that `new_size` is greater than the current size of `ni`. +fn growNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - requested_size: u64, -) (Allocator.Error || Io.Cancelable || IoError)!void { + new_size: u64, + grow_mode: GrowMode, +) Error!void { mf.nodes_lock.assertUnlocked(); - const io = mf.io; + const node = ni.get(mf); + const old_offset, const old_size = node.location().resolve(mf); - const new_size = node.flags.alignment.forward(@intCast(requested_size)); - // Resize the entire file + assert(node.flags.alignment.check(old_size)); + assert(node.flags.alignment.check(new_size)); + assert(new_size > old_size); + const parent_ni = node.parent.unwrap() orelse { assert(ni == .root); - try mf.ensureCapacityForSetLocation(gpa); - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + mf.memory_map.file.setLength(mf.io, new_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, }; - try mf.memory_map.file.setLength(io, new_size); - try mf.ensureTotalCapacityInner(@intCast(new_size)); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); + try mf.ensureTotalCapacityPrecise(@intCast(new_size)); + try ni.setLocation(mf, gpa, old_offset, new_size); + // We need to move any footers to be at the *new* end of the file. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = old_size - old_footers_offset; + try mf.moveRange( + old_footers_offset, + old_footers_offset + (new_size - old_size), + footers_size, + ); + // Also update the footers' locations. + var cur_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + } return; }; - const parent = parent_ni.get(mf); - _, var old_parent_size = parent.location().resolve(mf); - const trailing_end = trailing_end: { - const next_ni = node.next.unwrap() orelse break :trailing_end old_parent_size; - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }; - assert(old_offset + old_size <= trailing_end); - if (old_offset + new_size <= trailing_end) { - // Expand the node into trailing free space - try mf.ensureCapacityForSetLocation(gpa); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); - return; - } - insert_range: { - if (!is_linux) break :insert_range; - if (mf.flags.fallocate_insert_range_unsupported) break :insert_range; - - // We need the node to be aligned to `mf.flags.block_size` in the file in order to use this - // fast path. It is not sufficient to check `node.flags.alignment`, because that doesn't - // necessarily mean that all *parent* nodes are equally aligned; instead we must compute the - // actual file offset. - const range_file_offset = ni.fileLocation(mf, false).offset + old_size; - const range_size = node.flags.alignment.forward( - @intCast(requested_size +| requested_size / growth_factor), - ) - old_size; - if (!mf.flags.block_size.check(@intCast(range_file_offset))) break :insert_range; - if (!mf.flags.block_size.check(@intCast(range_size))) break :insert_range; - - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, - }; - // Ask the filesystem driver to insert extents into the file without copying any data - const last_offset, const last_size = parent.last.unwrap().?.location(mf).resolve(mf); - const last_end = last_offset + last_size; - assert(last_end <= old_parent_size); - _, const file_size = Node.Index.root.location(mf).resolve(mf); - while (true) switch (linux.errno(switch (std.math.order(range_file_offset, file_size)) { - .lt => linux.fallocate( - mf.memory_map.file.handle, - linux.FALLOC.FL_INSERT_RANGE, - @intCast(range_file_offset), - @intCast(range_size), - ), - .eq => linux.ftruncate(mf.memory_map.file.handle, @intCast(range_file_offset + range_size)), - .gt => unreachable, - })) { - .SUCCESS => { - var enclosing_ni = ni; + + switch (node.flags.position) { + .header => { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + try mf.ensureAdditionalHeaderCapacity(gpa, parent_ni, new_size - old_size); + + // `old_offset` is still valid because header nodes don't move when the parent resizes. + + const last_header_ni: Node.Index = last_header: { + var header_ni = ni; while (true) { - try mf.ensureCapacityForSetLocation(gpa); - const enclosing = enclosing_ni.get(mf); - const enclosing_offset, const old_enclosing_size = - enclosing.location().resolve(mf); - const new_enclosing_size = old_enclosing_size + range_size; - enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); - if (enclosing_ni == .root) { - assert(enclosing_offset == 0); - try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); - break; - } - var after_oni = enclosing.next; - while (after_oni.unwrap()) |after_ni| { - try mf.ensureCapacityForSetLocation(gpa); - const after = after_ni.get(mf); - const after_offset, const after_size = after.location().resolve(mf); - after_ni.setLocationAssumeCapacity( - mf, - range_size + after_offset, - after_size, - ); - after_oni = after.next; - } - enclosing_ni = enclosing.parent.unwrap().?; + const next_ni = header_ni.next(mf).unwrap() orelse break; + if (next_ni.position(mf) != .header) break; + header_ni = next_ni; } - return; - }, - .INTR => continue, - .BADF, .FBIG, .INVAL => unreachable, - .IO => return error.InputOutput, - .NODEV => return error.NotFile, - .NOSPC => return error.NoSpaceLeft, - .NOSYS, .OPNOTSUPP => { - mf.flags.fallocate_insert_range_unsupported = true; - break :insert_range; - }, - .PERM => return error.PermissionDenied, - .SPIPE => return error.Unseekable, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), - }; - } - if (node.next == .none) { - // As this is the last node, we simply need more space in the parent - const new_parent_size = old_offset + new_size; - try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); - try mf.ensureCapacityForSetLocation(gpa); - ni.setLocationAssumeCapacity(mf, old_offset, new_size); - return; - } - if (!node.flags.fixed) { - // Make space at the end of the parent for this floating node - const last = parent.last.unwrap().?.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); - const new_parent_size = new_offset + new_size; - if (new_parent_size > old_parent_size) - try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); - try mf.ensureCapacityForSetLocation(gpa); - const next_ni = node.next.unwrap().?; - next_ni.get(mf).prev = node.prev; - if (node.prev.unwrap()) |prev_ni| { - try prev_ni.setNext(gpa, .wrap(next_ni), mf); - } else { - parent.first = .wrap(next_ni); - } - try parent.last.unwrap().?.setNext(gpa, .wrap(ni), mf); - node.prev = parent.last; - try ni.setNext(gpa, .none, mf); - parent.last = .wrap(ni); - if (node.flags.has_content) { + break :last_header header_ni; + }; + const last_header_offset, const last_header_size = last_header_ni.location(mf).resolve(mf); + const old_headers_size = last_header_offset + last_header_size; + + // This is the first footer *inside* of `ni`. + const first_sub_footer_oni = ni.firstFooter(mf); + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size old_size - first_sub_footer_offset; + }; + + // We need to shift two things forwards; any header nodes which follow us, and any + // footer nodes *within* us (since they need to be at the end of our new size). const parent_file_offset = parent_ni.fileLocation(mf, false).offset; try mf.moveRange( - parent_file_offset + old_offset, - parent_file_offset + new_offset, - old_size, + parent_file_offset + old_offset + old_size - sub_footers_size, + parent_file_offset + old_offset + new_size - sub_footers_size, + old_headers_size - (old_offset + old_size - sub_footers_size), ); - } - ni.setLocationAssumeCapacity(mf, new_offset, new_size); - return; - } - // Search for the first floating node following this fixed node - var last_fixed_ni = ni; - var first_floating_oni = node.next; - var shift = new_size - old_size; - var max_shift_align: Alignment = .@"1"; - var direction: enum { forward, reverse } = .forward; - while (true) { - const last_fixed = last_fixed_ni.get(mf); - assert(last_fixed.flags.fixed); - const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); - const new_last_fixed_offset = old_last_fixed_offset + shift; - if (first_floating_oni.unwrap()) |first_floating_ni| make_space: { - const first_floating = first_floating_ni.get(mf); - const old_first_floating_offset, const first_floating_size = - first_floating.location().resolve(mf); - assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); - if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) - break :make_space; - assert(direction == .forward); - max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); - if (first_floating.flags.fixed) { - shift = max_shift_align.forward(@intCast( - @max(shift, first_floating_size), - )); - - // Not enough space, try the next node - last_fixed_ni = first_floating_ni; - first_floating_oni = first_floating.next; - continue; + + // Any footers inside of us have had their offsets changed due to us growing: + if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { + var cur_ni = first_sub_footer_ni; + while (true) { + const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation( + mf, + gpa, + old_sub_footer_offset + (new_size - old_size), + sub_footer_size, + ); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } - // Move the found floating node to make space for preceding fixed nodes - const last = parent.last.unwrap().?.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_first_floating_offset = max_shift_align.forward( - @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), + + // Update the offsets of all header nodes following us: + { + var moved_header_ni = last_header_ni; + while (moved_header_ni != ni) { + assert(moved_header_ni.position(mf) == .header); + const moved_header_offset, const moved_header_size = moved_header_ni.location(mf).resolve(mf); + try moved_header_ni.setLocation( + mf, + gpa, + moved_header_offset - old_size + new_size, + moved_header_size, + ); + moved_header_ni = moved_header_ni.prev(mf).unwrap().?; + } + } + + // Finally, update our own size: + try ni.setLocation(mf, gpa, old_offset, new_size); + return; + }, + .floating => { + try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode); + }, + .footer => { + if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) { + return; + } + + try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size); + + const first_footer_ni: Node.Index = first_footer: { + var footer_ni = ni; + while (true) { + const prev_ni = footer_ni.prev(mf).unwrap() orelse break; + if (prev_ni.position(mf) != .footer) break; + footer_ni = prev_ni; + } + break :first_footer footer_ni; + }; + + // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself + // a footer within its parent). + const first_sub_footer_oni = ni.firstFooter(mf); + const sub_footers_size = size: { + const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0; + const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf); + break :size old_size - first_sub_footer_offset; + }; + + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0]; + const new_footers_size = old_footers_size - old_size + new_size; + + // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node + // itself needs to shift its contents, because our offset was shifted backwards by + // `new_size - old_size`, and the added bytes should go at the end of this footer node. + // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so + // we *shouldn't* shift *that* data. + const old_footers_start = parent_size - old_footers_size; + const new_footers_start = parent_size - new_footers_size; + const end_offset = node.location().resolve(mf)[0] + old_size; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_footers_start, + parent_file_offset + new_footers_start, + end_offset - old_footers_start - sub_footers_size, ); - const new_parent_size = new_first_floating_offset + first_floating_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - parent_ni, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); + + // Update our own offset and size: + try ni.setLocation(mf, gpa, end_offset - new_size, new_size); + + // Any footers inside of us have had their offsets changed due to us growing: + if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| { + var cur_ni = first_sub_footer_ni; + while (true) { + const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation( + mf, + gpa, + old_sub_footer_offset + (new_size - old_size), + sub_footer_size, + ); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } } - try mf.ensureCapacityForSetLocation(gpa); - if (parent.last.unwrap().? != first_floating_ni) { - const old_last = parent.last.unwrap().?; - first_floating.prev = .wrap(old_last); - parent.last = .wrap(first_floating_ni); - try old_last.setNext(gpa, .wrap(first_floating_ni), mf); - try last_fixed_ni.setNext(gpa, first_floating.next, mf); - if (first_floating.next.unwrap()) |next_ni| { - next_ni.get(mf).prev = .wrap(last_fixed_ni); + + // Finally, update the offsets of every footer before us: + if (node.prev.unwrap()) |prev_ni| { + var maybe_footer_ni = prev_ni; + while (true) { + switch (maybe_footer_ni.position(mf)) { + .header, .floating => break, + .footer => {}, + } + const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf); + try maybe_footer_ni.setLocation( + mf, + gpa, + moved_footer_offset + old_size - new_size, + moved_footer_size, + ); + maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break; } - try first_floating_ni.setNext(gpa, .none, mf); } - if (first_floating.flags.has_content) { - const parent_file_offset = - parent_ni.fileLocation(mf, false).offset; + + return; + }, + } +} + +/// Moves a floating node to an unused region with the given size, which may be greater than the +/// current size. If `new_alignment` is not `null`, then the offset and size of the new region will +/// have that alignment instead of `ni.alignment(mf)`. +/// +/// Asserts that `ni` is a floating node (and not `.root`). +/// +/// Asserts that `new_size` is aligned to `new_alignment orelse ni.alignment(mf)`. +/// +/// Asserts that `new_size` is greater than or equal to the current size of `ni`. +fn growFloatingNodeWithAlignment( + mf: *MappedFile, + gpa: Allocator, + ni: Node.Index, + new_alignment: ?Alignment, + new_size: u64, + grow_mode: GrowMode, +) Error!void { + mf.nodes_lock.assertUnlocked(); + + const parent_ni = ni.parent(mf).unwrap().?; // `ni` cannot be `.root` + const old_offset, const old_size = ni.location(mf).resolve(mf); + + const alignment = new_alignment orelse ni.alignment(mf); + + assert(new_size >= old_size); + assert(ni.position(mf) == .floating); + assert(alignment.check(new_size)); + + grow_in_place: { + if (!alignment.check(old_offset)) { + break :grow_in_place; + } + const limit: u64 = limit: { + const next_ni = ni.next(mf).unwrap() orelse break :limit parent_ni.location(mf).resolve(mf)[1]; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :limit next_offset; + }; + if (old_offset + new_size > limit) { + break :grow_in_place; // the parent is not big enough + } + // Great, we can grow this node without changing its offset or moving any siblings. + try ni.setLocation(mf, gpa, old_offset, new_size); + // If we have any footers, we need to move them to the end of our new size, and update their + // offsets accordingly. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + if (footers_have_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + // This gets the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; try mf.moveRange( - parent_file_offset + old_first_floating_offset, - parent_file_offset + new_first_floating_offset, - first_floating_size, + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + old_offset + new_size - footers_size, + footers_size, ); } - first_floating_ni.setLocationAssumeCapacity( - mf, - new_first_floating_offset, - first_floating_size, - ); - // Continue the search after the just-moved floating node - first_floating_oni = last_fixed.next; - continue; - } else { - assert(direction == .forward); - const new_parent_size = new_last_fixed_offset + last_fixed_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - parent_ni, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); + } + return; + } + + const new_loc: struct { + offset: u64, + prev: Node.Index.Optional, + } = new_loc: { + _, const parent_size = parent_ni.location(mf).resolve(mf); + + { + // See if there's space at the start of the parent. + const last_header_oni = parent_ni.lastHeader(mf); + const headers_end: u64 = if (last_header_oni.unwrap()) |last_header_ni| headers_end: { + const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); + break :headers_end last_header_off + last_header_size; + } else 0; + const limit: u64 = limit: { + const after_header_oni: Node.Index.Optional = after_header: { + if (last_header_oni.unwrap()) |last_header_ni| { + break :after_header last_header_ni.next(mf); + } + break :after_header parent_ni.first(mf); + }; + if (after_header_oni.unwrap()) |after_header_ni| { + break :limit after_header_ni.location(mf).resolve(mf)[0]; + } else { + break :limit parent_size; + } + }; + if (alignment.forward(headers_end) + new_size <= limit) { + // There's space here! + break :new_loc .{ + // Put ourselves at the *end* of this range, so that the free space remains at the start of the parent. + .offset = alignment.backward(limit - new_size), + .prev = last_header_oni, + }; } } - try mf.ensureCapacityForSetLocation(gpa); - if (last_fixed_ni == ni) { - // The original fixed node now has enough space - last_fixed_ni.setLocationAssumeCapacity( - mf, - old_last_fixed_offset, - new_size, + + // Otherwise, use space at the end of the parent, or make space there if necessary. + + const first_footer_oni = parent_ni.firstFooter(mf); + + // We know there is a node before the footer[s], because `ni` itself is such a node. + const prev_ni: Node.Index = if (first_footer_oni.unwrap()) |first_footer_ni| prev: { + break :prev first_footer_ni.prev(mf).unwrap().?; + } else prev: { + break :prev parent_ni.last(mf).unwrap().?; + }; + + const result_offset: u64 = result_offset: { + if (prev_ni == ni and alignment.check(old_offset)) { + // We're already at the end of the parent, and our offset is already well-aligned. + // The only reason we didn't simply grow in place earlier is that the parent wasn't + // big enough---but now we're resizing the parent anyway, so growing in-place stops + // us from unnecessarily moving! + break :result_offset old_offset; + } + // Otherwise, just move after the last node. + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :result_offset alignment.forward(prev_offset + prev_size); + }; + + const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| footers_size: { + const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_offset; + } else 0; + + const min_parent_size = result_offset + new_size + footers_size; + if (parent_size < min_parent_size) { + // Okay, at this point we're planning to expand the parent---so before we actually do + // that, let's first try the Linux "insert range" fast path. We didn't try it before now + // because it would have been more efficient to just move ourselves into existing space. + // + // If we were given a custom alignment, we cannot pass `grow_mode` directly into the + // "insert range" path, because that function is unaware of `new_alignment`. + const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact; + if (alignment.check(old_offset) and + try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode)) + { + // The Linux fast path did our job for us! + return; + } + + // Grow the parent and move to the end of the parent. + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, ); - return; - } - // Move a fixed node into trailing free space - if (last_fixed.flags.has_content) { - const parent_file_offset = parent_ni.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_last_fixed_offset, - parent_file_offset + new_last_fixed_offset, - last_fixed_size, + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + + break :new_loc .{ + .offset = result_offset, + .prev = .wrap(prev_ni), + }; + }; + + // We've found our new location in `parent_ni`, now to actually move ourselves there. + + // Footers need to move to a different place than the rest of our content. + const footers_size: u64, const footers_have_content: bool = footers: { + const first_footer_ni = ni.firstFooter(mf).unwrap() orelse { + break :footers .{ 0, false }; + }; + + var cur_ni = first_footer_ni; + var footers_have_content = false; + while (true) { + footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content; + const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf); + // Our footers' offsets must change to be at the end of our new size. + try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size); + cur_ni = cur_ni.next(mf).unwrap() orelse break; + } + + // This is the *new* offset because we already updated the offsets above. + const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf); + const footers_size = new_size - new_footers_offset; + + break :footers .{ footers_size, footers_have_content }; + }; + + if (ni.get(mf).flags.has_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_off + old_offset, + parent_file_off + new_loc.offset, + old_size - footers_size, + ); + if (footers_have_content) try mf.moveRange( + parent_file_off + old_offset + old_size - footers_size, + parent_file_off + new_loc.offset + new_size - footers_size, + footers_size, + ); + } else { + assert(!footers_have_content); + } + + try ni.setLocation(mf, gpa, new_loc.offset, new_size); + + if (new_loc.prev != ni.toOptional()) { + // We're potentially in a different place in `parent_ni`'s child list, so remove and re-add ourselves. + try mf.removeNodesFromChildList(gpa, ni, ni); + try mf.addNodesToChildListAfter(gpa, new_loc.prev, ni, ni); + } +} + +/// Attempts to grow `ni` to `new_size` using `FALLOCATE_FL_INSERT_RANGE` on Linux. This strategy +/// has the advantage that it does not require manually moving any bytes in the file, but has the +/// disadvantages that it may increase the file size more than necessary, and that it changes the +/// offsets of all following nodes, recursively. +/// +/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false` +/// without changing any nodes' locations or invalidating any slices. +/// +/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node +/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`, +/// the actual new size of `ni` may be greater than `new_size`. +fn growNodeViaInsertRange( + mf: *MappedFile, + gpa: Allocator, + ni: Node.Index, + new_size: u64, + grow_mode: GrowMode, +) Error!bool { + if (!is_linux or mf.flags.fallocate_insert_range_unsupported) { + return false; + } + + _, const old_size = ni.location(mf).resolve(mf); + + // We don't compute the size of the range yet, because depending on `grow_mode` we might want to + // bump it based on our sibling and parent nodes' alignments. However, we can do an early check + // for cases where we should obviously exit. + const requested_range_size = new_size - old_size; + if (!mf.flags.block_size.check(requested_range_size)) { + // The requested size isn't exactly aligned. + switch (grow_mode) { + .exact => return false, + .minimum => { + // We can still choose to allow it by increasing the size a bit, but we shouldn't do + // that if it would *significantly* increase the requested size. + const block_size = mf.flags.block_size.toByteUnits(); + if (requested_range_size < block_size * 2) { + // Bumping this size up to the next block boundary would be a quite significant + // increase; let's not do it. + return false; + } + }, + } + } + // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump + // the size a little more. + const min_range_size: u64 = s: { + const exact_size = new_size - old_size; + if (mf.flags.block_size.check(exact_size)) { + break :s exact_size; + } + switch (grow_mode) { + .exact => return false, + .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) { + // We're growing by at least a few blocks, so allow ourselves to bump the size + // slightly to give it the needed alignment. + break :s mf.flags.block_size.forward(exact_size); + } else { + return false; + }, + } + }; + assert(min_range_size > 0); + assert(mf.flags.block_size.check(min_range_size)); + + const range_file_offset: u64 = range_file_offset: { + const node_file_offset = ni.fileLocation(mf, false).offset; + const last_ni = ni.last(mf).unwrap() orelse { + // If `ni` has no children (i.e. is a leaf node), we need to insert exactly at its end. + const range_file_offset = node_file_offset + old_size; + if (!mf.flags.block_size.check(range_file_offset)) { + return false; + } + break :range_file_offset range_file_offset; + }; + const first_footer_oni = ni.firstFooter(mf); + const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: { + const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf); + break :size old_size - first_footer_offset; + } else 0; + const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: { + break :pre_footer first_footer_ni.prev(mf); + } else .wrap(last_ni); + const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: { + const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf); + break :end pre_footer_off + pre_footer_size; + } else 0; + + const min_file_offset = node_file_offset + pre_footer_end; + const max_file_offset = node_file_offset + old_size - footers_size; + // We can go anywhere between `min_file_offset` and `max_file_offset`. + const candidate_file_offset = mf.flags.block_size.forward(min_file_offset); + if (candidate_file_offset > max_file_offset) { + return false; + } + break :range_file_offset candidate_file_offset; + }; + assert(mf.flags.block_size.check(range_file_offset)); + + const range_size: u64 = range_size: { + // For this strategy to be valid, the number of bytes we insert needs to be compatible with + // the alignments of all nodes following us (and following our parents, their parents, etc). + // We also probably don't want to trigger too many "node moved" events, since doing that + // repeatedly could result in a lot of extra work. Therefore, while we traverse parents and + // siblings to check their alignment requirements, we will also set an arbitrary limit on + // the number of nodes we can move, and give up if we walk more than that. + const max_moved_nodes = 32; + var num_moved: u32 = 0; + var cur_ni = ni; + // Alignment required for `range_size`: initially the block size (required for the syscall), + // then updated as we traverse based on how the operation would affect surrounding nodes. + var need_range_align: Alignment = mf.flags.block_size.max(ni.alignment(mf)); + while (true) { + // `cur_ni` will grow as a result of the range insertion. Its size must be well-aligned. + need_range_align = need_range_align.max(cur_ni.alignment(mf)); + + // Siblings following `cur_ni` don't get bigger, but their offsets change. + while (cur_ni.next(mf).unwrap()) |next_ni| { + // Only floating children need well-aligned offsets. + if (next_ni.position(mf) == .floating) { + need_range_align = need_range_align.max(next_ni.alignment(mf)); + } + num_moved += 1; + if (num_moved > max_moved_nodes) return false; + cur_ni = next_ni; + } + + // Move up to the parent. + cur_ni = cur_ni.parent(mf).unwrap() orelse break; + } + // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment + // requirement to figure out whether we're actually going to insert a range. + if (need_range_align.check(requested_range_size)) { + break :range_size requested_range_size; + } + // Perhaps we're allowed to grow by more than `requested_range_size`? + switch (grow_mode) { + .exact => return false, + .minimum => { + const candidate_range_size = need_range_align.forward(min_range_size); + // Allow growing by up to 50% more than was requested. + if (candidate_range_size <= requested_range_size +| requested_range_size / 2) { + break :range_size candidate_range_size; + } else { + return false; + } + }, + } + }; + + // This `range_size` is compatible with everyone's alignment requirements, and we won't move too + // many nodes, so let's do it! + + mf.memory_map.write(mf.io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; + }; + + // If we happen to be inserting at the very end of the file, we need to resize the file instead + // of using `FALLOCATE_FL_INSERT_RANGE`. + if (range_file_offset == Node.Index.root.location(mf).resolve(mf)[1]) { + mf.memory_map.file.setLength(mf.io, range_file_offset + range_size) catch |err| switch (err) { + error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, + }; + } else { + while (true) switch (linux.errno(linux.fallocate( + mf.memory_map.file.handle, + linux.FALLOC.FL_INSERT_RANGE, + @intCast(range_file_offset), + @intCast(range_size), + ))) { + .SUCCESS => break, + .INTR => continue, + .NOSYS, .OPNOTSUPP => { + // After all that setup work, it turns out the operation is actually unsupported! + mf.flags.fallocate_insert_range_unsupported = true; + return false; + }, + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .IO => error.InputOutput, + .NODEV => error.NotFile, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .SPIPE => error.Unseekable, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, + }; + } + + // We did it! Now to update all the sizes and offsets. This loop is exactly the same shape as + // above, except we're updating locations instead of checking alignments. + var cur_ni = ni; + while (true) { + const this_offset, const this_old_size = cur_ni.location(mf).resolve(mf); + if (cur_ni == .root) { + try mf.ensureTotalCapacityPrecise(@intCast(this_old_size + range_size)); + } + try cur_ni.setLocation(mf, gpa, this_offset, this_old_size + range_size); + + while (cur_ni.next(mf).unwrap()) |next_ni| { + const next_old_offset, const next_size = next_ni.location(mf).resolve(mf); + try next_ni.setLocation(mf, gpa, next_old_offset + range_size, next_size); + cur_ni = next_ni; + } + + cur_ni = cur_ni.parent(mf).unwrap() orelse break; + } + + // The only thing left is to update the offsets of any footers inside of `ni`. + if (ni.firstFooter(mf).unwrap()) |first_footer_ni| { + var footer_ni = first_footer_ni; + while (true) { + const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf); + try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size); + footer_ni = footer_ni.next(mf).unwrap() orelse break; + } + } + + return true; +} + +/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes following its current +/// headers, so that the headers can grow into that space. +fn ensureAdditionalHeaderCapacity( + mf: *MappedFile, + gpa: Allocator, + parent_ni: Node.Index, + extra_capacity: u64, +) Error!void { + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const last_header_oni = parent_ni.lastHeader(mf); + const first_footer_oni = parent_ni.firstFooter(mf); + + const headers_size: u64 = headers_size: { + const last_header_ni = last_header_oni.unwrap() orelse break :headers_size 0; + const last_header_off, const last_header_size = last_header_ni.location(mf).resolve(mf); + break :headers_size last_header_off + last_header_size; + }; + + const footers_size: u64 = footers_size: { + const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_off; + }; + + const first_floating_oni: Node.Index.Optional = if (last_header_oni.unwrap()) |last_header_ni| first_floating: { + const after_header_ni = last_header_ni.next(mf).unwrap() orelse break :first_floating .none; + break :first_floating switch (after_header_ni.position(mf)) { + .header => unreachable, + .floating => .wrap(after_header_ni), + .footer => .none, + }; + } else first_floating: { + const first_ni = parent_ni.first(mf).unwrap() orelse break :first_floating .none; + break :first_floating switch (first_ni.position(mf)) { + .header => unreachable, + .floating => .wrap(first_ni), + .footer => .none, + }; + }; + const first_floating_ni = first_floating_oni.unwrap() orelse { + // This node has only headers and footers. + const min_parent_size = headers_size + extra_capacity + footers_size; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + return; + }; + + const last_floating_ni = if (first_footer_oni.unwrap()) |first_footer_ni| last_floating: { + break :last_floating first_footer_ni.prev(mf).unwrap().?; + } else last_floating: { + break :last_floating parent_ni.last(mf).unwrap().?; + }; + assert(last_floating_ni.position(mf) == .floating); // we know `parent_ni` contains at least `first_floating_ni` + + // Find the first floating child, if any, which does not overlap the new header space. + const first_good_floating_oni: Node.Index.Optional = first_good_floating: { + var floating_ni = first_floating_ni; + while (true) { + const floating_offset, _ = floating_ni.location(mf).resolve(mf); + if (floating_offset >= headers_size + extra_capacity) { + break :first_good_floating .wrap(floating_ni); + } + const next_ni = floating_ni.next(mf).unwrap() orelse { + break :first_good_floating .none; + }; + switch (next_ni.position(mf)) { + .header => unreachable, // after the last header + .floating => floating_ni = next_ni, + .footer => break :first_good_floating .none, + } + } + }; + + if (first_good_floating_oni == first_floating_ni.toOptional()) { + // None of the floating children are in our way! That means there's already enough space. + return; + } + + const last_moving_ni = if (first_good_floating_oni.unwrap()) |first_good_floating_ni| last_moving: { + break :last_moving first_good_floating_ni.prev(mf).unwrap().?; + } else last_moving: { + break :last_moving last_floating_ni; + }; + + // We are going to move all nodes between `first_floating_ni` and `last_moving_ni` to the end of + // the parent. We'll move all the node data in one big block. + + const moving_offset: u64 = first_floating_ni.location(mf).resolve(mf)[0]; + const moving_size: u64 = size: { + const last_moving_off, const last_moving_size = last_moving_ni.location(mf).resolve(mf); + break :size last_moving_off + last_moving_size - moving_offset; + }; + + var moving_alignment: Alignment = .@"1"; + var moving_has_content = false; // optimization: no need to move data if it's all uninitialized + { + var cur_ni = first_floating_ni; + while (true) { + moving_alignment = moving_alignment.max(cur_ni.alignment(mf)); + moving_has_content = moving_has_content or cur_ni.get(mf).flags.has_content; + if (cur_ni == last_moving_ni) break; + cur_ni = cur_ni.next(mf).unwrap().?; } - last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); - // Retry the previous nodes now that there is enough space - first_floating_oni = .wrap(last_fixed_ni); - last_fixed_ni = last_fixed.prev.unwrap().?; - direction = .reverse; } + + const first_free_offset = free_offset: { + const last_floating_off, const last_floating_size = last_floating_ni.location(mf).resolve(mf); + break :free_offset @max(last_floating_off + last_floating_size, headers_size + extra_capacity); + }; + // Alignment is a little tricky here. We don't necessarily want the new offset to be aligned to + // `moving_alignment` exactly, because if (e.g.) the first floating node is align(2) and the + // second is align(4), then the overall range we're moving may not be 4-byte aligned even though + // one of the nodes is. Instead, the old and new offsets must be congruent modulo the alignment. + const aligned_dest_offset = moving_alignment.forward(first_free_offset); + const dest_offset = aligned_dest_offset + (moving_offset - moving_alignment.backward(moving_offset)); + assert(dest_offset % moving_alignment.toByteUnits() == moving_offset % moving_alignment.toByteUnits()); + + // This expression is correct because `dest_offset` is after all floating nodes (except the ones + // we're moving there of course). + const min_parent_size = dest_offset + moving_size + footers_size; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } + + if (moving_has_content) { + const parent_file_off = parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_off + moving_offset, + parent_file_off + dest_offset, + moving_size, + ); + } + + // Remove everything between `first_floating_ni` and `last_moving_ni` from the linked list, then + // re-insert them in their new position. + try mf.removeNodesFromChildList(gpa, first_floating_ni, last_moving_ni); + try mf.addNodesToChildListBefore(gpa, first_footer_oni, first_floating_ni, last_moving_ni); + + // Finally, we need to update the locations of all of those nodes. + var cur_ni = first_floating_ni; + while (true) { + assert(cur_ni.position(mf) == .floating); + const old_offset, const old_size = cur_ni.location(mf).resolve(mf); + const new_offset = old_offset - moving_offset + dest_offset; + assert(cur_ni.alignment(mf).check(new_offset)); + try cur_ni.setLocation(mf, gpa, new_offset, old_size); + if (cur_ni == last_moving_ni) break; + cur_ni = cur_ni.next(mf).unwrap().?; + } +} + +/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current +/// footers, so that the footers can grow into that space. +fn ensureAdditionalFooterCapacity( + mf: *MappedFile, + gpa: Allocator, + parent_ni: Node.Index, + extra_capacity: u64, +) Error!void { + // This is way easier than the header case, because we don't need to actually move anything; we + // just need to expand the parent if there isn't space, and that will add padding after the + // parent's floating children, which is exactly where we want it. + + const first_footer_oni = parent_ni.firstFooter(mf); + + _, const parent_size = parent_ni.location(mf).resolve(mf); + + const footers_size: u64 = footers_size: { + const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0; + const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf); + break :footers_size parent_size - first_footer_off; + }; + + const header_and_floating_end: u64 = end: { + const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: { + break :before_footers first_footer_ni.prev(mf); + } else before_footers: { + break :before_footers parent_ni.last(mf); + }; + const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0; + const offset, const size = before_footers_ni.location(mf).resolve(mf); + break :end offset + size; + }; + + assert(header_and_floating_end + footers_size <= parent_size); + + const min_parent_size = header_and_floating_end + footers_size + extra_capacity; + if (parent_size < min_parent_size) { + const new_parent_size = parent_ni.alignment(mf).forward( + min_parent_size +| min_parent_size / growth_factor, + ); + try mf.growNode(gpa, parent_ni, new_parent_size, .minimum); + } +} + +fn removeNodesFromChildList( + mf: *MappedFile, + gpa: Allocator, + first_remove_ni: Node.Index, + last_remove_ni: Node.Index, +) Allocator.Error!void { + const parent_ni = first_remove_ni.parent(mf).unwrap().?; + assert(last_remove_ni.parent(mf).unwrap().? == parent_ni); + + const prev_oni = first_remove_ni.prev(mf); + const next_oni = last_remove_ni.next(mf); + + if (prev_oni.unwrap()) |prev_ni| { + assert(prev_ni.next(mf).unwrap().? == first_remove_ni); + try prev_ni.setNext(gpa, next_oni, mf); + } else { + assert(parent_ni.first(mf).unwrap().? == first_remove_ni); + parent_ni.get(mf).first = next_oni; + } + + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.prev(mf).unwrap().? == last_remove_ni); + next_ni.get(mf).prev = prev_oni; + } else { + assert(parent_ni.last(mf).unwrap().? == last_remove_ni); + parent_ni.get(mf).last = prev_oni; + } +} +/// Assumes `first_add_ni` and `last_add_ni` are connected, and that all nodes in between them +/// already have their `parent` field correctly populated. +/// +/// To add a single node, set `first_add_ni` equal to `last_add_ni`. +fn addNodesToChildListBefore( + mf: *MappedFile, + gpa: Allocator, + /// `null` means to add at the end of the parent. + next_oni: Node.Index.Optional, + first_add_ni: Node.Index, + last_add_ni: Node.Index, +) Allocator.Error!void { + const parent_ni = first_add_ni.parent(mf).unwrap().?; + assert(last_add_ni.parent(mf).unwrap().? == parent_ni); + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.parent(mf).unwrap().? == parent_ni); + } + + const prev_oni: Node.Index.Optional = if (next_oni.unwrap()) |next_ni| prev: { + break :prev next_ni.prev(mf); + } else prev: { + break :prev parent_ni.last(mf); + }; + + first_add_ni.get(mf).prev = prev_oni; + try last_add_ni.setNext(gpa, next_oni, mf); + + if (prev_oni.unwrap()) |prev_ni| { + assert(prev_ni.next(mf) == next_oni); + try prev_ni.setNext(gpa, .wrap(first_add_ni), mf); + } else { + assert(parent_ni.first(mf) == next_oni); + parent_ni.get(mf).first = .wrap(first_add_ni); + } + + if (next_oni.unwrap()) |next_ni| { + assert(next_ni.prev(mf) == prev_oni); + next_ni.get(mf).prev = .wrap(last_add_ni); + } else { + assert(parent_ni.last(mf) == prev_oni); + parent_ni.get(mf).last = .wrap(last_add_ni); + } +} +fn addNodesToChildListAfter( + mf: *MappedFile, + gpa: Allocator, + /// `null` means to add at the start of the parent. + prev_oni: Node.Index.Optional, + first_add_ni: Node.Index, + last_add_ni: Node.Index, +) Allocator.Error!void { + const next_oni: Node.Index.Optional = next: { + if (prev_oni.unwrap()) |prev_ni| break :next prev_ni.next(mf); + const parent_ni = first_add_ni.parent(mf).unwrap().?; + break :next parent_ni.first(mf); + }; + return mf.addNodesToChildListBefore(gpa, next_oni, first_add_ni, last_add_ni); } fn realignNode( @@ -1206,107 +2068,37 @@ fn realignNode( gpa: Allocator, ni: Node.Index, new_alignment: Alignment, - opts: Node.Index.RealignNodeOptions, -) (Allocator.Error || Io.Cancelable || IoError)!void { +) Error!void { mf.nodes_lock.assertUnlocked(); - const node = ni.get(mf); - { - const prev_alignment = node.flags.alignment; - node.flags.alignment = new_alignment; - if (new_alignment.compare(.lte, prev_alignment)) return; - } - - const old_offset, const size = node.location().resolve(mf); - const parent_ni = node.parent.unwrap() orelse { - assert(ni == .root); - return mf.resizeNode(gpa, ni, size); - }; - - const new_size = new_alignment.forward(@intCast(size)); - if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); - - _, const parent_size = parent_ni.location(mf).resolve(mf); - const trailing_end = trailing_end: { - const next_ni = node.next.unwrap() orelse break :trailing_end parent_size; - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }; - - if (opts.try_backwards) { - const backward_offset = new_alignment.backward(@intCast(old_offset)); - const prev_end = prev_end: { - const prev_ni = node.prev.unwrap() orelse break :prev_end 0; - const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); - break :prev_end prev_offset + prev_size; - }; - - if (backward_offset >= prev_end) { - try mf.ensureCapacityForSetLocation(gpa); - - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_offset) + backward_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..@intCast(old_file_offset + size)], 0); - } - - if (backward_offset + new_size <= trailing_end) { - ni.setLocationAssumeCapacity(mf, backward_offset, new_size); - } else { - ni.setLocationAssumeCapacity(mf, backward_offset, size); - try mf.resizeNode(gpa, ni, new_size); - } - - return; + const old_offset, const old_size = ni.location(mf).resolve(mf); + + if (ni == .root or ni.position(mf) != .floating) { + // Only this node's size is aligned, not its offset. + if (!new_alignment.check(old_size)) { + assert(new_alignment.compare(.gt, ni.alignment(mf))); + try mf.growNode( + gpa, + ni, + new_alignment.forward(old_size), + .exact, // because `growNode` is not aware that the size needs to match `new_alignment` + ); } - } - - const forward_offset = new_alignment.forward(@intCast(old_offset)); - if (forward_offset + new_size <= trailing_end) { - // Shift into the free space if possible - try mf.ensureCapacityForSetLocation(gpa); - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - old_offset) + forward_offset; - if (new_file_offset < old_file_offset + size) { - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - } else try mf.moveRange(old_file_offset, new_file_offset, size); - @memset(mf.memory_map.memory[@intCast(new_file_offset + size)..][0..@intCast(new_size - size)], 0); - } - - ni.setLocationAssumeCapacity(mf, forward_offset, new_size); } else { - const temp_size = new_alignment.forward(@intCast(new_size + 1)); - try mf.resizeNode(gpa, ni, temp_size); - const new_offset, _ = ni.location(mf).resolve(mf); - - try mf.ensureCapacityForSetLocation(gpa); - - // Non-fixed nodes may now be aligned if the resize moved them - const new_forward_offset = new_alignment.forward(@intCast(new_offset)); - const final_offset = if (new_forward_offset != new_offset) final_offset: { - if (node.flags.has_content) { - const old_file_offset = ni.fileLocation(mf, false).offset; - const new_file_offset = (old_file_offset - new_offset) + new_forward_offset; - @memmove( - mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], - mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], - ); - @memset(mf.memory_map.memory[@intCast(old_file_offset)..@intCast(new_file_offset)], 0); - } - - break :final_offset new_forward_offset; - } else new_offset; - - ni.setLocationAssumeCapacity(mf, final_offset, new_size); + // This is a floating node, so its size and offset are both aligned. + if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) { + assert(new_alignment.compare(.gt, ni.alignment(mf))); + try mf.growFloatingNodeWithAlignment( + gpa, + ni, + new_alignment, + new_alignment.forward(old_size), + .minimum, + ); + } } + + ni.get(mf).flags.alignment = new_alignment; } fn updateWriters(mf: *MappedFile) void { @@ -1317,10 +2109,47 @@ fn updateWriters(mf: *MappedFile) void { } } -fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { - // make a copy of this node at the new location - try mf.copyRange(old_file_offset, new_file_offset, size); - // delete the copy of this node at the old location +fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) Error!void { + if (old_file_offset == new_file_offset) return; + + if (old_file_offset >= new_file_offset + size or + new_file_offset >= old_file_offset + size) + { + const n = try mf.copyFileRange( + mf.memory_map.file, + old_file_offset, + new_file_offset, + size, + ); + @memcpy( + mf.memory_map.memory[@intCast(new_file_offset + n)..][0..@intCast(size - n)], + mf.memory_map.memory[@intCast(old_file_offset + n)..][0..@intCast(size - n)], + ); + + try mf.zeroRange(old_file_offset, size); + + return; + } + + // TODO: if the non-overlapping region is greater than or equal to a filesystem block, is it + // ever worth doing multiple `copyFileRange` calls instead of a big `@memmove`? + + @memmove( + mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(size)], + mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], + ); + + if (new_file_offset > old_file_offset) { + const clear_size = new_file_offset - old_file_offset; + assert(clear_size < size); + try mf.zeroRange(old_file_offset, clear_size); + } else { + const clear_size = old_file_offset - new_file_offset; + assert(clear_size < size); + try mf.zeroRange(new_file_offset + size, clear_size); + } +} +fn zeroRange(mf: *MappedFile, file_offset: u64, size: u64) Error!void { if (is_linux and !mf.flags.fallocate_punch_hole_unsupported and size >= mf.flags.block_size.toByteUnits() * 2 - 1) @@ -1328,147 +2157,149 @@ fn moveRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: while (true) switch (linux.errno(linux.fallocate( mf.memory_map.file.handle, linux.FALLOC.FL_PUNCH_HOLE | linux.FALLOC.FL_KEEP_SIZE, - @intCast(old_file_offset), + @intCast(file_offset), @intCast(size), ))) { .SUCCESS => return, .INTR => continue, - .BADF, .FBIG, .INVAL => unreachable, - .IO => return error.InputOutput, - .NODEV => return error.NotFile, - .NOSPC => return error.NoSpaceLeft, .NOSYS, .OPNOTSUPP => { mf.flags.fallocate_punch_hole_unsupported = true; break; // fall back to slow path }, - .PERM => return error.PermissionDenied, - .SPIPE => return error.Unseekable, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .IO => error.InputOutput, + .NODEV => error.NotFile, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .SPIPE => error.Unseekable, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, }; } - @memset(mf.memory_map.memory[@intCast(old_file_offset)..][0..@intCast(size)], 0); + @memset(mf.memory_map.memory[@intCast(file_offset)..][0..@intCast(size)], 0); } - -fn copyRange(mf: *MappedFile, old_file_offset: u64, new_file_offset: u64, size: u64) (Io.Cancelable || IoError)!void { - const copy_size = try mf.copyFileRange(mf.memory_map.file, old_file_offset, new_file_offset, size); - if (copy_size < size) @memcpy( - mf.memory_map.memory[@intCast(new_file_offset + copy_size)..][0..@intCast(size - copy_size)], - mf.memory_map.memory[@intCast(old_file_offset + copy_size)..][0..@intCast(size - copy_size)], - ); -} - fn copyFileRange( mf: *MappedFile, old_file: Io.File, old_file_offset: u64, new_file_offset: u64, size: u64, -) (Io.Cancelable || IoError)!u64 { +) Error!u64 { + if (!is_linux or mf.flags.copy_file_range_unsupported) { + return 0; + } + + const min_size = mf.flags.block_size.toByteUnits() * 2 - 1; + if (size < min_size) return 0; + const io = mf.io; - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map.write(io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; var remaining_size = size; - if (is_linux and !mf.flags.copy_file_range_unsupported) { - var old_file_offset_mut: i64 = @intCast(old_file_offset); - var new_file_offset_mut: i64 = @intCast(new_file_offset); - while (remaining_size >= mf.flags.block_size.toByteUnits() * 2 - 1) { - const copy_len = linux.copy_file_range( - old_file.handle, - &old_file_offset_mut, - mf.memory_map.file.handle, - &new_file_offset_mut, - @intCast(remaining_size), - 0, - ); - switch (linux.errno(copy_len)) { - .SUCCESS => { - if (copy_len == 0) break; - remaining_size -= copy_len; - if (remaining_size == 0) break; - }, - .INTR => continue, - .BADF, .FBIG, .INVAL, .OVERFLOW => unreachable, - .IO => return error.InputOutput, - .ISDIR => return error.IsDir, - .NOMEM => return error.SystemResources, - .NOSPC => return error.NoSpaceLeft, - .NOSYS, .OPNOTSUPP, .XDEV => { - mf.flags.copy_file_range_unsupported = true; - break; - }, - .PERM => return error.PermissionDenied, - .TXTBSY => return error.FileBusy, - else => |e| return std.posix.unexpectedErrno(e), - } + var old_file_offset_mut: i64 = @intCast(old_file_offset); + var new_file_offset_mut: i64 = @intCast(new_file_offset); + while (remaining_size >= min_size) { + const copy_len = linux.copy_file_range( + old_file.handle, + &old_file_offset_mut, + mf.memory_map.file.handle, + &new_file_offset_mut, + @intCast(remaining_size), + 0, + ); + switch (linux.errno(copy_len)) { + .SUCCESS => { + if (copy_len == 0) break; + remaining_size -= copy_len; + if (remaining_size == 0) break; + }, + .INTR => continue, + .NOSYS, .OPNOTSUPP, .XDEV => { + mf.flags.copy_file_range_unsupported = true; + break; + }, + else => |e| { + mf.io_err = switch (e) { + .SUCCESS, .INTR, .NOSYS, .OPNOTSUPP, .XDEV => unreachable, // handled above + .BADF => unreachable, + .FBIG => unreachable, + .INVAL => unreachable, + .OVERFLOW => unreachable, + .IO => error.InputOutput, + .ISDIR => error.IsDir, + .NOMEM => error.SystemResources, + .NOSPC => error.NoSpaceLeft, + .PERM => error.PermissionDenied, + .TXTBSY => error.FileBusy, + else => std.posix.unexpectedErrno(e), + }; + return error.MappedFileIo; + }, } } return size - remaining_size; } -fn ensureCapacityForSetLocation(mf: *MappedFile, gpa: Allocator) Allocator.Error!void { - try mf.large.ensureUnusedCapacity(gpa, 2); - try mf.updates.ensureUnusedCapacity(gpa, 2); -} - pub fn ensureTotalCapacity(mf: *MappedFile, new_capacity: usize) Error!void { - mf.ensureTotalCapacityInner(new_capacity) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} -fn ensureTotalCapacityInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { if (mf.memory_map.memory.len >= new_capacity) return; - try mf.ensureTotalCapacityPreciseInner(new_capacity +| new_capacity / growth_factor); + try mf.ensureTotalCapacityPrecise(new_capacity +| new_capacity / growth_factor); } pub fn ensureTotalCapacityPrecise(mf: *MappedFile, new_capacity: usize) Error!void { - mf.ensureTotalCapacityPreciseInner(new_capacity) catch |err| switch (err) { - error.OutOfMemory, - error.Canceled, - => |e| return e, - - else => |e| { - mf.io_err = e; - return error.MappedFileIo; - }, - }; -} -fn ensureTotalCapacityPreciseInner(mf: *MappedFile, new_capacity: usize) (Allocator.Error || Io.Cancelable || IoError)!void { if (mf.memory_map.memory.len >= new_capacity) return; const io = mf.io; - const aligned_capacity = mf.flags.block_size.forward(new_capacity); + const aligned_capacity: usize = @intCast( + mf.flags.block_size.forward(new_capacity), + ); if (mf.memory_map.memory.len > 0) { if (mf.memory_map.setLength(io, aligned_capacity)) |_| { return; } else |err| switch (err) { error.OperationUnsupported => {}, - else => |e| return e, + error.OutOfMemory, error.Canceled => |e| return e, + else => |e| { + mf.io_err = e; + return error.MappedFileIo; + }, } - mf.memory_map.write(io) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForWriting => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map.write(io) catch |err| { + mf.io_err = switch (err) { + error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForWriting => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; unmap(mf); } const file = mf.memory_map.file; - mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| switch (err) { - error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking - error.NotOpenForReading => return error.Unexpected, // we definitely opened the file for writing - else => |e| return e, + mf.memory_map = Io.File.MemoryMap.create(io, file, .{ .len = aligned_capacity }) catch |err| { + mf.io_err = switch (err) { + error.OutOfMemory, error.Canceled => |e| return e, + error.WouldBlock => error.Unexpected, // file was not opened as non-blocking + error.NotOpenForReading => error.Unexpected, // we definitely opened the file for writing + else => |e| e, + }; + return error.MappedFileIo; }; } @@ -1487,7 +2318,7 @@ pub fn flush(mf: *MappedFile) (Io.Cancelable || error{MappedFileIo})!void { error.WouldBlock, // file was not opened as non-blocking error.NotOpenForWriting, // we definitely opened the file for writing - error.ReadOnlyFileSystem, + error.ReadOnlyFileSystem, // again, we opened the file for writing => { mf.io_err = error.Unexpected; return error.MappedFileIo; @@ -1512,211 +2343,276 @@ fn verify(mf: *MappedFile) void { assert(root.next == .none); mf.verifyNode(.root); } - fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { const parent = parent_ni.get(mf); - const parent_offset, const parent_size = parent.location().resolve(mf); - var prev_ni: Node.Index = .none; + _, const parent_size = parent.location().resolve(mf); + + var prev_oni: Node.Index.Optional = .none; var prev_end: u64 = 0; - var ni = parent.first; - while (true) { - if (ni == .none) { - assert(parent.last == prev_ni); - return; - } + var prev_pos: Node.Position = .header; + var oni = parent.first; + while (oni.unwrap()) |ni| { const node = ni.get(mf); - assert(node.parent == parent_ni); + assert(node.parent == parent_ni.toOptional()); + assert(node.prev == prev_oni); + const offset, const size = node.location().resolve(mf); - assert(node.flags.alignment.check(@intCast(offset))); - assert(node.flags.alignment.check(@intCast(size))); const end = offset + size; - assert(end <= parent_offset + parent_size); + + assert(node.flags.alignment.check(size)); assert(offset >= prev_end); - assert(node.prev == prev_ni); + assert(end <= parent_size); + + switch (node.flags.position) { + .header => { + assert(prev_pos == .header); + assert(offset == prev_end); + }, + .floating => { + assert(prev_pos != .footer); + assert(node.flags.alignment.check(offset)); + }, + .footer => { + if (prev_pos == .footer) assert(offset == prev_end); + }, + } + mf.verifyNode(ni); - prev_ni = ni; + + prev_oni = .wrap(ni); prev_end = end; - ni = node.next; + prev_pos = ni.position(mf); + + oni = node.next; + } + assert(parent.last == prev_oni); + if (prev_pos == .footer) { + assert(prev_end == parent_size); } } -const testing = std.testing; -fn testVerifyContent(mf: *@This(), ni: Node.Index, value: u8, init_len: usize) !void { - // Not using std.mem.allEqual, so we can get useful output - const slice = ni.slice(mf); - var buf: [256]u8 = undefined; - @memset(buf[0..init_len], value); - @memset(buf[init_len..], 0); - try testing.expectEqualSlices(u8, buf[0..slice.len], slice); +test "fuzz node operations" { + try std.testing.fuzz({}, fuzzOneNodeOperations, .{}); } +fn fuzzOneNodeOperations(_: void, smith: *std.testing.Smith) anyerror!void { + const gpa = std.testing.allocator; + const io = std.testing.io; -test { - const gpa = testing.allocator; - - var tmp_dir = testing.tmpDir(.{}); + var tmp_dir = std.testing.tmpDir(.{}); defer tmp_dir.cleanup(); - var file = try tmp_dir.dir.createFile(testing.io, "test.mf", .{ .read = true }); - defer file.close(testing.io); + var tmp_file = try tmp_dir.dir.createFile(io, "test.mf", .{ .read = true }); + defer tmp_file.close(io); - var mf = try init(file, gpa, testing.io); + var mf: MappedFile = try .init(tmp_file, gpa, io); defer mf.deinit(gpa); - const a = try mf.addFirstChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); - const c = try mf.addLastChildNode(gpa, .root, .{ .fixed = true, .alignment = .@"4" }); - const b = try mf.addNodeAfter(gpa, a, .{ .fixed = true, .alignment = .@"16" }); - const d = try mf.addNodeAfter(gpa, b, .{ .alignment = .@"4" }); - - const a_init_size = 8; - const b_init_size = 16; - const c_init_size = 24; - const d_init_size = 28; - - // Resize without content - { - // Verify size is aligned forward - try d.resize(&mf, gpa, d_init_size - 1); - try a.resize(&mf, gpa, a_init_size - 2); - try c.resize(&mf, gpa, c_init_size); - try b.resize(&mf, gpa, b_init_size); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, const b_size = b.location(&mf).resolve(&mf); - const c_loc, const c_size = c.location(&mf).resolve(&mf); - _, const d_size = d.location(&mf).resolve(&mf); - try testing.expect(a_size >= a_init_size); - try testing.expect(b_size >= b_init_size); - try testing.expect(c_size >= c_init_size); - try testing.expect(d_size >= d_init_size); - try testing.expect(b_loc >= a_loc + a_size); - try testing.expect(c_loc >= b_loc + b_size); - } - - const a_exp_size = 24; - const b_exp_size = 28; - const c_exp_size = 48; - const d_exp_size = 32; - - // Resize with content - { - @memset(a.slice(&mf)[0..a_init_size], 0xaa); - @memset(b.slice(&mf)[0..b_init_size], 0xbb); - @memset(c.slice(&mf)[0..c_init_size], 0xcc); - @memset(d.slice(&mf)[0..d_init_size], 0xdd); - - try a.resize(&mf, gpa, a_exp_size); - try b.resize(&mf, gpa, b_exp_size); - try c.resize(&mf, gpa, c_exp_size); - try d.resize(&mf, gpa, d_exp_size); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, const b_size = b.location(&mf).resolve(&mf); - const c_loc, const c_size = c.location(&mf).resolve(&mf); - _, const d_size = d.location(&mf).resolve(&mf); - try testing.expect(a_size >= a_exp_size); - try testing.expect(b_size >= b_exp_size); - try testing.expect(c_size >= c_exp_size); - try testing.expect(d_size >= d_exp_size); - try testing.expect(b_loc >= a_loc + a_size); - try testing.expect(c_loc >= b_loc + b_size); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, b, 0xbb, b_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - } - - const child_init: []const struct { Alignment, usize } = &.{ - .{ .@"16", 16 }, - .{ .@"1", 1 }, - .{ .@"1", 19 }, - .{ .@"1", 3 }, - .{ .@"8", 30 }, - .{ .@"2", 5 }, - .{ .@"1", 60 }, - .{ .@"2", 2 }, - .{ .@"16", 32 }, + var nodes: std.array_hash_map.Auto(MappedFile.Node.Index, struct { + parent: MappedFile.Node.Index.Optional, + position: MappedFile.Node.Position, + num_headers: u32, + num_footers: u32, + /// For leaf nodes, this value is whether we have initialized the contents of the node or + /// not. For non-leaf nodes, this value is unspecified and should be ignored. + initialized: bool, + }) = .empty; + defer nodes.deinit(gpa); + + // When initializing a leaf node, we will place its 4-byte node index at the start of its range, + // and the bitwise NOT of its node index at the end of its range (both little-endian). This is + // just a simple way to put distinct values we can validate at all node boundaries. + + try nodes.putNoClobber(gpa, .root, .{ + .parent = .none, + .position = .floating, + .num_headers = 0, + .num_footers = 0, + .initialized = false, + }); + + // Allow a range of alignments, with most nodes having a small alignment of 1--32 bytes (most + // commonly 1 byte), but with a small chance for some large alignments too. + const alignment_weights: []const std.testing.Smith.Weight = comptime &.{ + .value(Alignment, .@"1", 20), + .value(Alignment, .@"2", 5), + .value(Alignment, .@"4", 5), + .value(Alignment, .@"8", 5), + .value(Alignment, .@"16", 5), + .value(Alignment, .@"32", 5), + .value(Alignment, .fromByteUnits(0x200), 1), + .value(Alignment, .fromByteUnits(0x400), 1), + .value(Alignment, .fromByteUnits(0x800), 1), + .value(Alignment, .fromByteUnits(0x1000), 1), + .value(Alignment, .fromByteUnits(0x2000), 1), + .value(Alignment, .fromByteUnits(0x4000), 1), + .value(Alignment, .fromByteUnits(0x8000), 1), }; - var children: [child_init.len]Node.Index = undefined; + const min_nonzero_size = 2 * @sizeOf(MappedFile.Node.Index); + const max_size = 0x10_000; + const initial_size_weights: []const std.testing.Smith.Weight = comptime &.{ + // initially, make nodes just as likely to be empty as non-empty + .value(u64, 0, max_size - min_nonzero_size + 1), + .rangeAtMost(u64, min_nonzero_size, max_size, 1), + }; + + while (!smith.eos()) switch (smith.value(enum { add, resize, realign })) { + .add => { + const parent_ni = nodes.keys()[smith.index(nodes.count())]; + + const alignment = smith.valueWeighted(Alignment, alignment_weights); + const size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); - // Differently-aligned fixed sibling nodes - { - for (children[0 .. children.len - 1], child_init[0 .. children.len - 1], 0..) |*ni, opts, i| { - ni.* = try mf.addLastChildNode(gpa, b, .{ - .alignment = opts.@"0", - .size = opts.@"1", - .fixed = true, + const position = smith.valueWeighted(Node.Position, comptime &.{ + // make floating nodes more common than header and footer nodes + .value(Node.Position, .header, 1), + .value(Node.Position, .footer, 1), + .value(Node.Position, .floating, 4), }); + const new_ni: Node.Index = switch (position) { + .header => new_ni: { + const parent_info = nodes.getPtr(parent_ni).?; + const prev_oni: Node.Index.Optional = prev_oni: { + const n = smith.valueRangeAtMost(u32, 0, parent_info.num_headers); + if (n == 0) break :prev_oni .none; + var cur_ni = parent_ni.first(&mf).unwrap().?; + for (1..n) |_| cur_ni = cur_ni.next(&mf).unwrap().?; + break :prev_oni .wrap(cur_ni); + }; + const new_ni = try parent_ni.addHeaderChildAfter(&mf, gpa, prev_oni, .{ + .size = size, + .alignment = alignment, + }); + parent_info.num_headers += 1; + break :new_ni new_ni; + }, + + .floating => try parent_ni.addFloatingChild(&mf, gpa, .{ + .size = size, + .alignment = alignment, + }), + + .footer => new_ni: { + const parent_info = nodes.getPtr(parent_ni).?; + const next_oni: Node.Index.Optional = next_oni: { + const n = smith.valueRangeAtMost(u32, 0, parent_info.num_footers); + if (n == 0) break :next_oni .none; + var cur_ni = parent_ni.last(&mf).unwrap().?; + for (1..n) |_| cur_ni = cur_ni.prev(&mf).unwrap().?; + break :next_oni .wrap(cur_ni); + }; + const new_ni = try parent_ni.addFooterChildBefore(&mf, gpa, next_oni, .{ + .size = size, + .alignment = alignment, + }); + parent_info.num_footers += 1; + break :new_ni new_ni; + }, + }; + + const initialize = size > 0 and smith.value(bool); + if (initialize) { + const slice = new_ni.slice(&mf); + std.mem.writeInt(u32, slice[0..4], @backingInt(new_ni), .little); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(new_ni), .little); + } + + try nodes.putNoClobber(gpa, new_ni, .{ + .parent = .wrap(parent_ni), + .position = position, + .num_headers = 0, + .num_footers = 0, + .initialized = initialize, + }); + }, + + .resize => { + const ni = nodes.keys()[smith.index(nodes.count())]; + const node_info = nodes.getPtr(ni).?; + + const alignment = ni.alignment(&mf); + + if (ni.first(&mf) == .none and smith.value(bool)) { + // Since this is a leaf node, we can use `resizeLeaf`. + const new_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); + try ni.resizeLeaf(&mf, gpa, new_size); + if (new_size == 0) { + node_info.initialized = false; + } + } else { + const min_size = alignment.forward(smith.valueWeighted(u64, initial_size_weights)); + try ni.ensureMinimumSize(&mf, gpa, min_size); + } + + if (ni.first(&mf) == .none) { + // This is a leaf node, so it can contain data. + if (node_info.initialized) { + // It's already initialized, so we'll write the expected footer at the new end. + const slice = ni.slice(&mf); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); + } else if (ni.location(&mf).resolve(&mf)[1] > 0) { + // It was uninitialized, but it has a non-zero size, so maybe we'd like to + // initialize it now? + if (smith.value(bool)) { + node_info.initialized = true; + const slice = ni.slice(&mf); + std.mem.writeInt(u32, slice[0..4], @backingInt(ni), .little); + std.mem.writeInt(u32, slice[slice.len - 4 ..][0..4], ~@backingInt(ni), .little); + } + } + } + }, + .realign => { + const ni = nodes.keys()[smith.index(nodes.count())]; + const new_alignment = smith.valueWeighted(Alignment, alignment_weights); + if (new_alignment.compare(.gt, ni.alignment(&mf))) { + _, const old_size = ni.location(&mf).resolve(&mf); + try ni.realign(&mf, gpa, new_alignment); + if (ni.first(&mf) == .none and nodes.get(ni).?.initialized) { + const slice = ni.slice(&mf); + @memmove(slice[slice.len - 4 ..][0..4], slice[old_size - 4 ..][0..4]); + } + } + }, + }; - @memset(ni.slice(&mf)[0..opts.@"1"], @intCast(i + 1)); + mf.verify(); + + for (nodes.keys(), nodes.values()) |ni, expected| { + try std.testing.expectEqual(expected.parent, ni.parent(&mf)); + if (ni != .root) { + try std.testing.expectEqual(expected.position, ni.position(&mf)); } - // Shift differently-aligned nodes by inserting a node - children[children.len - 1] = try mf.addNodeAfter(gpa, children[3], .{ - .alignment = child_init[children.len - 1].@"0", - .size = child_init[children.len - 1].@"1", - .fixed = true, - }); - @memset(children[children.len - 1].slice(&mf), @intCast(children.len)); - - mf.verify(); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + { + var num_headers: u32 = 0; + var header_oni = ni.lastHeader(&mf); + while (header_oni.unwrap()) |header_ni| { + num_headers += 1; + header_oni = header_ni.prev(&mf); + } + try std.testing.expectEqual(expected.num_headers, num_headers); } - } - - // Shifting child nodes forward due via resize of parent.prev - { - try testing.expect(a.location(&mf).resolve(&mf)[1] < 64); - try a.resize(&mf, gpa, 64); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + { + var num_footers: u32 = 0; + var footer_oni = ni.firstFooter(&mf); + while (footer_oni.unwrap()) |footer_ni| { + num_footers += 1; + footer_oni = footer_ni.next(&mf); + } + try std.testing.expectEqual(expected.num_footers, num_footers); } - } - - // Re-align last node into trailing free space within parent - { - try b.resize(&mf, gpa, b.location(&mf).resolve(&mf)[1] + 64); - - const last = children[children.len - 2]; - try last.realign(&mf, gpa, .@"4", true); - mf.verify(); - - for (children, child_init, 0..) |ni, opts, i| - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - } - - // Re-align, shifting sibling nodes - { - try children[1].realign(&mf, gpa, .@"8", true); - mf.verify(); - - for (children, child_init, 0..) |ni, opts, i| - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - } - - // Shrink and shift start of trailing node into free space - { - try mf.shrinkNode(gpa, a, 16, true); - mf.verify(); - - const a_loc, const a_size = a.location(&mf).resolve(&mf); - const b_loc, _ = b.location(&mf).resolve(&mf); - try testing.expectEqual(b_loc, a_loc + a_size); - - try testVerifyContent(&mf, a, 0xaa, a_init_size); - try testVerifyContent(&mf, c, 0xcc, c_init_size); - try testVerifyContent(&mf, d, 0xdd, d_init_size); - for (children, child_init, 0..) |ni, opts, i| { - try testVerifyContent(&mf, ni, @intCast(i + 1), opts.@"1"); + + if (ni.first(&mf) == .none and expected.initialized) { + const slice = ni.sliceConst(&mf); + if (slice.len > 0) { + try std.testing.expect(slice.len >= min_nonzero_size); + const header = std.mem.readInt(u32, slice[0..4], .little); + const footer = std.mem.readInt(u32, slice[slice.len - 4 ..][0..4], .little); + try std.testing.expectEqual(@backingInt(ni), header); + try std.testing.expectEqual(~@backingInt(ni), footer); + } } } } diff --git a/src/main.zig b/src/main.zig index 7e33a7fdf2d15ec9fa8e1417c0533fb702d22dfd..4584c950abd7855d781492de48e25204a8de10c1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -36,6 +36,7 @@ const Module = @import("Module.zig"); test { _ = @import("codegen.zig"); + _ = @import("link/MappedFile.zig"); } const thread_stack_size = 60 << 20; -- 2.54.0 From 7ece5e656da5cdd991f9fd1cb289f71c586a9210 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 24 Aug 2026 08:39:50 +0100 Subject: [PATCH 4/6] link.MappedFile: remove unnecessary child iterator There are now functions for `first`, `last`, `prev`, and `next`, so there is no need for a separate abstraction wrapping a simple linked list iteration. --- src/link/Coff.zig | 75 ++++++++++++++++++++++------------------- src/link/Elf2.zig | 27 +++++++++------ src/link/MappedFile.zig | 18 ---------- 3 files changed, 56 insertions(+), 64 deletions(-) diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 09ea7b6a25e8dcba3cec1d1996266de05863b809..b7045cac54bb00945e2b9a6458d422151aca54ff 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -3546,16 +3546,20 @@ fn objectSectionMapIndex( try coff.symbols.ensureUnusedCapacity(gpa, 1); const parent_ni = parent.node(coff); var prev_oni: MappedFile.Node.Index.Optional = .none; - var next_it = parent_ni.children(&coff.mf); - while (next_it.next()) |next_ni| switch (std.mem.order( - u8, - name_slice, - coff.getNode(next_ni).object_section.name(coff).toSlice(coff), - )) { - .lt => break, - .eq => unreachable, - .gt => prev_oni = .wrap(next_ni), - }; + { + var child_oni = parent_ni.first(&coff.mf); + while (child_oni.unwrap()) |child_ni| : (child_oni = child_ni.next(&coff.mf)) { + switch (std.mem.order( + u8, + name_slice, + coff.getNode(child_ni).object_section.name(coff).toSlice(coff), + )) { + .lt => break, + .eq => unreachable, + .gt => prev_oni = .wrap(child_ni), + } + } + } const ni = try parent_ni.addHeaderChildAfter(&coff.mf, gpa, prev_oni, .{ .alignment = alignment, }); @@ -7025,7 +7029,7 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { if (coff.isArchive() and coff.members.items.len > 0) { const last_member = coff.members.items[coff.members.items.len - 1]; // See .archive_member branch for reasoning - assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni.toOptional()); + assert(Node.known.file.last(&coff.mf).unwrap().? == last_member.content_ni); try coff.flushResized(last_member.content_ni); } }, @@ -7717,30 +7721,31 @@ pub fn printNode( if (mf_node.flags.has_content) " has_content" else "", }); } - var leaf = true; - var child_it = ni.children(&coff.mf); - while (child_it.next()) |child_ni| { - leaf = false; - try coff.printNode(tid, w, child_ni, indent + 1); - } - if (leaf) { - const file_loc = ni.fileLocation(&coff.mf, false); - if (file_loc.size == 0) return; - var address = file_loc.offset; - const line_len = 0x10; - var line_it = std.mem.window( - u8, - coff.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)], - line_len, - line_len, - ); - while (line_it.next()) |line_bytes| : (address += line_len) { - try w.splatByteAll(' ', indent + 1); - try w.print("{x:0>8} ", .{address}); - for (line_bytes) |byte| try w.print("{x:0>2} ", .{byte}); - try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1); - for (line_bytes) |byte| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.'); - try w.writeByte('\n'); + if (ni.first(&coff.mf).unwrap()) |first_ni| { + // non-leaf, just print children + var child_ni = first_ni; + while (true) { + try coff.printNode(tid, w, child_ni, indent + 1); + child_ni = child_ni.next(&coff.mf).unwrap() orelse break; } + return; + } + const file_loc = ni.fileLocation(&coff.mf, false); + if (file_loc.size == 0) return; + var address = file_loc.offset; + const line_len = 0x10; + var line_it = std.mem.window( + u8, + coff.mf.memory_map.memory[@intCast(file_loc.offset)..][0..@intCast(file_loc.size)], + line_len, + line_len, + ); + while (line_it.next()) |line_bytes| : (address += line_len) { + try w.splatByteAll(' ', indent + 1); + try w.print("{x:0>8} ", .{address}); + for (line_bytes) |byte| try w.print("{x:0>2} ", .{byte}); + try w.splatByteAll(' ', 3 * (line_len - line_bytes.len) + 1); + for (line_bytes) |byte| try w.writeByte(if (std.ascii.isPrint(byte)) byte else '.'); + try w.writeByte('\n'); } } diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 0acc9a6fe86b17f887eb41a55f35ea199bb1dca7..7f5cbd60fdb44f392a195cf07ad96ff7fcd402f7 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -7902,8 +7902,10 @@ fn flushElfOffset(elf: *Elf, ni: MappedFile.Node.Index) void { } }, } - var child_it = ni.children(&elf.mf); - while (child_it.next()) |child_ni| elf.flushElfOffset(child_ni); + var child_oni = ni.first(&elf.mf); + while (child_oni.unwrap()) |child_ni| : (child_oni = child_ni.next(&elf.mf)) { + elf.flushElfOffset(child_ni); + } }, .section => |shndx| switch (elf.shdrPtr(shndx)) { inline else => |shdr| elf.targetStore(&shdr.offset, @intCast(elf_offset)), @@ -8203,9 +8205,10 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo _, const size = ni.location(&elf.mf).resolve(&elf.mf); switch (elf.getNode(ni)) { .archive => { - var child_it = ni.reverseChildren(&elf.mf); - if (child_it.next()) |last_ni| { - if (child_it.next()) |prev_ni| if (prev_ni.hasNextMoved(&elf.mf)) return; + if (ni.last(&elf.mf).unwrap()) |last_ni| { + if (last_ni.prev(&elf.mf).unwrap()) |prev_ni| { + if (prev_ni.hasNextMoved(&elf.mf)) return; + } const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf); _ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{ size - offset, @@ -8894,13 +8897,15 @@ pub fn printNode( if (mf_node.flags.has_content) " has_content" else "", }); } - var leaf = true; - var child_it = ni.children(&elf.mf); - while (child_it.next()) |child_ni| { - leaf = false; - try elf.printNode(tid, w, child_ni, indent + 1); + if (ni.first(&elf.mf).unwrap()) |first_ni| { + // non-leaf, just print children + var child_ni = first_ni; + while (true) { + try elf.printNode(tid, w, child_ni, indent + 1); + child_ni = child_ni.next(&elf.mf).unwrap() orelse break; + } + return; } - if (!leaf) return; const file_loc = ni.fileLocation(&elf.mf, false); var address = file_loc.offset; if (file_loc.size == 0) { diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index 40af66dbf8c53093c492a79d69064879175cc4ef..a790ecbd3b303fbe97ea860481e41913d2aa3806 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -486,24 +486,6 @@ pub const Node = extern struct { return ni.get(mf).prev; } - pub fn ChildIterator(comptime direction: enum { prev, next }) type { - return struct { - mf: *const MappedFile, - ni: Node.Index.Optional, - pub fn next(it: *@This()) ?Node.Index { - const ni = it.ni.unwrap() orelse return null; - it.ni = @field(ni.get(it.mf), @tagName(direction)); - return ni; - } - }; - } - pub fn children(ni: Node.Index, mf: *const MappedFile) ChildIterator(.next) { - return .{ .mf = mf, .ni = ni.get(mf).first }; - } - pub fn reverseChildren(ni: Node.Index, mf: *const MappedFile) ChildIterator(.prev) { - return .{ .mf = mf, .ni = ni.get(mf).last }; - } - pub fn childrenMoved(ni: Node.Index, gpa: Allocator, mf: *MappedFile) Allocator.Error!void { var child_oni = ni.get(mf).last; while (child_oni.unwrap()) |child_ni| { -- 2.54.0 From dce4edf4d03c1e2b7b4110ea51c09d1a14db2982 Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Sun, 23 Aug 2026 21:38:28 +0100 Subject: [PATCH 5/6] Elf2: slightly simplify archive handling Eliminate the need to call this `ensureElfNodeSize` function all the time, with the result of failing to do so being introduction of a rare but possible miscompilation bug. Instead, add one of the new "footer" nodes to keep space available at the end of the `.elf` node. --- src/link/Elf2.zig | 53 ++++++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 7f5cbd60fdb44f392a195cf07ad96ff7fcd402f7..a6987c98db982537ea3f32da79d72d0de1f9fc18 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -203,6 +203,8 @@ const Node = union(enum) { archive, /// This includes the archive magic and long file member. archive_header, + /// This is a footer of the `.elf` node, and contains the next archive entry's file header. + archive_elf_footer, elf, ehdr, shdr, @@ -2952,6 +2954,7 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId { const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -3599,8 +3602,8 @@ fn initHeaders( }, phnum }; }; - const expected_nodes_len = @as(usize, if (is_archive) 2 else 0) + // .archive, .archive_header - 3 + // `.file`, `.ehdr`, and `.shdr` nodes + const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer + 3 + // `.elf`, `.ehdr`, and `.shdr` nodes (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node @@ -3643,6 +3646,12 @@ fn initHeaders( .enable_next_moved = true, }); elf.nodes.appendAssumeCapacity(.elf); + + _ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{ + .alignment = .@"2", + .size = @sizeOf(std.elf.ar_hdr), + }); + elf.nodes.appendAssumeCapacity(.archive_elf_footer); } else { elf.ni.elf = .root; elf.nodes.appendAssumeCapacity(.elf); @@ -3729,6 +3738,14 @@ fn initHeaders( } elf.phdrs.items[phndx.gnu_stack] = .none; + } else { + elf.ni.rodata = elf.ni.elf; + elf.ni.text = elf.ni.elf; + elf.ni.data = elf.ni.elf; + elf.ni.data_rel_ro = elf.ni.elf; + if (comp.config.any_non_single_threaded) { + elf.ni.tls = .wrap(elf.ni.elf); + } } switch (class) { @@ -4522,8 +4539,6 @@ fn initHeaders( break :str try elf.string(.dynstr, slice); }, }; - - try elf.ensureElfNodeSize(); } pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void { @@ -4558,6 +4573,7 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { return switch (elf.getNode(ni)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -4578,6 +4594,7 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { return switch (elf.getNode(ni)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -4596,7 +4613,7 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { } fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { - .archive, .archive_header => unreachable, + .archive, .archive_header, .archive_elf_footer => unreachable, .elf => return 0, .ehdr, .shdr => unreachable, .segment => |phndx| switch (elf.phdrSlice()) { @@ -4622,6 +4639,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { const symbol_relocs: *SymbolReloc.Index, const got_relocs: ?*GotReloc.Index = switch (elf.getNode(ni)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -6255,8 +6273,6 @@ fn prelinkInner(elf: *Elf) Error!void { }; elf.input_pending_index += 1; } - - try elf.ensureElfNodeSize(); } fn prepareDynamic(elf: *Elf) Error!void { @@ -7056,6 +7072,7 @@ fn addGotRelocAssumeCapacity( switch (elf.getNode(node)) { .archive, .archive_header, + .archive_elf_footer, .elf, .ehdr, .shdr, @@ -7485,7 +7502,6 @@ fn flushInner( try elf.prepareDynamic(); - try elf.ensureElfNodeSize(); while (try elf.idle(tid)) {} // We've done the final `idle` loop, so everything is at its final place in the file. We have a @@ -7738,8 +7754,6 @@ fn genPending(elf: *Elf, pt: Zcu.PerThread) Error!void { }; break; } - - try elf.ensureElfNodeSize(); } fn genUav( @@ -7922,7 +7936,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void switch (elf.getNode(ni)) { .archive, .archive_header => unreachable, - .elf => {}, + .archive_elf_footer, .elf => {}, .ehdr, .shdr => elf.flushElfOffset(ni), .segment => |phndx| { elf.flushElfOffset(ni); @@ -8216,7 +8230,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo } }, .archive_header, .elf => {}, - .ehdr => unreachable, + .ehdr, .archive_elf_footer => unreachable, .shdr => {}, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| { @@ -8301,6 +8315,7 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! switch (elf.getNode(ni)) { .archive, + .archive_elf_footer, .ehdr, .shdr, .segment, @@ -8746,8 +8761,6 @@ fn updateExportInner( .uav => |uav| (try elf.uavMapIndex(uav, .none)).symbol(elf), }; - try elf.ensureElfNodeSize(); - // Initialize the global symbol with the same values that the local one currently has. If the // NAV/UAV is updated, then `updateNavInner` or `genUav` will update the global symbol sizes, // and `flushMoved` will update their values. @@ -8970,18 +8983,6 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error } } -/// Must be called deterministically after any call to `MappedFile.Node.Index.resize` -/// (of `elf.ni.elf` or one of its children) before any possible calls to `idle`. -fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void { - if (elf.ni.elf == .root) return; - var child_it = elf.ni.elf.reverseChildren(&elf.mf); - const last_end = if (child_it.next()) |last_ni| last_end: { - const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf); - break :last_end last_offset + last_size; - } else 0; - try elf.ni.elf.ensureMinimumSize(&elf.mf, elf.base.comp.gpa, last_end + @sizeOf(std.elf.ar_hdr)); -} - /// If `sym` has a PLT entry, returns the address of that entry (specifically, the address which a /// branch to the PLT should target). If `sym` does not have a PLT entry, returns `null`. fn pltEntryTargetAddr(elf: *Elf, sym: Symbol.Id) ?u64 { -- 2.54.0 From edf18aa383cc5553a51e57954cf7e076c4b745db Mon Sep 17 00:00:00 2001 From: Matthew Lugg Date: Mon, 24 Aug 2026 08:48:42 +0100 Subject: [PATCH 6/6] Elf2: do not assume nodes are initialized to zero This is not a guarantee of the `MappedFile` interface, because guaranteeing it can be inefficient. Right now it will usually be the case, but that may not hold in the future. --- src/link/Elf2.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index a6987c98db982537ea3f32da79d72d0de1f9fc18..8ef617fad2e1cc2d4c7a67412a714502d4e8e2f6 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -4249,8 +4249,8 @@ fn initHeaders( if (elf.targetEndian() != std.lang.Endian.native) { std.mem.byteSwapAllFields(info.Header(), header); } - // The initial bucket and chain values are all 0, but `MappedFile` initialized - // the node with zeroes anyway, so no need to memset. + // The initial bucket and chain values are all 0. + @memset(hash_slice[@sizeOf(info.Header())..], 0); }, } -- 2.54.0