diff --git a/src/InternPool.zig b/src/InternPool.zig index 96af077d99935e56e6290ca47b544b6e6bd076fc..94375c3f0ca7430d890588979d9409415c2054a6 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -5987,17 +5987,6 @@ pub const Alignment = enum(u6) { return n + 1; } - pub fn toStdMem(a: Alignment) std.mem.Alignment { - assert(a != .none); - return @fromBackingInt(@intCast(@backingInt(a))); - } - - pub fn fromStdMem(a: std.mem.Alignment) Alignment { - const r: Alignment = @fromBackingInt(@intCast(@backingInt(a))); - assert(r != .none); - return r; - } - pub fn toLlvm(a: Alignment) std.zig.llvm.Builder.Alignment { return @fromBackingInt(@intCast(@backingInt(a))); } diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 7303c030011bd8935595353982ca2458b93ecae8..f83074f558b19e84d803d17251a6d4014abdd2b2 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -21,6 +21,7 @@ const Zcu = @import("../Zcu.zig"); const ModuleDefinition = @import("../libs/mingw/def.zig").ModuleDefinition; const implib = @import("../libs/mingw/implib.zig"); const Path = std.Build.Cache.Path; +const Alignment = MappedFile.Alignment; base: link.File, options: link.File.OpenOptions, @@ -602,7 +603,7 @@ pub const Member = struct { }; pub const LongNamesTable = struct { - ni: MappedFile.Node.Index = .none, + ni: MappedFile.Node.Index.Optional = .none, entries: std.array_hash_map.Auto(void, Entry), pub const Entry = struct { @@ -832,7 +833,7 @@ pub const String = enum(u32) { pub const Section = struct { si: Symbol.Index, - relocation_table_ni: MappedFile.Node.Index, + relocation_table_ni: MappedFile.Node.Index.Optional, pub const RelocationIndex = enum(u16) { none, @@ -855,7 +856,7 @@ pub const Section = struct { sn: Symbol.SectionNumber, ) ?*align(2) std.coff.Relocation { if (sri == .none) return null; - const table_slice = sn.section(coff).relocation_table_ni.slice(&coff.mf); + const table_slice = sn.section(coff).relocation_table_ni.unwrap().?.slice(&coff.mf); return @ptrCast(@alignCast(&table_slice[@as(u32, sri.unwrap().?) * std.coff.Relocation.sizeOf()])); } }; @@ -891,7 +892,7 @@ const SpecialSymbol = enum { }; pub const Symbol = struct { - ni: MappedFile.Node.Index, + ni: MappedFile.Node.Index.Optional, rva: u32, value: std.meta.BareUnion(Symbol.Value), extra: std.meta.BareUnion(Symbol.Extra), @@ -986,7 +987,7 @@ pub const Symbol = struct { pub fn nodeOffset(sym: *const Symbol, coff: *Coff) u32 { return switch (sym.flags.value_tag) { .node_offset => offset: { - assert(switch (coff.getNode(sym.ni)) { + assert(switch (coff.getNode(sym.ni.unwrap().?)) { // Separate nodes are not created for these entries per-symbol .input_section, .import_address_table => true, else => false, @@ -1052,9 +1053,7 @@ pub const Symbol = struct { } pub fn node(si: Symbol.Index, coff: *Coff) MappedFile.Node.Index { - const ni = si.get(coff).ni; - assert(ni != .none); - return ni; + return si.get(coff).ni.unwrap().?; } pub fn sti(si: Symbol.Index, coff: *Coff) SymbolTable.Index { @@ -1075,7 +1074,7 @@ pub const Symbol = struct { pub fn flushMoved(si: Symbol.Index, coff: *Coff) !void { const sym = si.get(coff); - sym.rva = coff.computeNodeRva(sym.ni) + sym.nodeOffset(coff); + sym.rva = coff.computeNodeRva(sym.ni.unwrap().?) + sym.nodeOffset(coff); try si.applyLocationRelocs(coff); try si.applyTargetRelocs(coff, .none); @@ -1199,12 +1198,11 @@ pub const Reloc = extern struct { pub fn apply(reloc: *Reloc, coff: *Coff) !void { const loc_sym = reloc.loc.get(coff); - switch (loc_sym.ni) { - .none => return, - else => |ni| if (ni.hasMoved(&coff.mf)) return, - } - const loc_slice = loc_sym.ni.slice(&coff.mf)[@intCast(reloc.offset)..]; + const loc_sym_ni = loc_sym.ni.unwrap() orelse return; + if (loc_sym_ni.hasMoved(&coff.mf)) return; + + const loc_slice = loc_sym_ni.slice(&coff.mf)[@intCast(reloc.offset)..]; const target_endian = coff.targetEndian(); const target_machine = coff.targetLoad(&coff.headerPtr().machine); @@ -1331,9 +1329,12 @@ pub const Reloc = extern struct { } const target_sym = reloc.target.get(coff); - const is_abs = switch (target_sym.ni) { - .none => if (target_sym.section_number == .ABSOLUTE) true else return, - else => |ni| if (ni.hasMoved(&coff.mf)) return else false, + const is_abs = if (target_sym.ni.unwrap()) |ni| is_abs: { + if (ni.hasMoved(&coff.mf)) return; + break :is_abs false; + } else is_abs: { + if (target_sym.section_number != .ABSOLUTE) return; + break :is_abs true; }; const target_rva = target_sym.rva +% @as(u64, @bitCast(reloc.addend)); @@ -1573,7 +1574,7 @@ fn create( 33...64 => .@"PE32+", else => return error.UnsupportedCOFFArchitecture, }; - const section_align: std.mem.Alignment = switch (machine) { + const section_align: Alignment = switch (machine) { .AMD64, .I386 => @fromBackingInt(@intCast(12)), .SH3, .SH3DSP, .SH4, .SH5 => @fromBackingInt(@intCast(12)), .MIPS16, .MIPSFPU, .MIPSFPU16, .WCEMIPSV2 => @fromBackingInt(@intCast(12)), @@ -1617,22 +1618,22 @@ fn create( .entries = .empty, }, .import_table = .{ - .ni = .none, + .ni = undefined, .entries = .empty, .iat_symbol_indices = .empty, }, .export_table = .{ - .ni = .none, - .export_directory_table_ni = .none, + .ni = undefined, + .export_directory_table_ni = undefined, .export_address_table_si = .null, - .name_pointer_table_ni = .none, - .ordinal_table_ni = .none, - .name_table_ni = .none, + .name_pointer_table_ni = undefined, + .ordinal_table_ni = undefined, + .name_table_ni = undefined, .entries = .empty, }, .symbol_table = .{ - .ni = .none, - .strings_ni = .none, + .ni = undefined, + .strings_ni = undefined, .strings = .empty, .symbols = .empty, .pending_symbol_index = 0, @@ -1794,13 +1795,13 @@ fn initHeaders( minor_subsystem_version: u16, magic: std.coff.OptionalHeader.Magic, subsystem: std.coff.Subsystem, - section_align: std.mem.Alignment, + section_align: Alignment, file_name: []const u8, ) !void { const comp = coff.base.comp; const gpa = comp.gpa; const target_endian = coff.targetEndian(); - const file_align: std.mem.Alignment = comptime .fromByteUnits(default_file_alignment); + const file_align: Alignment = comptime .fromByteUnits(default_file_alignment); const is_image = coff.isImage(); const is_archive = coff.isArchive(); const target = &comp.root_mod.resolved_target.result; @@ -2191,7 +2192,7 @@ fn initHeaders( coff.export_table.export_address_table_si = coff.addSymbolAssumeCapacity(); const export_address_table_sym = coff.export_table.export_address_table_si.get(coff); - export_address_table_sym.ni = export_address_table_ni; + export_address_table_sym.ni = .wrap(export_address_table_ni); assert(export_address_table_sym.loc_relocs == .none); export_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); export_address_table_sym.section_number = @@ -2260,7 +2261,7 @@ pub fn initBuiltins(coff: *Coff) !void { if (coff.isImage()) { const si = try coff.globalSymbol(.{ .name = "__ImageBase", .type = .data }); const sym = si.get(coff); - sym.ni = Node.known.header; + sym.ni = .wrap(Node.known.header); } defer coff.flushSectionMerges() catch unreachable; @@ -2302,14 +2303,14 @@ pub fn initBuiltins(coff: *Coff) !void { const list_len_si = try coff.globalSymbol(.{ .name = list.global, .type = .data }); const list_len_sym = list_len_si.get(coff); list_len_sym.setExtra(.{ .size = addr_info.size }); - list_len_sym.ni = try coff.mf.addFirstChildNode(gpa, start_sym.ni, .{ + list_len_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, start_sym.ni.unwrap().?, .{ .size = addr_info.size, .fixed = true, - }); + })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_len_si }); list_len_sym.section_number = start_sym.section_number; - const start_slice = list_len_sym.ni.slice(&coff.mf); + const start_slice = list_len_sym.ni.unwrap().?.slice(&coff.mf); switch (addr_info.magic) { _ => unreachable, inline .PE32, .@"PE32+" => |t| { @@ -2324,14 +2325,14 @@ pub fn initBuiltins(coff: *Coff) !void { const list_end_si = coff.addSymbolAssumeCapacity(); const list_end_sym = list_end_si.get(coff); list_end_sym.setExtra(.{ .size = addr_info.size }); - list_end_sym.ni = try coff.mf.addFirstChildNode(gpa, end_sym.ni, .{ + list_end_sym.ni = .wrap(try coff.mf.addFirstChildNode(gpa, end_sym.ni.unwrap().?, .{ .size = addr_info.size, .fixed = true, - }); + })); coff.nodes.appendAssumeCapacity(.{ .builtin = list_end_si }); list_end_sym.section_number = start_sym.section_number; - @memset(list_end_sym.ni.slice(&coff.mf), 0); + @memset(list_end_sym.ni.unwrap().?.slice(&coff.mf), 0); try list_len_si.flushMoved(coff); try list_end_si.flushMoved(coff); @@ -2387,7 +2388,7 @@ fn getNode(coff: *const Coff, ni: MappedFile.Node.Index) Node { } fn computeNodeRva(coff: *Coff, ni: MappedFile.Node.Index) u32 { const parent_rva = parent_rva: { - const parent_si = switch (coff.getNode(ni.parent(&coff.mf))) { + const parent_si = switch (coff.getNode(ni.parent(&coff.mf).unwrap().?)) { .file, .header, .signature, @@ -2452,11 +2453,11 @@ fn computeSymbolSectionOffset( relative_to: enum { image, pseudo }, ) u32 { var section_offset: u32 = sym.nodeOffset(coff); - var parent_ni = sym.ni; + var parent_ni = sym.ni.unwrap().?; while (true) { const offset, _ = parent_ni.location(&coff.mf).resolve(&coff.mf); section_offset += @intCast(offset); - parent_ni = parent_ni.parent(&coff.mf); + parent_ni = parent_ni.parent(&coff.mf).unwrap().?; switch (coff.getNode(parent_ni)) { else => unreachable, .image_section => break, @@ -2475,7 +2476,7 @@ pub inline fn targetEndian(_: *const Coff) std.lang.Endian { fn targetAddrInfo(coff: *Coff) struct { size: u8, - alignment: std.mem.Alignment, + alignment: Alignment, magic: std.coff.OptionalHeader.Magic, } { const magic = coff.targetLoad(&coff.optionalHeaderStandardPtr().magic); @@ -2875,9 +2876,9 @@ fn navSection( switch (nav_resolved.@"linksection") { .none => coff.mf.flags.block_size, else => switch (nav_resolved.@"align") { - .none => Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu), - else => |alignment| alignment, - }.toStdMem(), + .none => .fromIp(Type.fromInterned(ip.typeOf(nav_resolved.value)).abiAlignment(zcu)), + else => |a| .fromIp(a), + }, }, attributes, )).symbol(coff); @@ -3151,7 +3152,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { else .NULL, }; - } else blk: switch (coff.getNode(sym.ni)) { + } else blk: switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => .{ try coff.getOrPutSymbolName(&sym.section_number.header(coff).name, null), 1, @@ -3192,7 +3193,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { }; }, else => { - log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni)), si }); + log.err("TODO implement symbol table init for {s} ({d})", .{ @tagName(coff.getNode(sym.ni.unwrap().?)), si }); unreachable; }, }; @@ -3255,13 +3256,13 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { std.mem.byteSwapAllFieldsAligned(std.coff.WeakExternalDefinition, .@"2", aux_ptr); break :aux_init; - } else switch (coff.getNode(sym.ni)) { + } else switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => |sec_si| { assert(si == sec_si); const header = sym.section_number.header(coff); const aux_ptr = coff.symbolTableSectionAuxEntryPtr(sti.*).?; aux_ptr.* = .{ - .length = @intCast(sym.ni.location(&coff.mf).resolve(&coff.mf)[1]), + .length = @intCast(sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[1]), .number_of_relocations = header.number_of_relocations, .number_of_linenumbers = header.number_of_linenumbers, .checksum = 0, @@ -3288,7 +3289,7 @@ fn flushSymbolTableEntry(coff: *Coff, index: u32, pt: Zcu.PerThread) !void { .ABSOLUTE, .DEBUG, => unreachable, - else => switch (coff.getNode(sym.ni)) { + else => switch (coff.getNode(sym.ni.unwrap().?)) { .image_section => 0, else => coff.computeSymbolSectionOffset(sym, .image), }, @@ -3397,7 +3398,7 @@ fn addSection(coff: *Coff, name: String, flags: std.coff.SectionHeader.Flags) !S { const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = rva; sym.section_number = @fromBackingInt(@intCast(section_table_len)); } @@ -3481,7 +3482,7 @@ const ObjectSectionAttributes = packed struct { fn pseudoSectionMapIndex( coff: *Coff, name: String, - alignment: std.mem.Alignment, + alignment: Alignment, attributes: ObjectSectionAttributes, ) !Node.PseudoSectionMapIndex { const gpa = coff.base.comp.gpa; @@ -3510,7 +3511,7 @@ fn pseudoSectionMapIndex( const si = coff.addSymbolAssumeCapacity(); pseudo_section_gop.value_ptr.* = si; const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = coff.computeNodeRva(ni); sym.section_number = parent.get(coff).section_number; assert(sym.loc_relocs == .none); @@ -3543,7 +3544,7 @@ fn objectSectionParentName(coff: *Coff, name: []const u8) []const u8 { fn objectSectionMapIndex( coff: *Coff, name: String, - alignment: std.mem.Alignment, + alignment: Alignment, attributes: ObjectSectionAttributes, ) !Node.ObjectSectionMapIndex { const gpa = coff.base.comp.gpa; @@ -3565,7 +3566,7 @@ fn objectSectionMapIndex( try coff.nodes.ensureUnusedCapacity(gpa, 1); try coff.symbols.ensureUnusedCapacity(gpa, 1); const parent_ni = parent.node(coff); - var prev_ni: MappedFile.Node.Index = .none; + var prev_oni: MappedFile.Node.Index.Optional = .none; var next_it = parent_ni.children(&coff.mf); while (next_it.next()) |next_ni| switch (std.mem.order( u8, @@ -3574,22 +3575,19 @@ fn objectSectionMapIndex( )) { .lt => break, .eq => unreachable, - .gt => prev_ni = next_ni, - }; - const ni = switch (prev_ni) { - .none => try coff.mf.addFirstChildNode(gpa, parent_ni, .{ - .alignment = alignment, - .fixed = true, - }), - else => try coff.mf.addNodeAfter(gpa, prev_ni, .{ - .alignment = alignment, - .fixed = true, - }), + .gt => prev_oni = .wrap(next_ni), }; + const ni = if (prev_oni.unwrap()) |prev_ni| try coff.mf.addNodeAfter(gpa, prev_ni, .{ + .alignment = alignment, + .fixed = true, + }) else try coff.mf.addFirstChildNode(gpa, parent_ni, .{ + .alignment = alignment, + .fixed = true, + }); const si = coff.addSymbolAssumeCapacity(); object_section_gop.value_ptr.* = si; const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.rva = coff.computeNodeRva(ni); sym.section_number = parent.get(coff).section_number; assert(sym.loc_relocs == .none); @@ -3598,17 +3596,17 @@ fn objectSectionMapIndex( break :sym sym; } else object_section_gop.value_ptr.get(coff); - const parent_ni = sym.ni.parent(&coff.mf); + const parent_ni = sym.ni.unwrap().?.parent(&coff.mf).unwrap().?; const parent_alignment = parent_ni.alignment(&coff.mf); if (alignment.compare(.gt, parent_alignment)) { log.debug("realignParent({s}, {d}) {d}->{d}", .{ name.toSlice(coff), parent_ni, parent_alignment, alignment }); try parent_ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); } - const old_alignment = sym.ni.alignment(&coff.mf); + const old_alignment = sym.ni.unwrap().?.alignment(&coff.mf); if (alignment.compare(.gt, old_alignment)) { log.debug("realignObject({s}) {d}->{d}", .{ name.toSlice(coff), old_alignment, alignment }); - try sym.ni.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); + try sym.ni.unwrap().?.realign(&coff.mf, gpa, alignment, .{ .try_backwards = true }); } try coff.verifyParentSectionAttributes( @@ -3764,8 +3762,10 @@ fn addRelocAssumeCapacity( if (coff.symbolTableSectionAuxEntryPtr(loc_sn.symbol(coff).sti(coff))) |aux_ptr| coff.targetStore(&aux_ptr.number_of_relocations, new_num_relocations); - if (section.relocation_table_ni == .none) { - section.relocation_table_ni = try coff.mf.addLastChildNode( + if (section.relocation_table_ni.unwrap()) |relocation_table_ni| { + try relocation_table_ni.resize(&coff.mf, gpa, new_size); + } else { + section.relocation_table_ni = .wrap(try coff.mf.addLastChildNode( gpa, coff.sectionParent(), .{ @@ -3774,10 +3774,8 @@ fn addRelocAssumeCapacity( .moved = true, .resized = true, }, - ); + )); coff.nodes.appendAssumeCapacity(.{ .relocation_table = loc_sn }); - } else { - try section.relocation_table_ni.resize(&coff.mf, gpa, new_size); } // TODO: These need to allocate from a free list, once deleting relocs from the table is supported @@ -4581,7 +4579,7 @@ fn loadObject( }, .SAME_SIZE => { // TODO: Verify that this node isn't resized after creation - _, const size = si.get(coff).ni.location(&coff.mf).resolve(&coff.mf); + _, const size = si.get(coff).ni.unwrap().?.location(&coff.mf).resolve(&coff.mf); if (size == section.header.size_of_raw_data) { symbol.si = si; break :comdat .skip; @@ -4598,9 +4596,9 @@ fn loadObject( }, .EXACT_MATCH => { const sym = si.get(coff); - const existing_crc = switch (coff.getNode(sym.ni)) { + const existing_crc = switch (coff.getNode(sym.ni.unwrap().?)) { .input_section => |isi| isi.inputSection(coff).crc, - else => Crc32.hash(sym.ni.sliceConst(&coff.mf)), + else => Crc32.hash(sym.ni.unwrap().?.sliceConst(&coff.mf)), }; if (existing_crc == section.comdat_crc) { @@ -4666,7 +4664,7 @@ fn loadObject( section.parent_si = (try coff.objectSectionMapIndex( section.name, - section.header.flags.ALIGN.alignment() orelse .@"1", + .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), .fromFlags(section.header.flags), )).symbol(coff); } @@ -4681,7 +4679,7 @@ fn loadObject( const ni = try coff.mf.addLastChildNode(gpa, section.parent_si.node(coff), .{ .size = section.header.size_of_raw_data, - .alignment = section.header.flags.ALIGN.alignment() orelse .@"1", + .alignment = .fromByteUnits(section.header.flags.ALIGN.toByteUnits() orelse 1), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .input_section = @fromBackingInt(@intCast(coff.input_sections.items.len)) }); @@ -4691,7 +4689,7 @@ fn loadObject( pending_symbols.values()[psi].si = section.si; const sym = section.si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = section.parent_si.get(coff).section_number; coff.input_sections.addOneAssumeCapacity().* = .{ @@ -4852,7 +4850,7 @@ fn loadObject( } if (section.comdat_psi.unwrap() == @as(u32, @intCast(i))) - coff.getNode(section.si.get(coff).ni).input_section.inputSection(coff).comdat_si = symbol.si; + coff.getNode(section.si.get(coff).ni.unwrap().?).input_section.inputSection(coff).comdat_si = symbol.si; } if (symbol.weak_external_psi.unwrap()) |weak_external_i| { @@ -4967,14 +4965,14 @@ fn loadObject( const section = §ions[symbol.section_number.toIndex()]; include_section = section.comdat_result == .include; if (include_section) { - const isi = coff.getNode(section.si.get(coff).ni).input_section; + const isi = coff.getNode(section.si.get(coff).ni.unwrap().?).input_section; isi.inputSection(coff).first_li = @fromBackingInt(@intCast(coff.input_symbols.items.len)); } } } if (include_section) { - assert(coff.getNode(symbol.si.get(coff).ni) == .input_section); + assert(coff.getNode(symbol.si.get(coff).ni.unwrap().?) == .input_section); symbol.si.get(coff).setExtra(.{ .isli = @fromBackingInt(@intCast(coff.input_symbols.items.len)) }); coff.input_symbols.addOneAssumeCapacity().* = .{ .si = symbol.si, @@ -5002,7 +5000,7 @@ fn failMultipleDefinitions( var err = try coff.base.comp.link_diags.addErrorWithNotes(num_notes); try err.addMsg("multiple definitions of '{s}'", .{name.toSlice(coff)}); - switch (coff.getNode(existing_si.get(coff).ni)) { + switch (coff.getNode(existing_si.get(coff).ni.unwrap().?)) { .input_section => |isi| { const other_ioi = isi.input(coff); err.addNote("first seen in input '{f}{f}'", .{ @@ -5474,12 +5472,12 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde try coff.nodes.ensureUnusedCapacity(gpa, 1); if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ - .alignment = zcu.navAlignment(nav_index).toStdMem(), + .alignment = .fromIp(zcu.navAlignment(nav_index)), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .nav = nmi }); const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -5490,7 +5488,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; { @@ -5515,7 +5513,7 @@ fn updateNavInner(coff: *Coff, pt: Zcu.PerThread, nav_index: InternPool.Nav.Inde try ni.resize(&coff.mf, gpa, si.get(coff).extra.size); var parent_ni = ni; while (true) { - parent_ni = parent_ni.parent(&coff.mf); + parent_ni = parent_ni.parent(&coff.mf).unwrap().?; switch (coff.getNode(parent_ni)) { else => unreachable, .image_section, .pseudo_section => break, @@ -5542,10 +5540,11 @@ pub fn lowerUav( try coff.pending_uavs.ensureUnusedCapacity(gpa, 1); const umi = try coff.uavMapIndex(uav_val); const si = umi.symbol(coff); - if (switch (si.get(coff).ni) { - .none => true, - else => |ni| uav_align.toStdMem().order(ni.alignment(&coff.mf)).compare(.gt), - }) { + const need_update: bool = update: { + const existing_ni = si.get(coff).ni.unwrap() orelse break :update true; + break :update Alignment.compare(.fromIp(uav_align), .gt, existing_ni.alignment(&coff.mf)); + }; + if (need_update) { const gop = coff.pending_uavs.getOrPutAssumeCapacity(umi); if (gop.found_existing) { gop.value_ptr.alignment = gop.value_ptr.alignment.max(uav_align); @@ -5603,16 +5602,16 @@ fn updateFuncInner( .debug, .safe, .fast, - => target_util.defaultFunctionAlignment(target), - .small => target_util.minFunctionAlignment(target), + => .fromIp(target_util.defaultFunctionAlignment(target)), + .small => .fromIp(target_util.minFunctionAlignment(target)), }, - else => |a| a.maxStrict(target_util.minFunctionAlignment(target)), - }.toStdMem(), + else => |a| .fromIp(a.maxStrict(target_util.minFunctionAlignment(target))), + }, .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .nav = nmi }); const sym = si.get(coff); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -5622,7 +5621,7 @@ fn updateFuncInner( sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var nw: MappedFile.Node.Writer = undefined; @@ -5662,7 +5661,6 @@ fn flushImplib( implib_file: []const u8, ) !void { // Emitting implibs is only valid for images - assert(coff.export_table.ni != .none); const comp = coff.base.comp; const gpa = comp.gpa; @@ -5797,7 +5795,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void { const loc_sym = loc_si.get(coff); // TODO: Make this a helper for anything that needs to report "referenced by" notes - switch (coff.getNode(loc_sym.ni)) { + switch (coff.getNode(loc_sym.ni.unwrap().?)) { .data_directories => { const dir: std.coff.IMAGE.DIRECTORY_ENTRY = @fromBackingInt(@intCast(reloc.offset / @sizeOf(std.coff.ImageDataDirectory))); @@ -5808,7 +5806,7 @@ fn reportUndefs(coff: *Coff, tid: Zcu.PerThread.Id) !void { const other_ioi = isi.input(coff); if (loc_sym.gmi == .none) { const section = isi.inputSection(coff); - const section_name = coff.getNode(loc_sym.ni.parent(&coff.mf)) + const section_name = coff.getNode(loc_sym.ni.unwrap().?.parent(&coff.mf).unwrap().?) .object_section.name(coff).toSlice(coff); if (section.comdat_si != .null) { @@ -6055,8 +6053,8 @@ fn resolve(coff: *Coff, tid: Zcu.PerThread.Id) !bool { const sub_prog_node = coff.idleProgNode( tid, coff.symbol_prog_node, - if (sym.ni != .none) - coff.getNode(sym.ni) + if (sym.ni.unwrap()) |sym_ni| + coff.getNode(sym_ni) else .{ .import_thunk = sym.gmi }, ); @@ -6173,7 +6171,7 @@ fn idleProgNode( break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{ ioi.path(coff).fmtEscapeString(), fmtMemberNameString(ioi.memberName(coff)), - coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff), + coff.getNode(isi.symbol(coff).node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff), }) catch &name; }, .import_thunk => |gmi| gmi.name(coff).toSlice(coff), @@ -6214,16 +6212,21 @@ fn flushUav( if (!isImage(coff)) try coff.symbol_table.symbols.ensureUnusedCapacity(gpa, 1); const sym = si.get(coff); const ni = try coff.mf.addLastChildNode(gpa, sec_si.node(coff), .{ - .alignment = uav_align.toStdMem(), + .alignment = .fromIp(uav_align), .moved = true, }); coff.nodes.appendAssumeCapacity(.{ .uav = umi }); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => { - if (si.get(coff).ni.alignment(&coff.mf).order(uav_align.toStdMem()).compare(.gte)) + if (Alignment.compare( + si.get(coff).ni.unwrap().?.alignment(&coff.mf), + .gte, + .fromIp(uav_align), + )) { return; + } si.deleteLocationRelocs(coff); }, } @@ -6233,7 +6236,7 @@ fn flushUav( if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var nw: MappedFile.Node.Writer = undefined; @@ -6497,7 +6500,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { lib_name, ImportTable.Adapter{ .coff = coff }, ); - const import_hint_name_align: std.mem.Alignment = .@"2"; + const import_hint_name_align: Alignment = .@"2"; if (!gop.found_existing) { errdefer _ = coff.import_table.entries.pop(); try coff.import_table.ni.resize( @@ -6507,7 +6510,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { ); const import_hint_name_table_len = import_hint_name_align.forward(lib_name.len + ".dll".len + 1); - const idata_section_ni = coff.import_table.ni.parent(&coff.mf); + const idata_section_ni = coff.import_table.ni.parent(&coff.mf).unwrap().?; const import_lookup_table_ni = try coff.mf.addLastChildNode(gpa, idata_section_ni, .{ .size = addr_info.size * 2, .alignment = addr_info.alignment, @@ -6521,7 +6524,7 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { const import_address_table_si = coff.addSymbolAssumeCapacity(); { const import_address_table_sym = import_address_table_si.get(coff); - import_address_table_sym.ni = import_address_table_ni; + import_address_table_sym.ni = .wrap(import_address_table_ni); assert(import_address_table_sym.loc_relocs == .none); import_address_table_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); import_address_table_sym.section_number = @@ -6648,13 +6651,13 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); const target = &comp.root_mod.resolved_target.result; - const alignment = switch (comp.root_mod.optimize_mode) { + const alignment: Alignment = switch (comp.root_mod.optimize_mode) { .debug, .safe, .fast, - => target_util.defaultFunctionAlignment(target), - .small => target_util.minFunctionAlignment(target), - }.toStdMem(); + => .fromIp(target_util.defaultFunctionAlignment(target)), + .small => .fromIp(target_util.minFunctionAlignment(target)), + }; const parent_si = (try coff.pseudoSectionMapIndex( .@".thunks", alignment, @@ -6668,12 +6671,12 @@ fn flushGlobal(coff: *Coff, gmi: Node.GlobalMapIndex) !bool { else => |tag| @panic(@tagName(tag)), .AMD64 => { const init = [_]u8{ 0xff, 0x25, 0x00, 0x00, 0x00, 0x00 }; - const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni, .{ + const ni = try coff.mf.addLastChildNode(gpa, parent_sym.ni.unwrap().?, .{ .alignment = alignment, .size = init.len, }); @memcpy(ni.slice(&coff.mf)[0..init.len], &init); - sym.ni = ni; + sym.ni = .wrap(ni); sym.extra.size = init.len; try coff.addReloc( si, @@ -6736,7 +6739,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol { try coff.symbols.ensureUnusedCapacity(gpa, 1); const optional_hdr_si = coff.addSymbolAssumeCapacity(); const optional_hdr_sym = optional_hdr_si.get(coff); - optional_hdr_sym.ni = Node.known.optional_header; + optional_hdr_sym.ni = .wrap(Node.known.optional_header); assert(optional_hdr_sym.loc_relocs == .none); optional_hdr_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); @@ -6783,7 +6786,7 @@ fn flushSpecialSymbol(coff: *Coff, pending: SpecialSymbol) !SpecialSymbol { try coff.symbols.ensureUnusedCapacity(gpa, 1); const data_dir_si = coff.addSymbolAssumeCapacity(); const data_dir_sym = data_dir_si.get(coff); - data_dir_sym.ni = Node.known.data_directories; + data_dir_sym.ni = .wrap(Node.known.data_directories); assert(data_dir_sym.loc_relocs == .none); data_dir_sym.loc_relocs = @fromBackingInt(@intCast(coff.relocs.items.len)); @@ -6826,7 +6829,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { .code => .{ .lazy_code = @fromBackingInt(@intCast(lmr.index)) }, .const_data => .{ .lazy_const_data = @fromBackingInt(@intCast(lmr.index)) }, }); - sym.ni = ni; + sym.ni = .wrap(ni); sym.section_number = sec_si.get(coff).section_number; }, else => si.deleteLocationRelocs(coff), @@ -6836,7 +6839,7 @@ fn flushLazy(coff: *Coff, pt: Zcu.PerThread, lmr: Node.LazyMapRef) !void { if (!isImage(coff) and sym.target_relocs != .none) try coff.pendingSymbolTableEntry(si); - break :ni sym.ni; + break :ni sym.ni.unwrap().?; }; var required_alignment: InternPool.Alignment = .none; @@ -6914,7 +6917,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void { const flags = coff.targetLoad(&sym.section_number.header(coff).flags); if (!flags.CNT_UNINITIALIZED_DATA) { const file_offset = if (isArchive(coff)) - sym.ni.location(&coff.mf).resolve(&coff.mf)[0] + sym.ni.unwrap().?.location(&coff.mf).resolve(&coff.mf)[0] else ni.fileLocation(&coff.mf, false).offset; @@ -6927,7 +6930,7 @@ fn flushMoved(coff: *Coff, ni: MappedFile.Node.Index) !void { .input_section => |isi| { try isi.symbol(coff).flushMoved(coff); for (coff.input_symbols.items[@backingInt(isi.firstSymbol(coff))..]) |input_symbol| { - if (input_symbol.si.get(coff).ni != ni) break; + if (input_symbol.si.get(coff).ni != ni.toOptional()) break; try input_symbol.si.flushMoved(coff); } }, @@ -7062,7 +7065,7 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { if (coff.isArchive() and coff.members.items.len > 0) { const last_member = coff.members.items[coff.members.items.len - 1]; // See .archive_member branch for reasoning - assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni); + assert(Node.known.file.reverseChildren(&coff.mf).ni == last_member.content_ni.toOptional()); try coff.flushResized(last_member.content_ni); } }, @@ -7090,19 +7093,15 @@ fn flushResized(coff: *Coff, ni: MappedFile.Node.Index) !void { => unreachable, .archive_member => |mi| { const content_ni = mi.get(coff).content_ni; - const next_ni = content_ni.next(&coff.mf); const content_offset, _ = content_ni.location(&coff.mf).resolve(&coff.mf); - const next_offset = switch (next_ni) { - .none => offset: { - assert(content_ni.parent(&coff.mf) == Node.known.file); - // This must take into account the final file size. If there are trailing - // bytes, they will be expected to contain another valid member header - break :offset coff.mf.memory_map.memory.len; - }, - else => offset: { - assert(coff.getNode(next_ni) == .archive_member_header); - break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0]; - }, + const next_offset = if (content_ni.next(&coff.mf).unwrap()) |next_ni| offset: { + assert(coff.getNode(next_ni) == .archive_member_header); + break :offset next_ni.location(&coff.mf).resolve(&coff.mf)[0]; + } else offset: { + assert(content_ni.parent(&coff.mf) == Node.known.file.toOptional()); + // This must take into account the final file size. If there are trailing + // bytes, they will be expected to contain another valid member header + break :offset coff.mf.memory_map.memory.len; }; // Not inserting IMAGE_ARCHIVE_PAD `\n` byte here, because we are expanding to full size @@ -7356,7 +7355,7 @@ fn virtualSlide(coff: *Coff, start_section_index: usize, start_rva: u32) !void { const section_sym = section.si.get(coff); section_sym.rva = rva; coff.targetStore(&header.virtual_address, rva); - try section_sym.ni.childrenMoved(coff.base.comp.gpa, &coff.mf); + try section_sym.ni.unwrap().?.childrenMoved(coff.base.comp.gpa, &coff.mf); rva += coff.targetLoad(&header.virtual_size); } switch (coff.optionalHeaderPtr()) { @@ -7430,7 +7429,7 @@ fn updateExportInner( // TODO: add an errMsg if this conflicts with an existing symbol const export_si = try coff.globalSymbol(.{ .name = name }); const export_sym = export_si.get(coff); - export_sym.ni = exported_ni; + export_sym.ni = .wrap(exported_ni); export_sym.rva = exported_sym.rva; export_sym.section_number = exported_sym.section_number; if (@"export".opts.linkage == .weak and !coff.isImage()) { @@ -7599,14 +7598,13 @@ fn printSymbol( si: Symbol.Index, ) !void { const sym = si.get(coff); - const node = coff.getNode(sym.ni); - try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{t: <26} | {x:08} ", .{ + try w.print("{d:0>6}@{d:0>2} {x:08} {s} {s} {s} n{d:0>8}+{x:08}:{s: <26} | {x:08} ", .{ si, sym.section_number, if (sym.flags.extra_tag == .size) @as(u64, sym.extra.size) - else if (sym.ni != .none) - sym.ni.location(&coff.mf).resolve(&coff.mf)[1] + else if (sym.ni.unwrap()) |ni| + ni.location(&coff.mf).resolve(&coff.mf)[1] else 0, switch (sym.flags.value_tag) { @@ -7627,7 +7625,7 @@ fn printSymbol( }, sym.ni, if (sym.flags.value_tag == .node_offset) sym.value.node_offset else 0, - node, + if (sym.ni.unwrap()) |ni| @tagName(coff.getNode(ni)) else "", sym.rva, }); @@ -7635,7 +7633,7 @@ fn printSymbol( try w.print("G {f}\n", .{fmtGlobalName(coff, sym.gmi)}); } else { try w.writeAll("| "); - try coff.printNodeName(w, tid, node); + try coff.printNodeName(w, tid, coff.getNode(sym.ni.unwrap().?)); if (sym.flags.extra_tag == .isli) try w.print(" | {s}", .{sym.extra.isli.name(coff).toSlice(coff)}); try w.writeByte('\n'); @@ -7672,7 +7670,7 @@ fn printNodeName( try w.print("({f}{f}, {s}", .{ ioi.path(coff).fmtEscapeString(), fmtMemberNameString(ioi.memberName(coff)), - coff.getNode(is.si.node(coff).parent(&coff.mf)).object_section.name(coff).toSlice(coff), + coff.getNode(is.si.node(coff).parent(&coff.mf).unwrap().?).object_section.name(coff).toSlice(coff), }); if (is.comdat_si != .null) { const comdat_sym = is.comdat_si.get(coff); diff --git a/src/link/Elf2.zig b/src/link/Elf2.zig index 1400d0174815615d4f853ebb53ff2bc73dcd7915..7f23494f4fe45d74c749f6b65b255b50a20781c7 100644 --- a/src/link/Elf2.zig +++ b/src/link/Elf2.zig @@ -18,14 +18,16 @@ const tracy = @import("../tracy.zig"); const Type = @import("../Type.zig"); const Value = @import("../Value.zig"); const Zcu = @import("../Zcu.zig"); +const Alignment = MappedFile.Alignment; base: link.File, options: link.File.OpenOptions, mf: MappedFile, ni: Node.Known, nodes: std.MultiArrayList(Node), +/// Does not contain an item for `SHN_UNDEF`. shdrs: std.ArrayList(Section), -phdrs: std.ArrayList(MappedFile.Node.Index), +phdrs: std.ArrayList(MappedFile.Node.Index.Optional), shndx: struct { got: Section.Index, /// Always `.UNDEF` on some targets (e.g. SPARC). @@ -99,7 +101,7 @@ dso_globals: std.array_hash_map.Auto(String(.strtab), struct { /// the section containing the symbol, and the symbol's offset within the section. I know this /// sounds like a terrible hack, but it is *genuinely* how you're supposed to do this. Copy /// relocations suck. - alignment: std.mem.Alignment, + alignment: Alignment, }), shstrtab: StringTable, strtab: StringTable, @@ -175,7 +177,7 @@ symbol_relocs: std.ArrayList(SymbolReloc), got_relocs: std.ArrayList(GotReloc), /// Set of relocations which must be re-applied if the size of the TLS segment changes. tls_size_symbol_relocs: std.array_hash_map.Auto(SymbolReloc.Index, void), -/// Index matches the index into `shdrs`. +/// Index matches the index into `shdrs`. Like `shdrs`, this map excludes `SHN_UNDEF`. section_by_name: std.array_hash_map.Auto(String(.shstrtab), void), /// Key is the name of a global symbol which has been moved to a new symtab index. Any relocation /// entries which target that symbol must be updated to reference the correct symbol index. @@ -339,8 +341,6 @@ const Node = union(enum) { }; pub const Known = struct { - archive: MappedFile.Node.Index, - archive_header: MappedFile.Node.Index, elf: MappedFile.Node.Index, ehdr: MappedFile.Node.Index, shdr: MappedFile.Node.Index, @@ -349,7 +349,7 @@ const Node = union(enum) { text: MappedFile.Node.Index, data: MappedFile.Node.Index, data_rel_ro: MappedFile.Node.Index, - tls: MappedFile.Node.Index, + tls: MappedFile.Node.Index.Optional, }; comptime { @@ -505,7 +505,7 @@ const Section = struct { } fn get(s: Index, elf: *Elf) *Section { - return &elf.shdrs.items[@backingInt(s)]; + return &elf.shdrs.items[@backingInt(s) - 1]; // overflow means you tried to get the `.UNDEF` section } fn name(s: Index, elf: *Elf) String(.shstrtab) { @@ -539,7 +539,7 @@ const Section = struct { } } - fn ensureAligned(shndx: Index, elf: *Elf, min_align: std.mem.Alignment) Error!void { + fn ensureAligned(shndx: Index, elf: *Elf, min_align: Alignment) Error!void { switch (elf.shdrPtr(shndx)) { inline else => |shdr| { if (elf.targetLoad(&shdr.addralign) >= min_align.toByteUnits()) { @@ -552,7 +552,7 @@ const Section = struct { if (min_align.compare(.gt, ni.alignment(&elf.mf))) { try ni.realign(&elf.mf, elf.base.comp.gpa, min_align, .{}); } - switch (elf.getNode(ni.parent(&elf.mf))) { + switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .elf => {}, .segment => |phndx| try elf.ensureSegmentAligned(phndx, min_align), else => unreachable, @@ -818,7 +818,7 @@ const GotReloc = struct { /// * A section /// * A NAV, UAV, or lazy code/data /// * `.none`, if this relocation was deleted (in which case it should be ignored) - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, /// The offset of the relocation inside of `node`. offset: u64, target: GotKey, @@ -942,8 +942,10 @@ const GotReloc = struct { fn apply(reloc: *GotReloc, elf: *Elf) void { assert(elf.ehdrType() != .REL); - if (reloc.node == .none) return; // deleted - if (reloc.node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) { + const node = reloc.node.unwrap() orelse { + return; // deleted + }; + if (node.hasMoved(&elf.mf) or elf.shndx.got.get(elf).ni.hasMoved(&elf.mf)) { // There's no point applying the relocation now, because it will be re-applied by // `flushMoved` at some point anyway. return; @@ -968,8 +970,9 @@ const GotReloc = struct { } } fn applyInner(reloc: *const GotReloc, elf: *Elf) error{ RelocationOverflow, RelocationMisaligned }!void { - const dest_vaddr = elf.getNodeVAddr(reloc.node) + reloc.offset; - const dest_slice = reloc.node.slice(&elf.mf)[@intCast(reloc.offset)..]; + const node = reloc.node.unwrap().?; + const dest_vaddr = elf.getNodeVAddr(node) + reloc.offset; + const dest_slice = node.slice(&elf.mf)[@intCast(reloc.offset)..]; const got_vaddr = elf.shndx.got.vaddr(elf); const got_index: u64 = elf.got.getIndex(reloc.target).?; @@ -1587,7 +1590,7 @@ const SymbolReloc = struct { } }, .sparc_le_hix22 => { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -1646,7 +1649,6 @@ const SymbolReloc = struct { fn apply(reloc: *SymbolReloc, elf: *Elf) void { assert(elf.ehdrType() != .REL); - assert(reloc.node != .none); if (reloc.node.hasMoved(&elf.mf) or reloc.target.hasMoved(elf)) { // There's no point applying the relocation now, because it will be re-applied by // `flushMoved` at some point anyway. @@ -1692,7 +1694,7 @@ const SymbolReloc = struct { .I_original => |tls| tls.tcb_size +% reloc.target.value(elf) +% addend, .I_modified => |tls| 0 -% tls.tp_off +% reloc.target.value(elf) +% addend, .II => { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -2044,7 +2046,7 @@ fn pltEntryIsDead(elf: *Elf, plt_index: usize) bool { } const AddLocalSymbolOptions = struct { - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, name: String(.strtab), value: u64, size: u64, @@ -2126,7 +2128,7 @@ const AddGlobalSymbolOptions = struct { } }; - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, name: Name, lib_name: ?[]const u8 = null, value: u64, @@ -2294,8 +2296,8 @@ fn addGlobalSymbolAssumeCapacity(elf: *Elf, opts: AddGlobalSymbolOptions) error{ } const old_head: String(.strtab) = old_head: { - if (opts.node == .none) break :old_head .empty; - const gop = elf.node_global_symbols.getOrPutAssumeCapacity(opts.node); + const node = opts.node.unwrap() orelse break :old_head .empty; + const gop = elf.node_global_symbols.getOrPutAssumeCapacity(node); const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty; gop.value_ptr.* = opts.name.strtab; break :old_head old_head; @@ -2363,7 +2365,7 @@ fn setGlobalSymbolValue( global_name: String(.strtab), global_ptr: *Symbol.Global, new: struct { - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, value: u64, size: u64, type: std.elf.STT, @@ -2371,18 +2373,17 @@ fn setGlobalSymbolValue( }, ) void { assert(new.shndx != .UNDEF); - const old_node = global_ptr.symtab_index.ptr(elf).node; - if (old_node != .none) { + if (global_ptr.symtab_index.ptr(elf).node.unwrap()) |old_node| { if (global_ptr.next_in_node != .empty) { const next = elf.globalByName(global_ptr.next_in_node).?; assert(next.prev_in_node == global_name); - assert(next.symtab_index.ptr(elf).node == old_node); + assert(next.symtab_index.ptr(elf).node.unwrap().? == old_node); next.prev_in_node = global_ptr.prev_in_node; } if (global_ptr.prev_in_node != .empty) { const prev = elf.globalByName(global_ptr.prev_in_node).?; assert(prev.next_in_node == global_name); - assert(prev.symtab_index.ptr(elf).node == old_node); + assert(prev.symtab_index.ptr(elf).node.unwrap().? == old_node); prev.next_in_node = global_ptr.next_in_node; } else { // We're the start of the linked list, so we need to change the head. @@ -2417,8 +2418,8 @@ fn setGlobalSymbolValue( global_ptr.symtab_index.ptr(elf).node = new.node; const old_head: String(.strtab) = old_head: { - if (new.node == .none) break :old_head .empty; - const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new.node); + const new_node = new.node.unwrap() orelse break :old_head .empty; + const gop = elf.node_global_symbols.getOrPutAssumeCapacity(new_node); const old_head: String(.strtab) = if (gop.found_existing) gop.value_ptr.* else .empty; gop.value_ptr.* = global_name; break :old_head old_head; @@ -2644,7 +2645,7 @@ const Symbol = struct { /// * A section (the symbol's value is some vaddr in that section) /// * An input section (the symbol's value is some vaddr in that input section) /// * A NAV, UAV, or lazy code/data (the symbol's value is exactly the vaddr of that node) - node: MappedFile.Node.Index, + node: MappedFile.Node.Index.Optional, /// The head of a linked list of relocations targeting this symbol. first_target_reloc: SymbolReloc.Index, @@ -2852,8 +2853,7 @@ const Symbol = struct { /// Returns `true` if the target of `s` has moved, meaning the symbol's value will change at /// some point due to a call to `flushMoved`. fn hasMoved(s: Symbol.Id, elf: *Elf) bool { - const node = s.index(elf).ptr(elf).node; - if (node != .none) { + if (s.index(elf).ptr(elf).node.unwrap()) |node| { return node.hasMoved(&elf.mf); } switch (s.unwrap()) { @@ -2998,7 +2998,7 @@ fn lazySymbolInner(elf: *Elf, lazy: link.File.LazySymbol) Error!link.File.Symbol ) catch unreachable; gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, name), .value = 0, .size = 0, @@ -3349,16 +3349,14 @@ fn create( .options = options, .mf = try .init(file, comp.gpa, io), .ni = .{ - .archive = .root, - .archive_header = .none, - .elf = .root, - .ehdr = .none, - .shdr = .none, - .rodata = .none, - .phdr = .none, - .text = .none, - .data = .none, - .data_rel_ro = .none, + .elf = undefined, + .ehdr = undefined, + .shdr = undefined, + .rodata = undefined, + .phdr = undefined, + .text = undefined, + .data = undefined, + .data_rel_ro = undefined, .tls = .none, }, .nodes = .empty, @@ -3489,7 +3487,7 @@ fn initHeaders( .EXEC => comp.config.link_mode == .dynamic, .DYN => true, }; - const addr_align: std.mem.Alignment = switch (class) { + const addr_align: Alignment = switch (class) { .NONE, _ => unreachable, .@"32" => .@"4", .@"64" => .@"8", @@ -3503,7 +3501,7 @@ fn initHeaders( // // It can be handy to temporarily set this to `.@"1"` when working on the linker, because it // prevents alignment bugs from being hidden by your filesystem's block alignment. - const node_block_align: std.mem.Alignment = elf.mf.flags.block_size; + const node_block_align: Alignment = elf.mf.flags.block_size; const plt: PltInfo = .fromMachine(machine); @@ -3601,18 +3599,19 @@ fn initHeaders( const expected_nodes_len = @as(usize, if (is_archive) 2 else 0) + // .archive, .archive_header 3 + // `.file`, `.ehdr`, and `.shdr` nodes - (shnum - 1) + // -1 because the null shdr does not have a `.section` node + (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node try elf.nodes.ensureTotalCapacity(gpa, expected_nodes_len); - try elf.shdrs.ensureTotalCapacity(gpa, shnum); - try elf.section_by_name.ensureUnusedCapacity(gpa, shnum); + try elf.shdrs.ensureTotalCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF + try elf.section_by_name.ensureUnusedCapacity(gpa, shnum - 1); // -1 to exclude SHN_UNDEF try elf.phdrs.resize(gpa, phnum); try elf.symtab.ensureTotalCapacity(gpa, 1); if (is_archive) { elf.nodes.appendAssumeCapacity(.archive); - elf.ni.archive_header = try elf.mf.addOnlyChildNode(gpa, elf.ni.archive, .{ + + const archive_header_ni = try elf.mf.addOnlyChildNode(gpa, .root, .{ .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2, .alignment = .@"2", .fixed = true, @@ -3620,7 +3619,8 @@ fn initHeaders( .bubbles_moved = false, .enable_next_moved = true, }); - const archive_header_slice = elf.ni.archive_header.slice(&elf.mf); + elf.nodes.appendAssumeCapacity(.archive_header); + const archive_header_slice = archive_header_ni.slice(&elf.mf); @memcpy(archive_header_slice[0..std.elf.ARMAG.len], std.elf.ARMAG); const strtab_ar_hdr: *std.elf.ar_hdr = @ptrCast(archive_header_slice[std.elf.ARMAG.len..]); strtab_ar_hdr.* = .{ @@ -3633,15 +3633,17 @@ fn initHeaders( .ar_fmag = std.elf.ARFMAG.*, }; - elf.nodes.appendAssumeCapacity(.archive_header); - elf.ni.elf = try elf.mf.addLastChildNode(gpa, elf.ni.archive, .{ + elf.ni.elf = try elf.mf.addLastChildNode(gpa, .root, .{ .alignment = node_block_align.max(.@"2"), .next_moved = true, .bubbles_moved = false, .enable_next_moved = true, }); + elf.nodes.appendAssumeCapacity(.elf); + } else { + elf.ni.elf = .root; + elf.nodes.appendAssumeCapacity(.elf); } - elf.nodes.appendAssumeCapacity(.elf); const entsize: struct { ph: u32, sh: u32 } = switch (class) { .NONE, _ => unreachable, @@ -3665,7 +3667,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.rodata }); - elf.phdrs.items[phndx.rodata] = elf.ni.rodata; + elf.phdrs.items[phndx.rodata] = .wrap(elf.ni.rodata); elf.ni.phdr = try elf.mf.addOnlyChildNode(gpa, elf.ni.rodata, .{ .size = @as(u64, phnum) * entsize.ph, @@ -3675,7 +3677,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.phdr }); - elf.phdrs.items[phndx.phdr] = elf.ni.phdr; + elf.phdrs.items[phndx.phdr] = .wrap(elf.ni.phdr); elf.ni.text = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ .alignment = node_block_align, @@ -3683,7 +3685,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.text }); - elf.phdrs.items[phndx.text] = elf.ni.text; + elf.phdrs.items[phndx.text] = .wrap(elf.ni.text); elf.ni.data = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ // Must be at least `addr_align` for `elf.ni.data_rel_ro` to be placed inside this node @@ -3692,7 +3694,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.data }); - elf.phdrs.items[phndx.data] = elf.ni.data; + elf.phdrs.items[phndx.data] = .wrap(elf.ni.data); if (plt.got_plt == null) { const plt_ni = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ @@ -3701,7 +3703,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.plt }); - elf.phdrs.items[phndx.plt] = plt_ni; + elf.phdrs.items[phndx.plt] = .wrap(plt_ni); } elf.ni.data_rel_ro = try elf.mf.addOnlyChildNode(gpa, elf.ni.data, .{ @@ -3712,14 +3714,14 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.relro }); - elf.phdrs.items[phndx.relro] = elf.ni.data_rel_ro; + elf.phdrs.items[phndx.relro] = .wrap(elf.ni.data_rel_ro); if (comp.config.any_non_single_threaded) { - elf.ni.tls = try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ + elf.ni.tls = .wrap(try elf.mf.addLastChildNode(gpa, elf.ni.rodata, .{ .alignment = node_block_align, .moved = true, .bubbles_moved = false, - }); + })); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.tls }); elf.phdrs.items[phndx.tls] = elf.ni.tls; } @@ -3785,14 +3787,14 @@ fn initHeaders( ehdr.phentsize = @sizeOf(ElfN.Phdr); ehdr.phnum = @min(phnum, std.elf.PN_XNUM); ehdr.shentsize = @sizeOf(ElfN.Shdr); - ehdr.shnum = 1; // Only the null shdr initially---will be incremented by `addSection` + ehdr.shnum = 1; // Only the SHN_UNDEF shdr initially---will be incremented by `addSection` ehdr.shstrndx = std.elf.SHN_UNDEF; if (elf.targetEndian() != native_endian) std.mem.byteSwapAllFields(ElfN.Ehdr, ehdr); }, } elf.ni.shdr = try elf.mf.addLastChildNode(gpa, elf.ni.elf, .{ - .size = 1 * entsize.sh, // as above, only the null shdr initially + .size = 1 * entsize.sh, // as above, only the SHN_UNDEF initially .alignment = addr_align.max(node_block_align), .moved = true, .resized = true, @@ -3916,7 +3918,7 @@ fn initHeaders( }; } - if (comp.config.any_non_single_threaded) { + if (elf.ni.tls.unwrap()) |tls_segment_ni| { const ph_tls = &phdr[phndx.tls]; ph_tls.* = .{ .type = .TLS, @@ -3926,7 +3928,7 @@ fn initHeaders( .filesz = 0, .memsz = 0, .flags = .{ .R = true }, - .@"align" = @intCast(elf.ni.tls.alignment(&elf.mf).toByteUnits()), + .@"align" = @intCast(tls_segment_ni.alignment(&elf.mf).toByteUnits()), }; } @@ -3987,7 +3989,6 @@ fn initHeaders( .entsize = 0, }; if (target_endian != native_endian) std.mem.byteSwapAllFields(ElfN.Shdr, sh_undef); - elf.shdrs.appendAssumeCapacity(.{ .lsi = .null, .ni = .none, .rela = .{ .shndx = .UNDEF } }); elf.symtab.addOneAssumeCapacity().* = .{ .node = .none, @@ -4092,7 +4093,7 @@ fn initHeaders( .node_align = node_block_align, }); } else { - elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt], .{ + elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{ .name = ".plt", .type = .PROGBITS, .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true }, @@ -4115,7 +4116,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.interp }); - elf.phdrs.items[phndx.interp] = interp_ni; + elf.phdrs.items[phndx.interp] = .wrap(interp_ni); const sec_interp_shndx = try elf.addSection(interp_ni, .{ .name = ".interp", @@ -4135,7 +4136,7 @@ fn initHeaders( .bubbles_moved = false, }); elf.nodes.appendAssumeCapacity(.{ .segment = phndx.dynamic }); - elf.phdrs.items[phndx.dynamic] = dynamic_ni; + elf.phdrs.items[phndx.dynamic] = .wrap(dynamic_ni); const dynstr_shndx = try elf.addSection(elf.ni.rodata, .{ .name = ".dynstr", @@ -4347,7 +4348,7 @@ fn initHeaders( try elf.ensureUnusedSymbolCapacity(10, .maybe_global); // Despite the name, `__dso_handle` is necessary even in static binaries. _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = Section.Index.text.get(elf).ni, + .node = .wrap(Section.Index.text.get(elf).ni), .name = try .string(elf, "__dso_handle"), .value = Section.Index.text.vaddr(elf), .size = 0, @@ -4359,7 +4360,7 @@ fn initHeaders( error.MultipleDefinitions => unreachable, // no inputs are processed yet }; _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.plt.get(elf).ni, + .node = .wrap(elf.shndx.plt.get(elf).ni), .name = try .string(elf, "_PROCEDURE_LINKAGE_TABLE_"), .value = elf.shndx.plt.vaddr(elf), .size = 0, @@ -4371,7 +4372,7 @@ fn initHeaders( error.MultipleDefinitions => unreachable, // no inputs are processed yet }; _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.got.get(elf).ni, + .node = .wrap(elf.shndx.got.get(elf).ni), .name = try .string(elf, "_GLOBAL_OFFSET_TABLE_"), .value = switch (machine) { .AARCH64, @@ -4468,7 +4469,7 @@ fn initHeaders( }; if (have_dynamic_section) { _ = elf.addGlobalSymbolAssumeCapacity(.{ - .node = elf.shndx.dynamic.get(elf).ni, + .node = .wrap(elf.shndx.dynamic.get(elf).ni), .name = try .string(elf, "_DYNAMIC"), .value = elf.shndx.dynamic.vaddr(elf), .size = 0, @@ -4484,16 +4485,16 @@ fn initHeaders( assert(maybe_interp == null); assert(!have_dynamic_section); } - if (comp.config.any_non_single_threaded) elf.shndx.tdata = try elf.addSection(elf.ni.tls, .{ + if (elf.ni.tls.unwrap()) |tls_segment_ni| elf.shndx.tdata = try elf.addSection(tls_segment_ni, .{ .name = ".tdata", .flags = .{ .WRITE = true, .ALLOC = true, .TLS = true }, .node_align = node_block_align, }); assert(elf.nodes.len == expected_nodes_len); - assert(elf.shdrs.items.len == shnum); + assert(elf.shdrs.items.len == shnum - 1); // -1 to exclude SHN_UNDEF - for (0..shnum) |shndx_raw| { + for (1..shnum) |shndx_raw| { // start at 1 to exclude SHN_UNDEF const shndx: Section.Index = @fromBackingInt(@intCast(shndx_raw)); elf.section_by_name.putAssumeCapacityNoClobber(shndx.name(elf), {}); } @@ -4569,7 +4570,7 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index { .uav, .lazy_code, .lazy_const_data, - => elf.getNode(ni.parent(&elf.mf)).section, + => elf.getNode(ni.parent(&elf.mf).unwrap().?).section, }; } fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { @@ -4593,7 +4594,7 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { }; } fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 { - const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf))) { + const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) { .archive, .archive_header => unreachable, .elf => return 0, .ehdr, .shdr => unreachable, @@ -4660,7 +4661,7 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void { if (got_relocs) |ptr| { if (ptr.* != .none) { for (elf.got_relocs.items[@backingInt(ptr.*)..]) |*reloc| { - if (reloc.node != ni) break; + if (reloc.node != ni.toOptional()) break; reloc.delete(elf); } } @@ -4691,7 +4692,7 @@ fn flushMovedNodeRelocs( if (first_got_reloc != .none) { for (elf.got_relocs.items[@backingInt(first_got_reloc)..]) |*reloc| { - if (reloc.node != node) break; + if (reloc.node != node.toOptional()) break; reloc.apply(elf); } } @@ -4756,7 +4757,7 @@ fn targetPtrSize(elf: *const Elf) u8 { /// Page alignment for the target platform. /// Usually this returns the maximum page size supported on the /// target to maximize compatibility but there can be exceptions. -fn targetPageAlign(elf: *const Elf) std.mem.Alignment { +fn targetPageAlign(elf: *const Elf) Alignment { return .fromByteUnits(switch (elf.ehdrMachine()) { .AARCH64 => 0x10000, .LOONGARCH => 0x10000, @@ -4810,7 +4811,7 @@ const PltInfo = struct { /// entry, not the `.plt` entry. The `.plt.sec` section has no header entries, and is aligned to /// the same boundary as the `.plt` section. plt_sec: ?struct { entry_size: u8 }, - @"align": std.mem.Alignment, + @"align": Alignment, entry_size: u8, header_entries: u8, @@ -4941,8 +4942,9 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { switch (elf.identClass()) { .NONE, _ => unreachable, inline else => |class| { + const shdrs_len = elf.shdrs.items.len + 1; // +1 for SHN_UNDEF const shdr_slice: []class.ElfN().Shdr = @ptrCast(@alignCast( - raw_slice[0 .. elf.shdrs.items.len * @sizeOf(class.ElfN().Shdr)], + raw_slice[0 .. shdrs_len * @sizeOf(class.ElfN().Shdr)], )); const shdr_ptr = &shdr_slice[@backingInt(shndx)]; return @unionInit(ShdrPtr, @tagName(class), shdr_ptr); @@ -4951,7 +4953,7 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr { } fn arHdrPtr(elf: *Elf, ni: MappedFile.Node.Index) *align(2) std.elf.ar_hdr { - assert(elf.ni.elf != MappedFile.Node.Index.root); + assert(elf.ni.elf != .root); const file_offset = ni.fileLocation(&elf.mf, false).offset; return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) { else => unreachable, @@ -5055,7 +5057,7 @@ fn mapInputSection(elf: *Elf, opts: struct { const parent_node: MappedFile.Node.Index = parent: { if (!opts.flags.ALLOC) break :parent elf.ni.elf; if (opts.flags.EXECINSTR) break :parent elf.ni.text; - if (opts.flags.TLS) break :parent elf.ni.tls; + if (opts.flags.TLS) break :parent elf.ni.tls.unwrap().?; if (opts.flags.WRITE) break :parent elf.ni.data; break :parent elf.ni.rodata; }; @@ -5148,12 +5150,12 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node break :section .data_rel_ro; // TODO: it would be better to use `.rodata` if the NAV value doesn't have relocs } }; - const alignment: InternPool.Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) { + const alignment: Alignment = switch (Type.fromInterned(nav.resolved.?.type).zigTypeTag(zcu)) { .@"fn" => a: { const mod = zcu.navFileScope(nav_index).mod.?; const target = &mod.resolved_target.result; const min = target_util.minFunctionAlignment(target); - break :a switch (nav.resolved.?.@"align") { + break :a .fromIp(switch (nav.resolved.?.@"align") { else => |a| a.maxStrict(min), .none => switch (mod.optimize_mode) { .debug, @@ -5162,20 +5164,20 @@ fn navMapIndex(elf: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index) Error!Node => target_util.defaultFunctionAlignment(target), .small => min, }.maxStrict(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), - }; + }); }, else => switch (nav.resolved.?.@"align") { - .none => Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu), - else => |a| a, + .none => .fromIp(Type.fromInterned(nav.resolved.?.type).abiAlignment(zcu)), + else => |a| .fromIp(a), }, }; - try shndx.ensureAligned(elf, alignment.toStdMem()); + try shndx.ensureAligned(elf, alignment); const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ - .alignment = alignment.toStdMem(), + .alignment = alignment, }); nav_gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, nav.fqn.toSlice(ip)), .value = 0, .size = 0, @@ -5204,19 +5206,19 @@ fn uavMapIndex( try elf.pending_uavs.ensureUnusedCapacity(gpa, 1); const abi_align = Value.fromInterned(uav_val).typeOf(zcu).abiAlignment(zcu); - const resolved_align: InternPool.Alignment = switch (uav_align) { - .none => abi_align, - else => |a| a.minStrict(abi_align), + const resolved_align: Alignment = switch (uav_align) { + .none => .fromIp(abi_align), + else => |a| .fromIp(a.minStrict(abi_align)), }; const uav_gop = elf.uavs.getOrPutAssumeCapacity(uav_val); const umi: Node.UavMapIndex = @fromBackingInt(@intCast(uav_gop.index)); if (!uav_gop.found_existing) { const shndx: Section.Index = .data_rel_ro; // TODO: it would be better to use `.rodata` if the UAV value doesn't have relocs - try shndx.ensureAligned(elf, resolved_align.toStdMem()); + try shndx.ensureAligned(elf, resolved_align); const node = try elf.mf.addLastChildNode(gpa, shndx.get(elf).ni, .{ .moved = true, // see assert at end of `genUav` - .alignment = resolved_align.toStdMem(), + .alignment = resolved_align, }); var name_buf: [32]u8 = undefined; const name = std.fmt.bufPrint( @@ -5226,7 +5228,7 @@ fn uavMapIndex( ) catch unreachable; uav_gop.value_ptr.* = .{ .lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .name = try elf.string(.strtab, name), .value = 0, .size = 0, @@ -5239,11 +5241,11 @@ fn uavMapIndex( elf.const_prog_node.increaseEstimatedTotalItems(1); elf.pending_uavs.appendAssumeCapacity(umi); } else { - const node = uav_gop.value_ptr.lsi.index().ptr(elf).node; - const shndx = elf.getNode(node.parent(&elf.mf)).section; - try shndx.ensureAligned(elf, resolved_align.toStdMem()); - if (resolved_align.toStdMem().order(node.alignment(&elf.mf)).compare(.gt)) { - try node.realign(&elf.mf, gpa, resolved_align.toStdMem(), .{}); + const node = uav_gop.value_ptr.lsi.index().ptr(elf).node.unwrap().?; + const shndx = elf.getNode(node.parent(&elf.mf).unwrap().?).section; + try shndx.ensureAligned(elf, resolved_align); + if (resolved_align.order(node.alignment(&elf.mf)).compare(.gt)) { + try node.realign(&elf.mf, gpa, resolved_align, .{}); } } return umi; @@ -5459,7 +5461,7 @@ fn loadObject( .member = if (member) |m| try gpa.dupe(u8, m) else null, .extra = undefined, }; - if (elf.ni.elf != MappedFile.Node.Index.root) { + if (elf.ni.elf != .root) { try elf.nodes.ensureUnusedCapacity(gpa, 1); input.extra = .{ .node = try elf.mf.addLastChildNode(gpa, .root, .{ .size = fl.size + @sizeOf(std.elf.ar_hdr), @@ -5640,7 +5642,7 @@ fn loadObject( .node_fixed = true, }, }; - const need_align: std.mem.Alignment = .fromByteUnits( + const need_align: Alignment = .fromByteUnits( std.math.ceilPowerOfTwoAssert(usize, @intCast(@max(section.shdr.addralign, 1))), ); try opts.shndx.ensureAligned(elf, need_align); @@ -5754,7 +5756,7 @@ fn loadObject( ), .LOCAL => { const lsi = elf.addLocalSymbolAssumeCapacity(.{ - .node = input_section_node, + .node = .wrap(input_section_node), .name = try elf.string(.strtab, name), .value = input_sym.value, .size = input_sym.size, @@ -5765,7 +5767,7 @@ fn loadObject( }, .GLOBAL, .WEAK, .GNU_UNIQUE => |bind| { si.* = elf.addGlobalSymbolAssumeCapacity(.{ - .node = input_section_node, + .node = .wrap(input_section_node), .name = try .string(elf, name), .value = input_sym.value, .size = input_sym.size, @@ -5893,7 +5895,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars return diags.failParse(path, "bad machine", .{}); if (ehdr.shnum > 0) try fr.seekTo(ehdr.shoff); // We're going to need to know the alignment of every section later. - const section_aligns = try gpa.alloc(std.mem.Alignment, ehdr.shnum); + const section_aligns = try gpa.alloc(Alignment, ehdr.shnum); defer gpa.free(section_aligns); const dynamic_sh: ElfN.Shdr, const dynsym_sh: ElfN.Shdr = sh: { var dynamic_sh: ?ElfN.Shdr = null; @@ -5999,7 +6001,7 @@ fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadPars // We need to guess the worst-case alignment of the symbol. Yes, I know this seems // insane---refer to the doc comment on `alignment` in `Elf.dso_globals`. - const sym_align: std.mem.Alignment = switch (sym.value) { + const sym_align: Alignment = switch (sym.value) { 0 => section_aligns[sym.shndx], else => section_aligns[sym.shndx].min(@fromBackingInt(@intCast(@ctz(sym.value)))), }; @@ -6158,7 +6160,7 @@ fn createInitFiniArraySection( ) Error!void { assert(shndx.* == .UNDEF); const gpa = elf.base.comp.gpa; - const addr_align: std.mem.Alignment = switch (elf.identClass()) { + const addr_align: Alignment = switch (elf.identClass()) { .NONE, _ => unreachable, .@"32" => .@"4", .@"64" => .@"8", @@ -6178,14 +6180,14 @@ fn createInitFiniArraySection( const start_sym_name = try elf.string(.strtab, "__" ++ name ++ "_start"); const end_sym_name = try elf.string(.strtab, "__" ++ name ++ "_end"); elf.setGlobalSymbolValue(start_sym_name, elf.globals.strong_def.getPtr(start_sym_name).?, .{ - .node = shndx.get(elf).ni, + .node = .wrap(shndx.get(elf).ni), .value = shndx.vaddr(elf), .size = 0, .type = .NOTYPE, .shndx = shndx.*, }); elf.setGlobalSymbolValue(end_sym_name, elf.globals.strong_def.getPtr(end_sym_name).?, .{ - .node = shndx.get(elf).ni, + .node = .wrap(shndx.get(elf).ni), .value = shndx.vaddr(elf), .size = 0, .type = .NOTYPE, @@ -6218,7 +6220,7 @@ fn prelinkInner(elf: *Elf) Error!void { const comp = elf.base.comp; const gpa = comp.gpa; - if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == MappedFile.Node.Index.root) { + if (comp.zcu != null and !comp.config.use_llvm and elf.ni.elf == .root) { // We're using self-hosted codegen---add an input representing the Zig "object". try elf.ensureUnusedSymbolCapacity(1, .all_local); try elf.inputs.ensureUnusedCapacity(gpa, 1); @@ -6388,9 +6390,9 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { size: std.elf.Xword = 0, link: std.elf.Word = 0, info: std.elf.Word = 0, - addralign: std.mem.Alignment = .@"1", + addralign: Alignment = .@"1", entsize: std.elf.Word = 0, - node_align: std.mem.Alignment = .@"1", + node_align: Alignment = .@"1", fixed: bool = false, }) Error!Section.Index { switch (opts.type) { @@ -6447,7 +6449,7 @@ fn addSection(elf: *Elf, segment_ni: MappedFile.Node.Index, opts: struct { }); const addr = elf.computeNodeVAddr(ni); const lsi: Symbol.LocalIndex = if (opts.flags.ALLOC) elf.addLocalSymbolAssumeCapacity(.{ - .node = ni, + .node = .wrap(ni), .name = .empty, .value = addr, .size = 0, @@ -6499,7 +6501,7 @@ fn ensureUnusedRelocCapacity(elf: *Elf, node: MappedFile.Node.Index, len: usize) assert(elf.section_by_name.count() == elf.shdrs.items.len); try elf.section_by_name.ensureUnusedCapacity(gpa, 1); - const rela_shndx = try elf.addSection(.none, .{ + const rela_shndx = try elf.addSection(elf.ni.elf, .{ .name = rela_name, .type = .RELA, .link = @backingInt(Section.Index.symtab), @@ -6546,7 +6548,6 @@ fn addRelocAssumeCapacity( addend: i64, @"type": MachineRelocType, ) (Error || error{ UnknownRelocation, NonStaticRelocation, UnimplementedRelocation })!void { - assert(node != .none); switch (elf.ehdrType()) { .REL => { const rela_shndx = elf.getNodeShndx(node).get(elf).rela.shndx; @@ -6894,7 +6895,6 @@ fn addSymbolRelocAssumeCapacity( @"type": SymbolReloc.Type, ) Error!void { assert(elf.ehdrType() != .REL); - assert(node != .none); const rela_index: Section.RelaIndex.Optional = r: { if (elf.shndx.dynamic == .UNDEF) break :r .none; @@ -7089,7 +7089,7 @@ fn addGotRelocAssumeCapacity( } elf.got_relocs.appendAssumeCapacity(.{ - .node = node, + .node = .wrap(node), .offset = offset, .target = target, .addend = addend, @@ -7111,7 +7111,7 @@ fn updateGotEntry(elf: *Elf, got_index: usize) void { .tpoff => |sym_id| val: { // Only the executable's per-module TLS block is at a known offset from the TLS pointer. if (elf.base.comp.config.output_mode == .Exe and elf.classifySymbolValue(sym_id) != .dynamic) { - const tls_phndx = elf.getNode(elf.ni.tls).segment; + const tls_phndx = elf.getNode(elf.ni.tls.unwrap().?).segment; const tls_size: u64 = switch (elf.phdrSlice()) { inline else => |phdr| tls_size: { assert(elf.targetLoad(&phdr[tls_phndx].type) == .TLS); @@ -7336,7 +7336,7 @@ fn updateNavInner(elf: *Elf, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) if (!Type.fromInterned(nav.resolved.?.type).hasRuntimeBits(zcu)) return; const nmi = try elf.navMapIndex(zcu, nav_index); - const ni = nmi.symbol(elf).index().ptr(elf).node; + const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be @@ -7392,7 +7392,7 @@ fn updateFuncInner( const nmi = try elf.navMapIndex(zcu, func.owner_nav); log.debug("updateFunc({f}) = {d}", .{ nav.fqn.fmt(ip), nmi.symbol(elf) }); - const ni = nmi.symbol(elf).index().ptr(elf).node; + const ni = nmi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the NAV is marked as moved so that once we're done, `flushMoved` will eventually be @@ -7677,7 +7677,7 @@ fn idleProgNode( break :name std.fmt.bufPrint(&name, "{f}{f} {s}", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), }) catch &name; }, .nav => |nmi| { @@ -7737,7 +7737,7 @@ fn genUav( const gpa = comp.gpa; const uav_val = umi.uavValue(elf); - const ni = umi.symbol(elf).index().ptr(elf).node; + const ni = umi.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); var nw: MappedFile.Node.Writer = undefined; @@ -7766,7 +7766,7 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void { const gpa = zcu.gpa; const lazy = lmr.lazySymbol(elf); - const ni = lmr.symbol(elf).index().ptr(elf).node; + const ni = lmr.symbol(elf).index().ptr(elf).node.unwrap().?; elf.resetNodeRelocs(ni); // Ensure the lazy node is marked as moved so that once we're done, `flushMoved` will eventually @@ -7842,7 +7842,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { fr.seekTo(file_loc.offset) catch |err| switch (err) { error.Canceled => |e| return e, else => |e| return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), e, @@ -7853,7 +7853,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { defer nw.deinit(); const n_bytes = nw.interface.sendFileAll(&fr, .limited(@intCast(file_loc.size))) catch |err| switch (err) { error.ReadFailed => return diags.fail("failed to read input section '{s}' from \"{f}{f}\": {t}", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), fr.err orelse (fr.seek_err orelse fr.size_err.?), @@ -7861,7 +7861,7 @@ fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void { error.WriteFailed => return nw.err.?, }; if (n_bytes != file_loc.size) return diags.fail("failed to read input section '{s}' from \"{f}{f}\": unexpected eof", .{ - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), path.fmtEscapeString(), fmtMemberString(ii.member(elf)), }); @@ -7994,7 +7994,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void const ii = isi.input(elf); var lsi, const end_lsi = ii.localSymbolRange(elf); while (lsi != end_lsi) : (lsi = @fromBackingInt(@backingInt(lsi) + 1)) { - if (lsi.index().ptr(elf).node != ni) continue; + if (lsi.index().ptr(elf).node != ni.toOptional()) continue; const visibility: std.elf.STV = switch (elf.symPtr(lsi.index())) { inline else => |sym| elf.targetLoad(&sym.other).visibility, }; @@ -8079,7 +8079,7 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void /// moving or resizing of a segment could reorder them and thereby affect how we handle *future* /// changes to segments. fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Error!void { - const segment_ni = elf.phdrs.items[orig_phndx]; + const segment_ni = elf.phdrs.items[orig_phndx].unwrap().?; assert(elf.getNode(segment_ni).segment == orig_phndx); const page_align = elf.targetPageAlign(); const node_align = segment_ni.alignment(&elf.mf); @@ -8165,7 +8165,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro const next_ni = elf.phdrs.items[next_phndx]; elf.phdrs.items[phndx] = next_ni; elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx }; - elf.phdrs.items[next_phndx] = segment_ni; + elf.phdrs.items[next_phndx] = .wrap(segment_ni); elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) }; phndx = @intCast(next_phndx); } @@ -8203,7 +8203,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo .shdr => {}, .segment => |phndx| switch (elf.phdrSlice()) { inline else => |phdr| { - assert(elf.phdrs.items[phndx] == ni); + assert(elf.phdrs.items[phndx].unwrap().? == ni); const ph = &phdr[phndx]; elf.targetStore(&ph.filesz, @intCast(size)); switch (elf.targetLoad(&ph.type)) { @@ -8301,51 +8301,45 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error! break :member_offset switch (tag) { else => unreachable, .archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true }, - .elf, .input_member => .{ offset, switch (ni.prev(&elf.mf)) { - .none => unreachable, - else => |prev_ni| !prev_ni.hasNextMoved(&elf.mf), - } }, + .elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) }, }; }; - const member_size = member_end: switch (ni.next(&elf.mf)) { - else => |next_ni| { - const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); - const next_member_size = next_member_end: switch (next_ni.next(&elf.mf)) { - else => |next_next_ni| { - const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); - break :next_member_end next_next_offset - @sizeOf(std.elf.ar_hdr); - }, - .none => { - _, const parent_size = - ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf); - break :next_member_end parent_size; - }, - } - next_offset; - const ar_hdr = elf.arHdrPtr(next_ni); - var name_buf: [16]u8 = undefined; - _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ - switch (elf.getNode(next_ni)) { - else => unreachable, - .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), - .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ - std.fs.path.basename(ii.path(elf).sub_path), - }), - } catch @panic("TODO: long archive member names"), - }) catch @panic("TODO: long archive member names"); - ar_hdr.ar_date = "0 ".*; - ar_hdr.ar_uid = "0 ".*; - ar_hdr.ar_gid = "0 ".*; - ar_hdr.ar_mode = "644 ".*; - _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch - @panic("archive member too large"); - ar_hdr.ar_fmag = std.elf.ARFMAG.*; - break :member_end next_offset - @sizeOf(std.elf.ar_hdr); - }, - .none => { - _, const parent_size = ni.parent(&elf.mf).location(&elf.mf).resolve(&elf.mf); - break :member_end parent_size; - }, - } - member_offset; + const member_size = if (ni.next(&elf.mf).unwrap()) |next_ni| member_size: { + const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf); + const next_member_size = if (next_ni.next(&elf.mf).unwrap()) |next_next_ni| next_member_size: { + const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf); + const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr); + break :next_member_size next_member_end - next_offset; + } else next_member_size: { + _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); + const next_member_end = parent_size; + break :next_member_size next_member_end - next_offset; + }; + const ar_hdr = elf.arHdrPtr(next_ni); + var name_buf: [16]u8 = undefined; + _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{ + switch (elf.getNode(next_ni)) { + else => unreachable, + .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}), + .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{ + std.fs.path.basename(ii.path(elf).sub_path), + }), + } catch @panic("TODO: long archive member names"), + }) catch @panic("TODO: long archive member names"); + ar_hdr.ar_date = "0 ".*; + ar_hdr.ar_uid = "0 ".*; + ar_hdr.ar_gid = "0 ".*; + ar_hdr.ar_mode = "644 ".*; + _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch + @panic("archive member too large"); + ar_hdr.ar_fmag = std.elf.ARFMAG.*; + const member_end = next_offset - @sizeOf(std.elf.ar_hdr); + break :member_size member_end - member_offset; + } else member_size: { + _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf); + const member_end = parent_size; + break :member_size member_end - member_offset; + }; if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{ member_size, }) catch @panic("archive member too large"); @@ -8775,12 +8769,13 @@ fn updateExportInner( // only emitting this error if the symbol we're conflicting with comes from an input // section (as opposed to the ZCU). const conflicting_global = elf.globalByName(try elf.string(.strtab, name)).?; - const conflicting_node = conflicting_global.symtab_index.ptr(elf).node; - if (elf.getNode(conflicting_node) == .input_section) { - return elf.base.comp.link_diags.fail( - "multiple definitions of '{s}'", - .{name}, - ); + if (conflicting_global.symtab_index.ptr(elf).node.unwrap()) |conflicting_node| { + if (elf.getNode(conflicting_node) == .input_section) { + return elf.base.comp.link_diags.fail( + "multiple definitions of '{s}'", + .{name}, + ); + } } }, }; @@ -8842,7 +8837,7 @@ pub fn printNode( try w.print("({f}{f}, {s})", .{ ii.path(elf).fmtEscapeString(), fmtMemberString(ii.member(elf)), - elf.getNode(isi.node(elf).parent(&elf.mf)).section.name(elf).slice(elf), + elf.getNode(isi.node(elf).parent(&elf.mf).unwrap().?).section.name(elf).slice(elf), }); }, .copied_global => |name| try w.print("(copy:{s})", .{name.slice(elf)}), @@ -8916,14 +8911,14 @@ pub fn printNode( } } -fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignment) Error!void { +fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: Alignment) Error!void { const gpa = elf.base.comp.gpa; // We need to loop through parent nodes because segments may be nested (e.g. a PT_TLS segment // inside a PT_LOAD segment). var phndx = start_phndx; while (true) { // Align the actual node - const seg_ni = elf.phdrs.items[phndx]; + const seg_ni = elf.phdrs.items[phndx].unwrap().?; if (min_align.compare(.gt, seg_ni.alignment(&elf.mf))) { try seg_ni.realign(&elf.mf, gpa, min_align, .{}); } @@ -8948,7 +8943,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen }, } // Continue on to the parent segment, if any - switch (elf.getNode(seg_ni.parent(&elf.mf))) { + switch (elf.getNode(seg_ni.parent(&elf.mf).unwrap().?)) { .segment => |parent_phndx| phndx = parent_phndx, .elf => return, else => unreachable, @@ -8959,7 +8954,7 @@ fn ensureSegmentAligned(elf: *Elf, start_phndx: u32, min_align: std.mem.Alignmen /// Must be called deterministically after any call to `MappedFile.Node.Index.resize` /// (of `elf.ni.elf` or one of its children) before any possible calls to `idle`. fn ensureElfNodeSize(elf: *Elf) MappedFile.Error!void { - if (elf.ni.elf == MappedFile.Node.Index.root) return; + if (elf.ni.elf == .root) return; var child_it = elf.ni.elf.reverseChildren(&elf.mf); const last_end = if (child_it.next()) |last_ni| last_end: { const last_offset, const last_size = last_ni.location(&elf.mf).resolve(&elf.mf); diff --git a/src/link/MappedFile.zig b/src/link/MappedFile.zig index a6b86e2fc7ac35353027705400fc95e63e229587..b1da80973c756c2aab7b30d88b3605171b06362f 100644 --- a/src/link/MappedFile.zig +++ b/src/link/MappedFile.zig @@ -13,14 +13,14 @@ const windows = std.os.windows; io: Io, flags: packed struct { - block_size: std.mem.Alignment, + block_size: Alignment, copy_file_range_unsupported: bool, fallocate_punch_hole_unsupported: bool, fallocate_insert_range_unsupported: bool, }, memory_map: Io.File.MemoryMap, nodes: std.ArrayList(Node), -free_ni: Node.Index, +free_ni: Node.Index.Optional, large: std.ArrayList(u64), updates: std.ArrayList(Node.Index), /// This progress node's estimated total items is increased once for each node appended to `updates`. @@ -62,6 +62,94 @@ pub const Error = Allocator.Error || Io.Cancelable || error{ MappedFileIo, }; +/// This separate `Alignment` type exists because neither of the other options is really suitable: +/// +/// * `std.mem.Alignment` is based on `usize`, which---while technically okay since the file is +/// memory-mapped---is in practice very annoying to work with in linker implementations +/// +/// * `InternPool.Alignment` is based on `u64`, which is better, but it has the value `.none`, which +/// is also really annoying to handle, because no alignment is ever nullable in this API +/// +/// At some point we should probably just change `InternPool.Alignment` to be non-optional, and add +/// a new `InternPool.Alignment.Optional` type for the case where it can actually be `.none`. At +/// that point we can transition this code to using `InternPool.Alignment` (although it should +/// probably be namespaced elsewhere, it has nothing to do with the `InternPool`!). +pub const Alignment = enum(u6) { + @"1" = 0, + @"2" = 1, + @"4" = 2, + @"8" = 3, + @"16" = 4, + @"32" = 5, + @"64" = 6, + _, + + pub fn fromIp(a: @import("../InternPool.zig").Alignment) Alignment { + assert(a != .none); + return @bitCast(a); + } + + pub fn toLog2Units(a: Alignment) u6 { + return @backingInt(a); + } + + pub fn fromLog2Units(a: u6) Alignment { + return @fromBackingInt(a); + } + + pub fn toByteUnits(a: Alignment) u64 { + return @as(u64, 1) << @backingInt(a); + } + + pub fn fromByteUnits(n: u64) Alignment { + assert(std.math.isPowerOfTwo(n)); + return @fromBackingInt(@intCast(@ctz(n))); + } + + pub fn order(lhs: Alignment, rhs: Alignment) std.math.Order { + return std.math.order(@backingInt(lhs), @backingInt(rhs)); + } + + pub fn compare(lhs: Alignment, op: std.math.CompareOperator, rhs: Alignment) bool { + return std.math.compare(@backingInt(lhs), op, @backingInt(rhs)); + } + + pub fn max(lhs: Alignment, rhs: Alignment) Alignment { + return @fromBackingInt(@max(@backingInt(lhs), @backingInt(rhs))); + } + + pub fn min(lhs: Alignment, rhs: Alignment) Alignment { + return @fromBackingInt(@min(@backingInt(lhs), @backingInt(rhs))); + } + + pub inline fn of(comptime T: type) Alignment { + return comptime .fromByteUnits(@alignOf(T)); + } + + /// Given that a base address is known to be aligned to `a`, computes the known alignment of + /// that base address plus `off`. + pub fn offset(a: Alignment, off: u64) Alignment { + return .fromLog2Units(@min(a.toLog2Units(), @ctz(off))); + } + + /// Align an address forwards to this alignment. + pub fn forward(a: Alignment, addr: u64) u64 { + const x = (@as(u64, 1) << @backingInt(a)) - 1; + return (addr + x) & ~x; + } + + /// Align an address backwards to this alignment. + pub fn backward(a: Alignment, addr: u64) u64 { + const x = (@as(u64, 1) << @backingInt(a)) - 1; + return addr & ~x; + } + + /// Check if an address is aligned to this amount. + pub fn check(a: Alignment, addr: u64) bool { + return @ctz(addr) >= @backingInt(a); + } +}; + pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancelable || IoError)!MappedFile { var mf: MappedFile = .{ .io = io, @@ -101,7 +189,7 @@ pub fn init(file: Io.File, gpa: Allocator, io: Io) (Allocator.Error || Io.Cancel .alignment = mf.flags.block_size, .fixed = true, } }); - assert(root_ni == Node.Index.root); + assert(root_ni == .root); try mf.ensureTotalCapacityInner(@intCast(size)); return mf; } @@ -117,17 +205,17 @@ pub fn deinit(mf: *MappedFile, gpa: Allocator) void { } pub const Node = extern struct { - parent: Node.Index, - prev: Node.Index, - next: Node.Index, - first: Node.Index, - last: Node.Index, + parent: Node.Index.Optional, + prev: Node.Index.Optional, + next: Node.Index.Optional, + first: Node.Index.Optional, + last: Node.Index.Optional, flags: Flags, location_payload: Location.Payload, pub const Flags = packed struct(u32) { location_tag: Location.Tag, - alignment: std.mem.Alignment, + alignment: Alignment, /// Whether this node can be moved. fixed: bool, /// Whether this node has been moved. @@ -142,7 +230,7 @@ pub const Node = extern struct { bubbles_moved: bool, /// Whether `next_moved` events are reported in `updates`. enable_next_moved: bool, - unused: @Int(.unsigned, 32 - @bitSizeOf(std.mem.Alignment) - 8) = 0, + unused: u18 = 0, }; pub const Location = union(enum(u1)) { @@ -180,46 +268,62 @@ pub const Node = extern struct { }; pub const Index = enum(u32) { - none, + root, _, - pub const root: Node.Index = .none; + pub const Optional = enum(u32) { + none = std.math.maxInt(u32), + _, + + pub fn unwrap(oi: Optional) ?Index { + return switch (oi) { + _ => @fromBackingInt(@backingInt(oi)), + .none => null, + }; + } + pub fn wrap(i: Index) Optional { + const oi: Optional = @bitCast(i); + assert(oi != .none); + return oi; + } + }; fn get(ni: Node.Index, mf: *const MappedFile) *Node { return &mf.nodes.items[@backingInt(ni)]; } - pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index { + /// Alias for `Optional.wrap`, provided for convenience when a result type is not available. + pub const toOptional = Optional.wrap; + + pub fn parent(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).parent; } - pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index { + pub fn next(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).next; } fn setNext( prev_ni: Node.Index, gpa: Allocator, - next_ni: Node.Index, + next_ni: Node.Index.Optional, mf: *MappedFile, ) Allocator.Error!void { - assert(prev_ni != .none); const prev_next = &prev_ni.get(mf).next; if (prev_next.* == next_ni) return; prev_next.* = next_ni; try prev_ni.nextMoved(gpa, mf); } - pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index { + pub fn prev(ni: Node.Index, mf: *const MappedFile) Node.Index.Optional { return ni.get(mf).prev; } pub fn ChildIterator(comptime direction: enum { prev, next }) type { return struct { mf: *const MappedFile, - ni: Node.Index, + ni: Node.Index.Optional, pub fn next(it: *@This()) ?Node.Index { - const ni = it.ni; - if (ni == .none) return null; + const ni = it.ni.unwrap() orelse return null; it.ni = @field(ni.get(it.mf), @tagName(direction)); return ni; } @@ -233,20 +337,20 @@ pub const Node = extern struct { } pub fn childrenMoved(ni: Node.Index, gpa: Allocator, mf: *MappedFile) Allocator.Error!void { - var child_ni = ni.get(mf).last; - while (child_ni != .none) { + var child_oni = ni.get(mf).last; + while (child_oni.unwrap()) |child_ni| { try child_ni.moved(gpa, mf); - child_ni = child_ni.get(mf).prev; + child_oni = child_ni.get(mf).prev; } } pub fn hasMoved(ni: Node.Index, mf: *const MappedFile) bool { var parent_ni = ni; - while (parent_ni != Node.Index.root) { + while (parent_ni != .root) { const parent_node = parent_ni.get(mf); if (!parent_node.flags.bubbles_moved) break; if (parent_node.flags.moved) return true; - parent_ni = parent_node.parent; + parent_ni = parent_node.parent.unwrap().?; } return false; } @@ -263,9 +367,8 @@ pub const Node = extern struct { if (ni.hasMoved(mf)) return; const node = ni.get(mf); node.flags.moved = true; - switch (node.prev) { - .none => {}, - else => |prev_ni| prev_ni.nextMovedAssumeCapacity(mf), + if (node.prev.unwrap()) |prev_ni| { + prev_ni.nextMovedAssumeCapacity(mf); } if (node.flags.resized or node.flags.next_moved) return; mf.updates.appendAssumeCapacity(ni); @@ -314,7 +417,7 @@ pub const Node = extern struct { mf.update_prog_node.increaseEstimatedTotalItems(1); } - pub fn alignment(ni: Node.Index, mf: *const MappedFile) std.mem.Alignment { + pub fn alignment(ni: Node.Index, mf: *const MappedFile) Alignment { return ni.get(mf).flags.alignment; } @@ -361,8 +464,11 @@ pub const Node = extern struct { while (true) { const parent_node = parent_ni.get(mf); if (set_has_content) parent_node.flags.has_content = true; - if (parent_ni == .none) break; - parent_ni = parent_node.parent; + if (parent_ni == .root) { + assert(parent_node.parent == .none); + break; + } + parent_ni = parent_node.parent.unwrap().?; const parent_offset, _ = parent_ni.location(mf).resolve(mf); offset += parent_offset; } @@ -402,12 +508,12 @@ pub const Node = extern struct { }; /// Moves and expands a node such that its offset and size are aligned to `new_alignment`. - /// Asserts that `ni` is not `Node.Index.root`. + /// Asserts that `ni` is not `.root`. pub fn realign( ni: Node.Index, mf: *MappedFile, gpa: Allocator, - new_alignment: std.mem.Alignment, + new_alignment: Alignment, opts: RealignNodeOptions, ) Error!void { mf.realignNode(gpa, ni, new_alignment, opts) catch |err| switch (err) { @@ -590,9 +696,9 @@ pub const Node = extern struct { }; fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { - parent: Node.Index = .none, - prev: Node.Index = .none, - next: Node.Index = .none, + parent: Node.Index.Optional = .none, + prev: Node.Index.Optional = .none, + next: Node.Index.Optional = .none, offset: u64 = 0, add_node: AddNodeOptions, }) (Allocator.Error || Io.Cancelable || IoError)!Node.Index { @@ -605,22 +711,32 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { defer mf.large.appendSliceAssumeCapacity(&.{ opts.offset, 0 }); break :location .{ .large, .{ .large = .{ .index = mf.large.items.len } } }; }; - const free_ni: Node.Index, const free_node = free: switch (mf.free_ni) { - .none => .{ @fromBackingInt(@intCast(mf.nodes.items.len)), mf.nodes.addOneAssumeCapacity() }, - else => |free_ni| { - const free_node = free_ni.get(mf); - mf.free_ni = free_node.next; - break :free .{ free_ni, free_node }; - }, + + const free_ni: Node.Index, const free_node: *Node = if (mf.free_ni.unwrap()) |free_ni| free: { + const free_node = free_ni.get(mf); + mf.free_ni = free_node.next; + break :free .{ free_ni, free_node }; + } else .{ + @fromBackingInt(@intCast(mf.nodes.items.len)), + mf.nodes.addOneAssumeCapacity(), }; - switch (opts.prev) { - .none => opts.parent.get(mf).first = free_ni, - else => |prev_ni| try prev_ni.setNext(gpa, free_ni, mf), + + if (opts.prev.unwrap()) |prev_ni| { + try prev_ni.setNext(gpa, .wrap(free_ni), mf); + } else if (opts.parent.unwrap()) |parent_ni| { + parent_ni.get(mf).first = .wrap(free_ni); + } else { + assert(free_ni == .root); } - switch (opts.next) { - .none => opts.parent.get(mf).last = free_ni, - else => |next_ni| next_ni.get(mf).prev = free_ni, + + if (opts.next.unwrap()) |next_ni| { + next_ni.get(mf).prev = .wrap(free_ni); + } else if (opts.parent.unwrap()) |parent_ni| { + parent_ni.get(mf).last = .wrap(free_ni); + } else { + assert(free_ni == .root); } + free_node.* = .{ .parent = opts.parent, .prev = opts.prev, @@ -659,7 +775,7 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct { pub const AddNodeOptions = struct { size: u64 = 0, - alignment: std.mem.Alignment = .@"1", + alignment: Alignment = .@"1", fixed: bool = false, moved: bool = false, resized: bool = false, @@ -678,7 +794,7 @@ pub fn addOnlyChildNode( const parent = parent_ni.get(mf); assert(parent.first == .none and parent.last == .none); return mf.addNode(gpa, .{ - .parent = parent_ni, + .parent = .wrap(parent_ni), .add_node = opts, }) catch |err| switch (err) { error.OutOfMemory, @@ -700,7 +816,7 @@ pub fn addFirstChildNode( try mf.nodes.ensureUnusedCapacity(gpa, 1); const parent = parent_ni.get(mf); return mf.addNode(gpa, .{ - .parent = parent_ni, + .parent = .wrap(parent_ni), .next = parent.first, .add_node = opts, }) catch |err| switch (err) { @@ -723,14 +839,12 @@ pub fn addLastChildNode( try mf.nodes.ensureUnusedCapacity(gpa, 1); const parent = parent_ni.get(mf); return mf.addNode(gpa, .{ - .parent = parent_ni, + .parent = .wrap(parent_ni), .prev = parent.last, - .offset = offset: switch (parent.last) { - .none => 0, - else => |last_ni| { - const last_offset, const last_size = last_ni.location(mf).resolve(mf); - break :offset last_offset + last_size; - }, + .offset = offset: { + const last_ni = parent.last.unwrap() orelse break :offset 0; + const last_offset, const last_size = last_ni.location(mf).resolve(mf); + break :offset last_offset + last_size; }, .add_node = opts, }) catch |err| switch (err) { @@ -750,13 +864,12 @@ pub fn addNodeAfter( prev_ni: Node.Index, opts: AddNodeOptions, ) Error!Node.Index { - assert(prev_ni != .none); try mf.nodes.ensureUnusedCapacity(gpa, 1); const prev = prev_ni.get(mf); const prev_offset, const prev_size = prev.location().resolve(mf); return mf.addNode(gpa, .{ .parent = prev.parent, - .prev = prev_ni, + .prev = .wrap(prev_ni), .next = prev.next, .offset = prev_offset + prev_size, .add_node = opts, @@ -783,10 +896,10 @@ fn shrinkNode( const old_offset, _ = node.location().resolve(mf); // This would require unmapping first - assert(ni != Node.Index.root); + assert(ni != .root); - if (node.last != .none) { - const last = node.last.get(mf); + if (node.last.unwrap()) |last_ni| { + const last = last_ni.get(mf); const last_offset, const last_size = last.location().resolve(mf); assert(last_offset + last_size > size); } @@ -795,15 +908,16 @@ fn shrinkNode( try mf.updates.ensureUnusedCapacity(gpa, 4); ni.setLocationAssumeCapacity(mf, old_offset, size); - if (!shift_next or node.next == .none) return; + if (!shift_next) return; + const next_ni = node.next.unwrap() orelse return; - const next = node.next.get(mf); + const next = next_ni.get(mf); const old_next_offset, const next_size = next.location().resolve(mf); const padding = old_next_offset - (old_offset + size); const new_next_offset = next.flags.alignment.forward(@intCast(old_next_offset - padding)); if (next.flags.has_content and new_next_offset < old_next_offset) { - const old_file_offset = node.next.fileLocation(mf, false).offset; + const old_file_offset = next_ni.fileLocation(mf, false).offset; const new_file_offset = (old_file_offset - old_next_offset) + new_next_offset; @memmove( mf.memory_map.memory[@intCast(new_file_offset)..][0..@intCast(next_size)], @@ -812,7 +926,7 @@ fn shrinkNode( @memset(mf.memory_map.memory[@intCast(new_file_offset + next_size)..@intCast(old_file_offset + next_size)], 0); } - node.next.setLocationAssumeCapacity(mf, new_next_offset, next_size); + next_ni.setLocationAssumeCapacity(mf, new_next_offset, next_size); } fn resizeNode( @@ -828,7 +942,8 @@ fn resizeNode( const new_size = node.flags.alignment.forward(@intCast(requested_size)); // Resize the entire file - if (ni == Node.Index.root) { + const parent_ni = node.parent.unwrap() orelse { + assert(ni == .root); try mf.ensureCapacityForSetLocation(gpa); mf.memory_map.write(io) catch |err| switch (err) { error.WouldBlock => return error.Unexpected, // file was not opened as non-blocking @@ -839,15 +954,13 @@ fn resizeNode( try mf.ensureTotalCapacityInner(@intCast(new_size)); ni.setLocationAssumeCapacity(mf, old_offset, new_size); return; - } - const parent = node.parent.get(mf); + }; + const parent = parent_ni.get(mf); _, var old_parent_size = parent.location().resolve(mf); - const trailing_end = trailing_end: switch (node.next) { - .none => old_parent_size, - else => |next_ni| { - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }, + const trailing_end = trailing_end: { + const next_ni = node.next.unwrap() orelse break :trailing_end old_parent_size; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :trailing_end next_offset; }; assert(old_offset + old_size <= trailing_end); if (old_offset + new_size <= trailing_end) { @@ -877,7 +990,7 @@ fn resizeNode( else => |e| return e, }; // Ask the filesystem driver to insert extents into the file without copying any data - const last_offset, const last_size = parent.last.location(mf).resolve(mf); + const last_offset, const last_size = parent.last.unwrap().?.location(mf).resolve(mf); const last_end = last_offset + last_size; assert(last_end <= old_parent_size); _, const file_size = Node.Index.root.location(mf).resolve(mf); @@ -900,13 +1013,13 @@ fn resizeNode( enclosing.location().resolve(mf); const new_enclosing_size = old_enclosing_size + range_size; enclosing_ni.setLocationAssumeCapacity(mf, enclosing_offset, new_enclosing_size); - if (enclosing_ni == Node.Index.root) { + if (enclosing_ni == .root) { assert(enclosing_offset == 0); try mf.ensureTotalCapacityInner(@intCast(new_enclosing_size)); break; } - var after_ni = enclosing.next; - while (after_ni != .none) { + var after_oni = enclosing.next; + while (after_oni.unwrap()) |after_ni| { try mf.ensureCapacityForSetLocation(gpa); const after = after_ni.get(mf); const after_offset, const after_size = after.location().resolve(mf); @@ -915,9 +1028,9 @@ fn resizeNode( range_size + after_offset, after_size, ); - after_ni = after.next; + after_oni = after.next; } - enclosing_ni = enclosing.parent; + enclosing_ni = enclosing.parent.unwrap().?; } return; }, @@ -939,32 +1052,33 @@ fn resizeNode( if (node.next == .none) { // As this is the last node, we simply need more space in the parent const new_parent_size = old_offset + new_size; - try mf.resizeNode(gpa, node.parent, new_parent_size +| new_parent_size / growth_factor); + try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); try mf.ensureCapacityForSetLocation(gpa); ni.setLocationAssumeCapacity(mf, old_offset, new_size); return; } if (!node.flags.fixed) { // Make space at the end of the parent for this floating node - const last = parent.last.get(mf); + const last = parent.last.unwrap().?.get(mf); const last_offset, const last_size = last.location().resolve(mf); const new_offset = node.flags.alignment.forward(@intCast(last_offset + last_size)); const new_parent_size = new_offset + new_size; if (new_parent_size > old_parent_size) - try mf.resizeNode(gpa, node.parent, new_parent_size +| new_parent_size / growth_factor); + try mf.resizeNode(gpa, parent_ni, new_parent_size +| new_parent_size / growth_factor); try mf.ensureCapacityForSetLocation(gpa); - const next_ni = node.next; + const next_ni = node.next.unwrap().?; next_ni.get(mf).prev = node.prev; - switch (node.prev) { - .none => parent.first = next_ni, - else => |prev_ni| try prev_ni.setNext(gpa, next_ni, mf), + if (node.prev.unwrap()) |prev_ni| { + try prev_ni.setNext(gpa, .wrap(next_ni), mf); + } else { + parent.first = .wrap(next_ni); } - try parent.last.setNext(gpa, ni, mf); + try parent.last.unwrap().?.setNext(gpa, .wrap(ni), mf); node.prev = parent.last; try ni.setNext(gpa, .none, mf); - parent.last = ni; + parent.last = .wrap(ni); if (node.flags.has_content) { - const parent_file_offset = node.parent.fileLocation(mf, false).offset; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; try mf.moveRange( parent_file_offset + old_offset, parent_file_offset + new_offset, @@ -976,94 +1090,89 @@ fn resizeNode( } // Search for the first floating node following this fixed node var last_fixed_ni = ni; - var first_floating_ni = node.next; + var first_floating_oni = node.next; var shift = new_size - old_size; - var max_shift_align: std.mem.Alignment = .@"1"; + var max_shift_align: Alignment = .@"1"; var direction: enum { forward, reverse } = .forward; while (true) { - assert(last_fixed_ni != .none); const last_fixed = last_fixed_ni.get(mf); assert(last_fixed.flags.fixed); const old_last_fixed_offset, const last_fixed_size = last_fixed.location().resolve(mf); const new_last_fixed_offset = old_last_fixed_offset + shift; - make_space: switch (first_floating_ni) { - else => { - const first_floating = first_floating_ni.get(mf); - const old_first_floating_offset, const first_floating_size = - first_floating.location().resolve(mf); - assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); - if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) - break :make_space; - assert(direction == .forward); - max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); - if (first_floating.flags.fixed) { - shift = max_shift_align.forward(@intCast( - @max(shift, first_floating_size), - )); + if (first_floating_oni.unwrap()) |first_floating_ni| make_space: { + const first_floating = first_floating_ni.get(mf); + const old_first_floating_offset, const first_floating_size = + first_floating.location().resolve(mf); + assert(old_last_fixed_offset + last_fixed_size <= old_first_floating_offset); + if (new_last_fixed_offset + last_fixed_size <= old_first_floating_offset) + break :make_space; + assert(direction == .forward); + max_shift_align = max_shift_align.max(first_floating.flags.alignment.max(last_fixed.flags.alignment)); + if (first_floating.flags.fixed) { + shift = max_shift_align.forward(@intCast( + @max(shift, first_floating_size), + )); - // Not enough space, try the next node - last_fixed_ni = first_floating_ni; - first_floating_ni = first_floating.next; - continue; - } - // Move the found floating node to make space for preceding fixed nodes - const last = parent.last.get(mf); - const last_offset, const last_size = last.location().resolve(mf); - const new_first_floating_offset = max_shift_align.forward( - @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), - ); - const new_parent_size = new_first_floating_offset + first_floating_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - node.parent, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); - } - try mf.ensureCapacityForSetLocation(gpa); - if (parent.last != first_floating_ni) { - const old_last = parent.last; - first_floating.prev = old_last; - parent.last = first_floating_ni; - try old_last.setNext(gpa, first_floating_ni, mf); - try last_fixed_ni.setNext(gpa, first_floating.next, mf); - switch (first_floating.next) { - .none => {}, - else => |next_ni| next_ni.get(mf).prev = last_fixed_ni, - } - try first_floating_ni.setNext(gpa, .none, mf); - } - if (first_floating.flags.has_content) { - const parent_file_offset = - node.parent.fileLocation(mf, false).offset; - try mf.moveRange( - parent_file_offset + old_first_floating_offset, - parent_file_offset + new_first_floating_offset, - first_floating_size, - ); - } - first_floating_ni.setLocationAssumeCapacity( - mf, - new_first_floating_offset, - first_floating_size, - ); - // Continue the search after the just-moved floating node - first_floating_ni = last_fixed.next; + // Not enough space, try the next node + last_fixed_ni = first_floating_ni; + first_floating_oni = first_floating.next; continue; - }, - .none => { - assert(direction == .forward); - const new_parent_size = new_last_fixed_offset + last_fixed_size; - if (new_parent_size > old_parent_size) { - try mf.resizeNode( - gpa, - node.parent, - new_parent_size +| new_parent_size / growth_factor, - ); - _, old_parent_size = parent.location().resolve(mf); + } + // Move the found floating node to make space for preceding fixed nodes + const last = parent.last.unwrap().?.get(mf); + const last_offset, const last_size = last.location().resolve(mf); + const new_first_floating_offset = max_shift_align.forward( + @intCast(@max(new_last_fixed_offset + last_fixed_size, last_offset + last_size)), + ); + const new_parent_size = new_first_floating_offset + first_floating_size; + if (new_parent_size > old_parent_size) { + try mf.resizeNode( + gpa, + parent_ni, + new_parent_size +| new_parent_size / growth_factor, + ); + _, old_parent_size = parent.location().resolve(mf); + } + try mf.ensureCapacityForSetLocation(gpa); + if (parent.last.unwrap().? != first_floating_ni) { + const old_last = parent.last.unwrap().?; + first_floating.prev = .wrap(old_last); + parent.last = .wrap(first_floating_ni); + try old_last.setNext(gpa, .wrap(first_floating_ni), mf); + try last_fixed_ni.setNext(gpa, first_floating.next, mf); + if (first_floating.next.unwrap()) |next_ni| { + next_ni.get(mf).prev = .wrap(last_fixed_ni); } - }, + try first_floating_ni.setNext(gpa, .none, mf); + } + if (first_floating.flags.has_content) { + const parent_file_offset = + parent_ni.fileLocation(mf, false).offset; + try mf.moveRange( + parent_file_offset + old_first_floating_offset, + parent_file_offset + new_first_floating_offset, + first_floating_size, + ); + } + first_floating_ni.setLocationAssumeCapacity( + mf, + new_first_floating_offset, + first_floating_size, + ); + // Continue the search after the just-moved floating node + first_floating_oni = last_fixed.next; + continue; + } else { + assert(direction == .forward); + const new_parent_size = new_last_fixed_offset + last_fixed_size; + if (new_parent_size > old_parent_size) { + try mf.resizeNode( + gpa, + parent_ni, + new_parent_size +| new_parent_size / growth_factor, + ); + _, old_parent_size = parent.location().resolve(mf); + } } try mf.ensureCapacityForSetLocation(gpa); if (last_fixed_ni == ni) { @@ -1077,7 +1186,7 @@ fn resizeNode( } // Move a fixed node into trailing free space if (last_fixed.flags.has_content) { - const parent_file_offset = node.parent.fileLocation(mf, false).offset; + const parent_file_offset = parent_ni.fileLocation(mf, false).offset; try mf.moveRange( parent_file_offset + old_last_fixed_offset, parent_file_offset + new_last_fixed_offset, @@ -1086,8 +1195,8 @@ fn resizeNode( } last_fixed_ni.setLocationAssumeCapacity(mf, new_last_fixed_offset, last_fixed_size); // Retry the previous nodes now that there is enough space - first_floating_ni = last_fixed_ni; - last_fixed_ni = last_fixed.prev; + first_floating_oni = .wrap(last_fixed_ni); + last_fixed_ni = last_fixed.prev.unwrap().?; direction = .reverse; } } @@ -1096,7 +1205,7 @@ fn realignNode( mf: *MappedFile, gpa: Allocator, ni: Node.Index, - new_alignment: std.mem.Alignment, + new_alignment: Alignment, opts: Node.Index.RealignNodeOptions, ) (Allocator.Error || Io.Cancelable || IoError)!void { mf.nodes_lock.assertUnlocked(); @@ -1109,25 +1218,27 @@ fn realignNode( } const old_offset, const size = node.location().resolve(mf); - if (ni == Node.Index.root) return mf.resizeNode(gpa, ni, size); + const parent_ni = node.parent.unwrap() orelse { + assert(ni == .root); + return mf.resizeNode(gpa, ni, size); + }; const new_size = new_alignment.forward(@intCast(size)); if (new_alignment.check(@intCast(old_offset))) return mf.resizeNode(gpa, ni, new_size); - _, const parent_size = node.parent.location(mf).resolve(mf); - const trailing_end = trailing_end: switch (node.next) { - .none => parent_size, - else => |next_ni| { - const next_offset, _ = next_ni.location(mf).resolve(mf); - break :trailing_end next_offset; - }, + _, const parent_size = parent_ni.location(mf).resolve(mf); + const trailing_end = trailing_end: { + const next_ni = node.next.unwrap() orelse break :trailing_end parent_size; + const next_offset, _ = next_ni.location(mf).resolve(mf); + break :trailing_end next_offset; }; if (opts.try_backwards) { const backward_offset = new_alignment.backward(@intCast(old_offset)); - const prev_end = if (node.prev == .none) 0 else prev: { - const prev_offset, const prev_size = node.prev.location(mf).resolve(mf); - break :prev prev_offset + prev_size; + const prev_end = prev_end: { + const prev_ni = node.prev.unwrap() orelse break :prev_end 0; + const prev_offset, const prev_size = prev_ni.location(mf).resolve(mf); + break :prev_end prev_offset + prev_size; }; if (backward_offset >= prev_end) { @@ -1399,7 +1510,7 @@ fn verify(mf: *MappedFile) void { assert(root.parent == .none); assert(root.prev == .none); assert(root.next == .none); - mf.verifyNode(Node.Index.root); + mf.verifyNode(.root); } fn verifyNode(mf: *MappedFile, parent_ni: Node.Index) void { @@ -1517,7 +1628,7 @@ test { try testVerifyContent(&mf, d, 0xdd, d_init_size); } - const child_init: []const struct { std.mem.Alignment, usize } = &.{ + const child_init: []const struct { Alignment, usize } = &.{ .{ .@"16", 16 }, .{ .@"1", 1 }, .{ .@"1", 19 },