authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-08-30 23:12:25+02:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-08-30 23:12:25+02:00
log8a6f4ae1827a5b6b28b07f57852008ea001bf02e
treeadf8e4ba8463c0f7e18d1402d8c50248d17408bf
parent107eafe87ebc24b2e644857f445188ffdd7a4799
parenta6ab86acd92b81dcdfa3f8192690de7a26fa13d7

Merge pull request 'Elf2 and MappedFile enhancements' (#36677) from mlugg/elf2-yet-again into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/36677

2 files changed, 874 insertions(+), 448 deletions(-)

src/link/Elf2.zig+469-212
...@@ -24,6 +24,7 @@ base: link.File,...@@ -24,6 +24,7 @@ base: link.File,
24options: link.File.OpenOptions,24options: link.File.OpenOptions,
25mf: MappedFile,25mf: MappedFile,
26ni: Node.Known,26ni: Node.Known,
27archive: ?Archive,
27nodes: std.MultiArrayList(Node),28nodes: std.MultiArrayList(Node),
28/// Does not contain an item for `SHN_UNDEF`.29/// Does not contain an item for `SHN_UNDEF`.
29shdrs: std.ArrayList(Section),30shdrs: std.ArrayList(Section),
...@@ -200,19 +201,39 @@ input_prog_node: std.Progress.Node,...@@ -200,19 +201,39 @@ input_prog_node: std.Progress.Node,
200const Error = link.Error || error{MappedFileIo};201const Error = link.Error || error{MappedFileIo};
201202
202const Node = union(enum) {203const Node = union(enum) {
204 /// Only used when emitting a static library.
205 ///
206 /// Contains a header node which is an `.archive_header`.
207 ///
208 /// Contains the following footer nodes:
209 /// * One `.archive_input_member` for each external input in the archive
210 /// * One `.archive_elf_member_header` containing the `ar_hdr` for the ZCU
211 /// * One `.elf` containing the ZCU's actual ELF object
212 ///
213 /// Padding between the headers and footers is absorbed into the "//" member (whose actual
214 /// content is in the `.archive_header` node).
203 archive,215 archive,
204 /// This includes the archive magic and long file member.216 /// Only used when emitting a static library.
217 ///
218 /// Contains the archive magic (`ARMAG`), as well as the `ar_hdr` and content for the long file
219 /// name string table member ("//").
205 archive_header,220 archive_header,
206 /// This is a footer of the `.elf` node, and contains the next archive entry's file header.221 /// Only used when emitting a static library.
207 archive_elf_footer,222 ///
223 /// Contains the `ar_hdr` and content for one non-ZCU archive member (external link input). Also
224 /// includes the single byte '\n' padding at the end of this archive member, if necessary.
225 archive_input_member: InputIndex,
226 /// Only used when emitting a static library.
227 ///
228 /// Contains the `ar_hdr` for the `.elf` node.
229 archive_elf_member_header,
230
208 elf,231 elf,
209 ehdr,232 ehdr,
210 shdr,233 shdr,
211 segment: u32,234 segment: u32,
212 /// The section '.plt' may contain relocations via `elf.plt_first_symbol_reloc`.235 /// The section '.plt' may contain relocations via `elf.plt_first_symbol_reloc`.
213 section: Section.Index,236 section: Section.Index,
214 /// Only valid for static libraries, represents one non-zcu archive member.
215 input_member: InputIndex,
216 /// May contain relocations.237 /// May contain relocations.
217 input_section: InputSection.Index,238 input_section: InputSection.Index,
218 /// Value is the name of a global which has an entry in `elf.copied_globals`, so, a global for239 /// Value is the name of a global which has an entry in `elf.copied_globals`, so, a global for
...@@ -404,6 +425,15 @@ const InputSection = struct {...@@ -404,6 +425,15 @@ const InputSection = struct {
404 };425 };
405};426};
406427
428const Archive = struct {
429 ni: MappedFile.Node.Index,
430 header_ni: MappedFile.Node.Index,
431 elf_member_header_ni: MappedFile.Node.Index,
432
433 elf_member_too_big: bool,
434 strtab_member_too_big: bool,
435};
436
407const Section = struct {437const Section = struct {
408 /// The node corresponding to this section.438 /// The node corresponding to this section.
409 ni: MappedFile.Node.Index,439 ni: MappedFile.Node.Index,
...@@ -2954,13 +2984,13 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId {...@@ -2954,13 +2984,13 @@ pub fn symbolForAtom(elf: *Elf, atom: link.File.AtomId) link.File.SymbolId {
2954 const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) {2984 const lsi: Symbol.LocalIndex = switch (elf.getNode(Node.fromAtom(atom))) {
2955 .archive,2985 .archive,
2956 .archive_header,2986 .archive_header,
2957 .archive_elf_footer,2987 .archive_input_member,
2988 .archive_elf_member_header,
2958 .elf,2989 .elf,
2959 .ehdr,2990 .ehdr,
2960 .shdr,2991 .shdr,
2961 .segment,2992 .segment,
2962 .section,2993 .section,
2963 .input_member,
2964 .input_section,2994 .input_section,
2965 .copied_global,2995 .copied_global,
2966 => unreachable,2996 => unreachable,
...@@ -3364,6 +3394,7 @@ fn create(...@@ -3364,6 +3394,7 @@ fn create(
3364 .data_rel_ro = undefined,3394 .data_rel_ro = undefined,
3365 .tls = .none,3395 .tls = .none,
3366 },3396 },
3397 .archive = null,
3367 .nodes = .empty,3398 .nodes = .empty,
3368 .shdrs = .empty,3399 .shdrs = .empty,
3369 .phdrs = .empty,3400 .phdrs = .empty,
...@@ -3558,51 +3589,58 @@ fn initHeaders(...@@ -3558,51 +3589,58 @@ fn initHeaders(
3558 .EXEC, .DYN => {},3589 .EXEC, .DYN => {},
3559 }3590 }
3560 var phnum: u32 = 0;3591 var phnum: u32 = 0;
3561 break :ph .{ .{3592 break :ph .{
3562 .phdr = phndx: {3593 .{
3563 defer phnum += 1;3594 .phdr = phndx: {
3564 break :phndx phnum;3595 defer phnum += 1;
3565 },3596 break :phndx phnum;
3566 .interp = if (maybe_interp) |_| phndx: {3597 },
3567 defer phnum += 1;3598 .interp = if (maybe_interp) |_| phndx: {
3568 break :phndx phnum;3599 defer phnum += 1;
3569 } else undefined,3600 break :phndx phnum;
3570 .rodata = phndx: {3601 } else undefined,
3571 defer phnum += 1;3602 .rodata = phndx: {
3572 break :phndx phnum;3603 defer phnum += 1;
3573 },3604 break :phndx phnum;
3574 .text = phndx: {3605 },
3575 defer phnum += 1;3606 .text = phndx: {
3576 break :phndx phnum;3607 defer phnum += 1;
3577 },3608 break :phndx phnum;
3578 .data = phndx: {3609 },
3579 defer phnum += 1;3610 .plt = if (plt.got_plt == null) phndx: {
3580 break :phndx phnum;3611 defer phnum += 1;
3581 },3612 break :phndx phnum;
3582 .plt = if (plt.got_plt == null) phndx: {3613 } else undefined,
3583 defer phnum += 1;3614 // `data` must be assigned after all other loadable segments so that it has the greatest
3584 break :phndx phnum;3615 // phndx of any loadable segment. This is so that `targetSegmentLoadAddressRestrictions`
3585 } else undefined,3616 // can be obeyed (specifically, the `.data_last` restriction, needed on SPARC).
3586 .tls = if (comp.config.any_non_single_threaded) phndx: {3617 .data = phndx: {
3587 defer phnum += 1;3618 defer phnum += 1;
3588 break :phndx phnum;3619 break :phndx phnum;
3589 } else undefined,3620 },
3590 .dynamic = if (have_dynamic_section) phndx: {3621 .tls = if (comp.config.any_non_single_threaded) phndx: {
3591 defer phnum += 1;3622 defer phnum += 1;
3592 break :phndx phnum;3623 break :phndx phnum;
3593 } else undefined,3624 } else undefined,
3594 .relro = phndx: {3625 .dynamic = if (have_dynamic_section) phndx: {
3595 defer phnum += 1;3626 defer phnum += 1;
3596 break :phndx phnum;3627 break :phndx phnum;
3597 },3628 } else undefined,
3598 .gnu_stack = phndx: {3629 .relro = phndx: {
3599 defer phnum += 1;3630 defer phnum += 1;
3600 break :phndx phnum;3631 break :phndx phnum;
3632 },
3633 .gnu_stack = phndx: {
3634 defer phnum += 1;
3635 break :phndx phnum;
3636 },
3601 },3637 },
3602 }, phnum };3638 // (I don't actually want the trailing comma below, but a `zig fmt` bug forces it.)
3639 phnum,
3640 };
3603 };3641 };
36043642
3605 const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_footer3643 const expected_nodes_len = @as(usize, if (is_archive) 3 else 0) + // .archive, .archive_header, .archive_elf_member_header
3606 3 + // `.elf`, `.ehdr`, and `.shdr` nodes3644 3 + // `.elf`, `.ehdr`, and `.shdr` nodes
3607 (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node3645 (shnum - 1) + // -1 because the SHN_UNDEF shdr does not have a `.section` node
3608 (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node3646 (phnum -| 1); // -1 because the GNU_STACK phdr does not have a `.segment` node
...@@ -3619,11 +3657,14 @@ fn initHeaders(...@@ -3619,11 +3657,14 @@ fn initHeaders(
3619 const archive_ni: MappedFile.Node.Index = .root;3657 const archive_ni: MappedFile.Node.Index = .root;
36203658
3621 const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{3659 const archive_header_ni = try archive_ni.addOnlyHeaderChild(&elf.mf, gpa, .{
3622 .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr) * 2,3660 // We intentionally do not set `.alignment = .@"2"` here, because the string table data
3623 .alignment = .@"2",3661 // in this node does not need to have an aligned length. (This node's offset is aligned
3624 .next_moved = true,3662 // regardless by virtue of it being a header.)
3625 .bubbles_moved = false,3663 .size = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr),
3664 // The archive header uses 'next_moved' events to resize the "//" member, so that it
3665 // absorbs all padding between `archive_header_ni` and the actual object file members.
3626 .enable_next_moved = true,3666 .enable_next_moved = true,
3667 .next_moved = true,
3627 });3668 });
3628 elf.nodes.appendAssumeCapacity(.archive_header);3669 elf.nodes.appendAssumeCapacity(.archive_header);
3629 const archive_header_slice = archive_header_ni.slice(&elf.mf);3670 const archive_header_slice = archive_header_ni.slice(&elf.mf);
...@@ -3635,23 +3676,47 @@ fn initHeaders(...@@ -3635,23 +3676,47 @@ fn initHeaders(
3635 .ar_uid = @splat(' '),3676 .ar_uid = @splat(' '),
3636 .ar_gid = @splat(' '),3677 .ar_gid = @splat(' '),
3637 .ar_mode = @splat(' '),3678 .ar_mode = @splat(' '),
3638 .ar_size = @splat(' '),3679 .ar_size = undefined, // populated by `flushNextMoved` for `archive_header_ni`
3639 .ar_fmag = std.elf.ARFMAG.*,3680 .ar_fmag = std.elf.ARFMAG.*,
3640 };3681 };
36413682
3642 elf.ni.elf = try archive_ni.addFloatingChild(&elf.mf, gpa, .{3683 elf.ni.elf = try archive_ni.addOnlyFooterChild(&elf.mf, gpa, .{
3643 .alignment = node_block_align.max(.@"2"),3684 .alignment = node_block_align.max(.@"2"),
3644 .next_moved = true,
3645 .bubbles_moved = false,3685 .bubbles_moved = false,
3646 .enable_next_moved = true,3686 .resized = true, // ensure that this node's `ar_hdr.ar_size` is updated at least once
3647 });3687 });
3648 elf.nodes.appendAssumeCapacity(.elf);3688 elf.nodes.appendAssumeCapacity(.elf);
36493689
3650 _ = try elf.ni.elf.addOnlyFooterChild(&elf.mf, gpa, .{3690 const elf_ar_hdr_ni = try archive_ni.addFooterChildBefore(&elf.mf, gpa, .wrap(elf.ni.elf), .{
3651 .alignment = .@"2",3691 .alignment = .@"2",
3652 .size = @sizeOf(std.elf.ar_hdr),3692 .size = @sizeOf(std.elf.ar_hdr),
3653 });3693 });
3654 elf.nodes.appendAssumeCapacity(.archive_elf_footer);3694 elf.nodes.appendAssumeCapacity(.archive_elf_member_header);
3695
3696 // Must be populated before we call `populateArchiveMemberName` below.
3697 elf.archive = .{
3698 .ni = archive_ni,
3699 .header_ni = archive_header_ni,
3700 .elf_member_header_ni = elf_ar_hdr_ni,
3701
3702 .elf_member_too_big = false,
3703 .strtab_member_too_big = false,
3704 };
3705
3706 const elf_ar_hdr: *std.elf.ar_hdr = @ptrCast(elf_ar_hdr_ni.slice(&elf.mf));
3707 elf_ar_hdr.* = .{
3708 .ar_name = undefined, // populated below
3709 .ar_date = "0 ".*,
3710 .ar_uid = "0 ".*,
3711 .ar_gid = "0 ".*,
3712 .ar_mode = "644 ".*,
3713 .ar_size = undefined, // populated by `flushResized` for the `.elf` node
3714 .ar_fmag = std.elf.ARFMAG.*,
3715 };
3716 const zcu_member_name = try std.fmt.allocPrint(gpa, "{s}_zcu.o", .{comp.root_name});
3717 defer gpa.free(zcu_member_name);
3718 // After this call returns, `elf_ar_hdr` is invalidated.
3719 try elf.populateArchiveMemberName(elf_ar_hdr, zcu_member_name);
3655 } else {3720 } else {
3656 elf.ni.elf = .root;3721 elf.ni.elf = .root;
3657 elf.nodes.appendAssumeCapacity(.elf);3722 elf.nodes.appendAssumeCapacity(.elf);
...@@ -4092,33 +4157,44 @@ fn initHeaders(...@@ -4092,33 +4157,44 @@ fn initHeaders(
4092 .addralign = addr_align,4157 .addralign = addr_align,
4093 .entsize = @intCast(addr_align.toByteUnits()),4158 .entsize = @intCast(addr_align.toByteUnits()),
4094 });4159 });
4095 if (plt.got_plt) |got_plt| {4160 {
4096 const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data;4161 const init_plt_size = plt.entry_size * plt.header_entries;
4097 elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{4162 if (plt.got_plt) |got_plt| {
4098 .name = ".got.plt",4163 const got_plt_segment_ni = if (elf.options.z_now) elf.ni.data_rel_ro else elf.ni.data;
4099 .type = .PROGBITS,4164 elf.shndx.got_plt = try elf.addSection(got_plt_segment_ni, .{
4100 .flags = .{ .WRITE = true, .ALLOC = true },4165 .name = ".got.plt",
4101 .size = got_plt.header_entries * elf.targetPtrSize(),4166 .type = .PROGBITS,
4102 .addralign = addr_align,4167 .flags = .{ .WRITE = true, .ALLOC = true },
4103 .entsize = @intCast(addr_align.toByteUnits()),4168 .size = got_plt.header_entries * elf.targetPtrSize(),
4104 });4169 .addralign = addr_align,
4105 elf.shndx.plt = try elf.addSection(elf.ni.text, .{4170 .entsize = @intCast(addr_align.toByteUnits()),
4106 .name = ".plt",4171 });
4107 .type = .PROGBITS,4172 elf.shndx.plt = try elf.addSection(elf.ni.text, .{
4108 .flags = .{ .ALLOC = true, .EXECINSTR = true },4173 .name = ".plt",
4109 .size = plt.entry_size * plt.header_entries,4174 .type = .PROGBITS,
4110 .addralign = plt.@"align",4175 .flags = .{ .ALLOC = true, .EXECINSTR = true },
4111 .node_align = node_block_align,4176 .size = plt.@"align".forward(init_plt_size),
4112 });4177 .addralign = plt.@"align",
4113 } else {4178 .node_align = node_block_align,
4114 elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{4179 });
4115 .name = ".plt",4180 } else {
4116 .type = .PROGBITS,4181 elf.shndx.plt = try elf.addSection(elf.phdrs.items[phndx.plt].unwrap().?, .{
4117 .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true },4182 .name = ".plt",
4118 .size = plt.entry_size * plt.header_entries,4183 .type = .PROGBITS,
4119 .addralign = plt.@"align",4184 .flags = .{ .ALLOC = true, .WRITE = true, .EXECINSTR = true },
4120 .node_align = node_block_align,4185 .size = plt.@"align".forward(init_plt_size),
4121 });4186 .addralign = plt.@"align",
4187 .node_align = node_block_align,
4188 });
4189 }
4190 // And the award for most annoying PLT requirement goes to SPARC, which decided that the
4191 // whole table should have a greater alignment than the size of the individual entries,
4192 // hence this bullshit:
4193 if (plt.@"align".forward(init_plt_size) != init_plt_size) {
4194 switch (elf.shdrPtr(elf.shndx.plt)) {
4195 inline else => |shdr| elf.targetStore(&shdr.size, init_plt_size),
4196 }
4197 }
4122 }4198 }
4123 if (plt.plt_sec != null) elf.shndx.plt_sec = try elf.addSection(elf.ni.text, .{4199 if (plt.plt_sec != null) elf.shndx.plt_sec = try elf.addSection(elf.ni.text, .{
4124 .name = ".plt.sec",4200 .name = ".plt.sec",
...@@ -4358,6 +4434,12 @@ fn initHeaders(...@@ -4358,6 +4434,12 @@ fn initHeaders(
4358 assert(elf.targetLoad(&shdr.size) == elf.got.count() * @sizeOf(Addr));4434 assert(elf.targetLoad(&shdr.size) == elf.got.count() * @sizeOf(Addr));
4359 },4435 },
4360 }4436 }
4437 if (elf.shndx.dynamic != .UNDEF) {
4438 try elf.shndx.rela_dyn.relaEnsureAdditionalCapacity(elf, elf.got.count());
4439 }
4440 for (0..elf.got.count()) |got_index| {
4441 elf.updateGotEntry(got_index);
4442 }
43614443
4362 // Create any always-provided linker-defined symbols. The symbols marking the `INIT_ARRAY`/4444 // Create any always-provided linker-defined symbols. The symbols marking the `INIT_ARRAY`/
4363 // `FINI_ARRAY`/`PREINIT_ARRAY` sections are instead created by `createInitFiniArraySection`4445 // `FINI_ARRAY`/`PREINIT_ARRAY` sections are instead created by `createInitFiniArraySection`
...@@ -4539,6 +4621,22 @@ fn initHeaders(...@@ -4539,6 +4621,22 @@ fn initHeaders(
4539 break :str try elf.string(.dynstr, slice);4621 break :str try elf.string(.dynstr, slice);
4540 },4622 },
4541 };4623 };
4624
4625 if (@"type" != .REL) switch (elf.targetSegmentLoadAddressRestrictions()) {
4626 .none => {},
4627 .data_last => switch (elf.phdrSlice()) {
4628 inline else => |phdr| {
4629 // Ensure that the segment after `.data` (if any) is not a loadable segment.
4630 const next_phndx = phndx.data + 1;
4631 if (next_phndx < phdr.len) {
4632 switch (elf.targetLoad(&phdr[next_phndx].type)) {
4633 .NULL, .LOAD => unreachable, // data segment should be the last loadable segment
4634 else => {},
4635 }
4636 }
4637 },
4638 },
4639 };
4542}4640}
45434641
4544pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void {4642pub fn startProgress(elf: *Elf, prog_node: std.Progress.Node) void {
...@@ -4573,12 +4671,12 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index {...@@ -4573,12 +4671,12 @@ fn getNodeShndx(elf: *const Elf, ni: MappedFile.Node.Index) Section.Index {
4573 return switch (elf.getNode(ni)) {4671 return switch (elf.getNode(ni)) {
4574 .archive,4672 .archive,
4575 .archive_header,4673 .archive_header,
4576 .archive_elf_footer,4674 .archive_input_member,
4675 .archive_elf_member_header,
4577 .elf,4676 .elf,
4578 .ehdr,4677 .ehdr,
4579 .shdr,4678 .shdr,
4580 .segment,4679 .segment,
4581 .input_member,
4582 => unreachable,4680 => unreachable,
4583 .section => |shndx| shndx,4681 .section => |shndx| shndx,
4584 .input_section,4682 .input_section,
...@@ -4594,12 +4692,12 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {...@@ -4594,12 +4692,12 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4594 return switch (elf.getNode(ni)) {4692 return switch (elf.getNode(ni)) {
4595 .archive,4693 .archive,
4596 .archive_header,4694 .archive_header,
4597 .archive_elf_footer,4695 .archive_input_member,
4696 .archive_elf_member_header,
4598 .elf,4697 .elf,
4599 .ehdr,4698 .ehdr,
4600 .shdr,4699 .shdr,
4601 .segment,4700 .segment,
4602 .input_member,
4603 .copied_global,4701 .copied_global,
4604 => unreachable,4702 => unreachable,
4605 .section => |shndx| shndx.vaddr(elf),4703 .section => |shndx| shndx.vaddr(elf),
...@@ -4613,14 +4711,18 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {...@@ -4613,14 +4711,18 @@ fn getNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4613}4711}
4614fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {4712fn computeNodeVAddr(elf: *Elf, ni: MappedFile.Node.Index) u64 {
4615 const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) {4713 const parent_vaddr = switch (elf.getNode(ni.parent(&elf.mf).unwrap().?)) {
4616 .archive, .archive_header, .archive_elf_footer => unreachable,4714 .archive,
4715 .archive_header,
4716 .archive_input_member,
4717 .archive_elf_member_header,
4718 => unreachable,
4617 .elf => return 0,4719 .elf => return 0,
4618 .ehdr, .shdr => unreachable,4720 .ehdr, .shdr => unreachable,
4619 .segment => |phndx| switch (elf.phdrSlice()) {4721 .segment => |phndx| switch (elf.phdrSlice()) {
4620 inline else => |phdr| elf.targetLoad(&phdr[phndx].vaddr),4722 inline else => |phdr| elf.targetLoad(&phdr[phndx].vaddr),
4621 },4723 },
4622 .section => |shndx| if (shndx == elf.shndx.tdata) 0 else shndx.vaddr(elf),4724 .section => |shndx| if (shndx == elf.shndx.tdata) 0 else shndx.vaddr(elf),
4623 .input_member, .input_section, .copied_global => unreachable,4725 .input_section, .copied_global => unreachable,
4624 inline .nav, .uav, .lazy_code, .lazy_const_data => |i| Symbol.Id.local(i.symbol(elf)).value(elf),4726 inline .nav, .uav, .lazy_code, .lazy_const_data => |i| Symbol.Id.local(i.symbol(elf)).value(elf),
4625 };4727 };
4626 const offset, _ = ni.location(&elf.mf).resolve(&elf.mf);4728 const offset, _ = ni.location(&elf.mf).resolve(&elf.mf);
...@@ -4639,12 +4741,12 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void {...@@ -4639,12 +4741,12 @@ fn resetNodeRelocs(elf: *Elf, ni: MappedFile.Node.Index) void {
4639 const symbol_relocs: *SymbolReloc.Index, const got_relocs: ?*GotReloc.Index = switch (elf.getNode(ni)) {4741 const symbol_relocs: *SymbolReloc.Index, const got_relocs: ?*GotReloc.Index = switch (elf.getNode(ni)) {
4640 .archive,4742 .archive,
4641 .archive_header,4743 .archive_header,
4642 .archive_elf_footer,4744 .archive_input_member,
4745 .archive_elf_member_header,
4643 .elf,4746 .elf,
4644 .ehdr,4747 .ehdr,
4645 .shdr,4748 .shdr,
4646 .segment,4749 .segment,
4647 .input_member,
4648 .copied_global,4750 .copied_global,
4649 => unreachable, // cannot contain relocs4751 => unreachable, // cannot contain relocs
4650 .section => unreachable, // cannot contain relocs (.plt and .dynamic unsupported)4752 .section => unreachable, // cannot contain relocs (.plt and .dynamic unsupported)
...@@ -4705,7 +4807,12 @@ fn flushMovedNodeRelocs(...@@ -4705,7 +4807,12 @@ fn flushMovedNodeRelocs(
4705 // changed, so update the `offset` field of the `ElfN.Rela` entry.4807 // changed, so update the `offset` field of the `ElfN.Rela` entry.
4706 reloc.relaSection(elf).relaSetOffset(elf, rela_index, node_vaddr + reloc.offset);4808 reloc.relaSection(elf).relaSetOffset(elf, rela_index, node_vaddr + reloc.offset);
4707 }4809 }
4708 reloc.apply(elf);4810 // This is not just the inverse of the above condition, because if `reloc` is relative
4811 // to the base of this DSO, then `rela_index` is an `R_*_RELATIVE` relocation, but we
4812 // still need to call `SymbolReloc.apply` to update that relocation's addend.
4813 if (elf.ehdrType() != .REL) {
4814 reloc.apply(elf);
4815 }
4709 }4816 }
4710 }4817 }
47114818
...@@ -4888,7 +4995,31 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo {...@@ -4888,7 +4995,31 @@ fn targetDynsymHashInfo(elf: *const Elf) DynsymHashInfo {
4888 // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`.4995 // TODO: Alpha and S390x will need to use either `."@4"` or `.@"8"` depending on `elf.identClass()`.
4889 };4996 };
4890}4997}
4891pub fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {4998/// Specifies any restrictions the current target has regarding how segments are ordered in the
4999/// virtual address space. Most targets do not have any such restrictions.
5000fn targetSegmentLoadAddressRestrictions(elf: *const Elf) enum {
5001 none,
5002 /// The "mutable data" segment must be the last loadable segment in the virtual address space.
5003 data_last,
5004} {
5005 return switch (elf.ehdrMachine()) {
5006 .AARCH64,
5007 .PPC64,
5008 .RISCV,
5009 .X86_64,
5010 .LOONGARCH,
5011 => .none,
5012
5013 // SPARC uses `R_SPARC_PC{10,22}` relocations to construct pointers to the GOT, but these
5014 // relocations write an *unsigned* PC-relative offset. This cannot even be worked around by
5015 // using a larger code model, because the crt `_start` assembly always uses these specific
5016 // relocations. Therefore, to avoid relocation errors, all code must appear before the GOT
5017 // in the virtual address space. The easiest way for us to do that is to ensure that the
5018 // "mutable data" segment, containing the GOT, is the last segment in the address space.
5019 .SPARCV9 => .data_last,
5020 };
5021}
5022fn targetLoad(elf: *const Elf, ptr: anytype) @typeInfo(@TypeOf(ptr)).pointer.child {
4892 const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer;5023 const pointer_ty = @typeInfo(@TypeOf(ptr)).pointer;
4893 const Child = pointer_ty.child;5024 const Child = pointer_ty.child;
4894 const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child);5025 const alignment = pointer_ty.attrs.@"align" orelse @alignOf(Child);
...@@ -4971,16 +5102,6 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr {...@@ -4971,16 +5102,6 @@ fn shdrPtr(elf: *Elf, shndx: Section.Index) ShdrPtr {
4971 }5102 }
4972}5103}
49735104
4974fn arHdrPtr(elf: *Elf, ni: MappedFile.Node.Index) *align(2) std.elf.ar_hdr {
4975 assert(elf.ni.elf != .root);
4976 const file_offset = ni.fileLocation(&elf.mf, false).offset;
4977 return @ptrCast(@alignCast(elf.mf.memory_map.memory[@intCast(switch (elf.getNode(ni)) {
4978 else => unreachable,
4979 .archive_header => file_offset + std.elf.ARMAG.len,
4980 .elf, .input_member => file_offset - @sizeOf(std.elf.ar_hdr),
4981 })..][0..@sizeOf(std.elf.ar_hdr)]));
4982}
4983
4984const SymPtr = union(std.elf.CLASS) {5105const SymPtr = union(std.elf.CLASS) {
4985 NONE: noreturn,5106 NONE: noreturn,
4986 @"32": *std.elf.Elf32.Sym,5107 @"32": *std.elf.Elf32.Sym,
...@@ -5480,19 +5601,61 @@ fn loadObject(...@@ -5480,19 +5601,61 @@ fn loadObject(
5480 .member = if (member) |m| try gpa.dupe(u8, m) else null,5601 .member = if (member) |m| try gpa.dupe(u8, m) else null,
5481 .extra = undefined,5602 .extra = undefined,
5482 };5603 };
5483 if (elf.ni.elf != .root) {5604 if (elf.archive) |*archive| {
5484 const archive_ni: MappedFile.Node.Index = .root;5605 // We're creating a static library, so just add this input as an archive member.
5606 assert(member == null); // don't try to put static library members into other static libraries
5607
5608 const first_member_oni = archive.header_ni.next(&elf.mf);
5609
5610 if (first_member_oni.unwrap()) |first_member_ni| switch (elf.getNode(first_member_ni)) {
5611 .archive_input_member, .archive_elf_member_header => {},
5612 .elf => unreachable, // always preceded by `.archive_elf_member_header`
5613 else => unreachable, // never a child of `.archive`
5614 };
5615
5485 try elf.nodes.ensureUnusedCapacity(gpa, 1);5616 try elf.nodes.ensureUnusedCapacity(gpa, 1);
5486 input.extra = .{ .node = try archive_ni.addFloatingChild(&elf.mf, gpa, .{5617 const new_member_ni = try archive.ni.addFooterChildBefore(&elf.mf, gpa, first_member_oni, .{
5487 .size = Alignment.@"2".forward(fl.size + @sizeOf(std.elf.ar_hdr)),5618 .size = Alignment.@"2".forward(@sizeOf(std.elf.ar_hdr) + fl.size),
5488 .alignment = .@"2",5619 .alignment = .@"2",
5489 .next_moved = true,5620 });
5490 .bubbles_moved = false,5621 elf.nodes.appendAssumeCapacity(.{ .archive_input_member = input_index });
5491 .enable_next_moved = true,5622 input.extra = .{ .node = new_member_ni };
5492 }) };
5493 elf.nodes.appendAssumeCapacity(.{ .input_member = input_index });
5494 elf.input_prog_node.increaseEstimatedTotalItems(1);5623 elf.input_prog_node.increaseEstimatedTotalItems(1);
54955624
5625 // The contents of the input will be written to the file by an idle task (`flushInput`), but
5626 // we do need to write the input's archive member header (`ar_hdr`) now, for two reasons:
5627 //
5628 // * If the input file has a long name, we need to add it to the archive member name string
5629 // table, which must happen deterministically (i.e. not in an idle task).
5630 //
5631 // * `flushInput` needs to know the actual file size (before padding to the alignment).
5632 const member_ar_hdr: *std.elf.ar_hdr = @ptrCast(
5633 new_member_ni.slice(&elf.mf)[0..@sizeOf(std.elf.ar_hdr)],
5634 );
5635 member_ar_hdr.* = .{
5636 .ar_name = undefined, // populated below
5637 .ar_date = "0 ".*,
5638 .ar_uid = "0 ".*,
5639 .ar_gid = "0 ".*,
5640 .ar_mode = "644 ".*,
5641 .ar_size = undefined, // populated below
5642 .ar_fmag = std.elf.ARFMAG.*,
5643 };
5644
5645 if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{fl.size})) |size_str| {
5646 @memset(member_ar_hdr.ar_size[size_str.len..], ' ');
5647 } else |err| switch (err) {
5648 error.NoSpaceLeft => return diags.failParse(
5649 path,
5650 "file size of {Bi} exceeds maximum size of archive member",
5651 .{fl.size},
5652 ),
5653 }
5654
5655 const member_name = std.fs.path.basename(path.sub_path);
5656 // After this call returns, `member_ar_hdr` is invalidated.
5657 try elf.populateArchiveMemberName(member_ar_hdr, member_name);
5658
5496 // Since we are not emitting the archive symbol table (yet?) we do not need to parse5659 // Since we are not emitting the archive symbol table (yet?) we do not need to parse
5497 // the symbols in this input.5660 // the symbols in this input.
5498 return;5661 return;
...@@ -5905,6 +6068,46 @@ fn loadObject(...@@ -5905,6 +6068,46 @@ fn loadObject(
5905 },6068 },
5906 }6069 }
5907}6070}
6071/// This function may resize the archive header, so therefore invalidates `member_ar_hdr`.
6072fn populateArchiveMemberName(elf: *Elf, member_ar_hdr: *std.elf.ar_hdr, member_name: []const u8) Error!void {
6073 if (std.mem.print(&member_ar_hdr.ar_name, "{s}/", .{member_name})) |name_str| {
6074 @memset(member_ar_hdr.ar_name[name_str.len..], ' ');
6075 return;
6076 } else |err| switch (err) {
6077 error.NoSpaceLeft => {}, // handled below
6078 }
6079
6080 const gpa = elf.base.comp.gpa;
6081 const archive_header_ni = elf.archive.?.header_ni;
6082
6083 // The member's name is too big to put directly in the `ar_name` field, so it needs to go in the
6084 // "long name" string table instead (in the special member named "//").
6085
6086 _, const old_archive_header_size = archive_header_ni.location(&elf.mf).resolve(&elf.mf);
6087
6088 // We're going to add a new string at the end of the table. Update `member_ar_hdr` first,
6089 // because resizing the string table will invalidate it.
6090 const string_table_offset = old_archive_header_size - (std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr));
6091 if (std.mem.print(&member_ar_hdr.ar_name, "/{d}", .{string_table_offset})) |name_str| {
6092 @memset(member_ar_hdr.ar_name[name_str.len..], ' ');
6093 } else |inner_err| switch (inner_err) {
6094 error.NoSpaceLeft => {
6095 // The string table offset is itself too big to represent. This means the string table's
6096 // *size* is definitely too big to represent (we only get 10 bytes for that whereas we
6097 // get 16 here!), so as long as we still add the string, we're guaranteed to get a link
6098 // error for that reason. Therefore, we can just ignore this error and carry on.
6099 },
6100 }
6101
6102 // We set the size of the archive header node exactly, because we want padding bytes to go into
6103 // the root `.archive` node. That way, those bytes could still be used to grow the string table
6104 // if necessary, but they could also be used for new archive members.
6105 try archive_header_ni.resizeLeaf(&elf.mf, gpa, old_archive_header_size + member_name.len + 2);
6106
6107 const dest_slice = archive_header_ni.slice(&elf.mf)[@intCast(old_archive_header_size)..];
6108 @memcpy(dest_slice[0 .. dest_slice.len - 2], member_name);
6109 @memcpy(dest_slice[dest_slice.len - 2 ..], "/\n"); // yes, the terminator is weird
6110}
5908fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadParseInputError || error{BadMagic})!void {6111fn loadDso(elf: *Elf, path: std.Build.Cache.Path, fr: *Io.File.Reader) (LoadParseInputError || error{BadMagic})!void {
5909 const comp = elf.base.comp;6112 const comp = elf.base.comp;
5910 const gpa = comp.gpa;6113 const gpa = comp.gpa;
...@@ -7072,12 +7275,12 @@ fn addGotRelocAssumeCapacity(...@@ -7072,12 +7275,12 @@ fn addGotRelocAssumeCapacity(
7072 switch (elf.getNode(node)) {7275 switch (elf.getNode(node)) {
7073 .archive,7276 .archive,
7074 .archive_header,7277 .archive_header,
7075 .archive_elf_footer,7278 .archive_input_member,
7279 .archive_elf_member_header,
7076 .elf,7280 .elf,
7077 .ehdr,7281 .ehdr,
7078 .shdr,7282 .shdr,
7079 .segment,7283 .segment,
7080 .input_member,
7081 .copied_global,7284 .copied_global,
7082 => unreachable, // cannot contain relocs,7285 => unreachable, // cannot contain relocs,
7083 .section,7286 .section,
...@@ -7129,6 +7332,7 @@ fn addGotRelocAssumeCapacity(...@@ -7129,6 +7332,7 @@ fn addGotRelocAssumeCapacity(
7129 });7332 });
7130}7333}
7131fn updateGotEntry(elf: *Elf, got_index: usize) void {7334fn updateGotEntry(elf: *Elf, got_index: usize) void {
7335 assert(elf.ehdrType() != .REL);
7132 const entry_value: union(enum) {7336 const entry_value: union(enum) {
7133 unsigned: u64,7337 unsigned: u64,
7134 signed: i64,7338 signed: i64,
...@@ -7514,6 +7718,17 @@ fn flushInner(...@@ -7514,6 +7718,17 @@ fn flushInner(
7514 diags.addError("failed to apply {d} relocations: misaligned value", .{elf.misaligned_reloc_count});7718 diags.addError("failed to apply {d} relocations: misaligned value", .{elf.misaligned_reloc_count});
7515 }7719 }
75167720
7721 if (elf.archive) |*archive| {
7722 if (archive.elf_member_too_big) diags.addError(
7723 "file size of {Bi} exceeds maximum size of archive member",
7724 .{elf.ni.elf.location(&elf.mf).resolve(&elf.mf)[1]},
7725 );
7726 if (archive.strtab_member_too_big) diags.addError(
7727 "archive file name string table exceeds maximum size",
7728 .{},
7729 );
7730 }
7731
7517 elf.flushDynamic();7732 elf.flushDynamic();
75187733
7519 const entry_addr: u64 = entry: {7734 const entry_addr: u64 = entry: {
...@@ -7545,6 +7760,9 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool {...@@ -7545,6 +7760,9 @@ pub fn idle(elf: *Elf, tid: Zcu.PerThread.Id) link.Error!bool {
7545 const comp = elf.base.comp;7760 const comp = elf.base.comp;
7546 const diags = &comp.link_diags;7761 const diags = &comp.link_diags;
75477762
7763 elf.mf.nodes_lock.lock();
7764 defer elf.mf.nodes_lock.unlock();
7765
7548 assert(elf.pending_uavs.items.len == 0);7766 assert(elf.pending_uavs.items.len == 0);
7549 for (&elf.lazy.values) |*lazy| {7767 for (&elf.lazy.values) |*lazy| {
7550 assert(lazy.pending_index == lazy.map.count());7768 assert(lazy.pending_index == lazy.map.count());
...@@ -7698,7 +7916,7 @@ fn idleProgNode(...@@ -7698,7 +7916,7 @@ fn idleProgNode(
7698 return prog_node.start(name: switch (node) {7916 return prog_node.start(name: switch (node) {
7699 else => |tag| @tagName(tag),7917 else => |tag| @tagName(tag),
7700 .section => |shndx| shndx.name(elf).slice(elf),7918 .section => |shndx| shndx.name(elf).slice(elf),
7701 .input_member => |ii| std.fmt.bufPrint(&name, "{f}{f}", .{7919 .archive_input_member => |ii| std.fmt.bufPrint(&name, "{f}{f}", .{
7702 ii.path(elf).fmtEscapeString(),7920 ii.path(elf).fmtEscapeString(),
7703 fmtMemberString(ii.member(elf)),7921 fmtMemberString(ii.member(elf)),
7704 }) catch &name,7922 }) catch &name,
...@@ -7825,7 +8043,6 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void {...@@ -7825,7 +8043,6 @@ fn genLazy(elf: *Elf, pt: Zcu.PerThread, lmr: Node.LazyMapRef) Error!void {
7825fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void {8043fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void {
7826 const comp = elf.base.comp;8044 const comp = elf.base.comp;
7827 const io = comp.io;8045 const io = comp.io;
7828 const gpa = comp.gpa;
7829 const diags = &comp.link_diags;8046 const diags = &comp.link_diags;
7830 const path = ii.path(elf);8047 const path = ii.path(elf);
7831 const file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| switch (err) {8048 const file = path.root_dir.handle.openFile(io, path.sub_path, .{}) catch |err| switch (err) {
...@@ -7833,23 +8050,40 @@ fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void {...@@ -7833,23 +8050,40 @@ fn flushInput(elf: *Elf, ii: Node.InputIndex) Error!void {
7833 else => |e| return diags.fail("failed to open input file \"{f}\": {t}", .{ path.fmtEscapeString(), e }),8050 else => |e| return diags.fail("failed to open input file \"{f}\": {t}", .{ path.fmtEscapeString(), e }),
7834 };8051 };
7835 defer file.close(io);8052 defer file.close(io);
8053
8054 const slice = ii.node(elf).slice(&elf.mf);
8055
8056 const member_ar_hdr: *const std.elf.ar_hdr = @ptrCast(slice[0..@sizeOf(std.elf.ar_hdr)]);
8057 const input_size: u32 = member_ar_hdr.size() catch |err| switch (err) {
8058 // We wrote the `ar_hdr` ourselves (in `loadObject`), so it is definitely valid.
8059 error.Overflow, error.InvalidCharacter => unreachable,
8060 };
8061
8062 switch (slice.len - @sizeOf(std.elf.ar_hdr) - input_size) {
8063 0 => {},
8064 1 => {
8065 // Alignment added one padding byte, which the format requires to have value '\n'.
8066 slice[slice.len - 1] = '\n';
8067 },
8068 else => unreachable, // node size should agree with the value we wrote into `ar_hdr.ar_size`
8069 }
8070
7836 var fr = file.reader(io, &.{});8071 var fr = file.reader(io, &.{});
7837 var nw: MappedFile.Node.Writer = undefined;8072 var w: Io.Writer = .fixed(slice[@sizeOf(std.elf.ar_hdr)..]);
7838 ii.node(elf).writer(&elf.mf, gpa, &nw);8073 const n_bytes_read = w.sendFileAll(&fr, .limited(input_size)) catch |err| switch (err) {
7839 defer nw.deinit();
7840 const size = nw.interface.buffer.len - @sizeOf(std.elf.ar_hdr);
7841 const n_bytes = nw.interface.sendFileAll(&fr, .limited(size)) catch |err| switch (err) {
7842 error.ReadFailed => return diags.fail("failed to read input \"{f}{f}\": {t}", .{8074 error.ReadFailed => return diags.fail("failed to read input \"{f}{f}\": {t}", .{
7843 path.fmtEscapeString(),8075 path.fmtEscapeString(),
7844 fmtMemberString(ii.member(elf)),8076 fmtMemberString(ii.member(elf)),
7845 fr.err orelse (fr.seek_err orelse fr.size_err.?),8077 fr.err orelse (fr.seek_err orelse fr.size_err.?),
7846 }),8078 }),
7847 error.WriteFailed => return nw.err.?,8079 error.WriteFailed => unreachable, // `.limited(input_size)` prevents us writing too many bytes
7848 };8080 };
7849 if (n_bytes + 1 < size) return diags.fail("failed to read input \"{f}{f}\": unexpected eof", .{8081 if (n_bytes_read != input_size) {
7850 path.fmtEscapeString(),8082 return diags.fail("failed to load input \"{f}{f}\": file truncated during compilation", .{
7851 fmtMemberString(ii.member(elf)),8083 path.fmtEscapeString(),
7852 });8084 fmtMemberString(ii.member(elf)),
8085 });
8086 }
7853}8087}
78548088
7855fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void {8089fn flushInputSection(elf: *Elf, isi: InputSection.Index) Error!void {
...@@ -7931,12 +8165,18 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void...@@ -7931,12 +8165,18 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
7931 const trace = tracy.trace(@src());8165 const trace = tracy.trace(@src());
7932 defer trace.end();8166 defer trace.end();
79338167
7934 elf.mf.nodes_lock.lock();
7935 defer elf.mf.nodes_lock.unlock();
7936
7937 switch (elf.getNode(ni)) {8168 switch (elf.getNode(ni)) {
7938 .archive, .archive_header => unreachable,8169 .archive => unreachable,
7939 .archive_elf_footer, .elf => {},8170 .archive_header => unreachable,
8171
8172 .archive_input_member,
8173 .archive_elf_member_header,
8174 .elf,
8175 => {
8176 assert(elf.archive != null);
8177 return;
8178 },
8179
7940 .ehdr, .shdr => elf.flushElfOffset(ni),8180 .ehdr, .shdr => elf.flushElfOffset(ni),
7941 .segment => |phndx| {8181 .segment => |phndx| {
7942 elf.flushElfOffset(ni);8182 elf.flushElfOffset(ni);
...@@ -8014,7 +8254,6 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void...@@ -8014,7 +8254,6 @@ fn flushMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!void
8014 elf.flushMovedPltSection(.plt_sec, old_addr, addr);8254 elf.flushMovedPltSection(.plt_sec, old_addr, addr);
8015 }8255 }
8016 },8256 },
8017 .input_member => {},
8018 .input_section => |isi| {8257 .input_section => |isi| {
8019 const old_section_addr = isi.ptr(elf).vaddr;8258 const old_section_addr = isi.ptr(elf).vaddr;
8020 const new_section_addr = elf.computeNodeVAddr(ni);8259 const new_section_addr = elf.computeNodeVAddr(ni);
...@@ -8114,6 +8353,18 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro...@@ -8114,6 +8353,18 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
8114 const page_align = elf.targetPageAlign();8353 const page_align = elf.targetPageAlign();
8115 const node_align = segment_ni.alignment(&elf.mf);8354 const node_align = segment_ni.alignment(&elf.mf);
8116 const ph_align = page_align.max(node_align);8355 const ph_align = page_align.max(node_align);
8356
8357 // If we determine that the segment's virtual address needs to move, then it's a good idea to
8358 // make it less likely that it needs to move *again* in the future, because it is expensive to
8359 // change a segment's load address (a lot of re-flushing is necessary). To do that, we reserve
8360 // more virtual address space than we need (multiplying the actual size by this value). That
8361 // way, there will usually be padding between segments which they can grow into.
8362 //
8363 // TODO: we might want to decrease this multiplier, or even omit it entirely, in cases where
8364 // virtual address space is constrained. For instance, 32-bit targets, or targets where short
8365 // PC-relative relocations between segments are common.
8366 const reserve_size_multiplier = 4;
8367
8117 switch (elf.phdrSlice()) {8368 switch (elf.phdrSlice()) {
8118 inline else => |phdr| {8369 inline else => |phdr| {
8119 const offset = elf.targetLoad(&phdr[orig_phndx].offset);8370 const offset = elf.targetLoad(&phdr[orig_phndx].offset);
...@@ -8172,15 +8423,46 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro...@@ -8172,15 +8423,46 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
8172 // backwards to the start of the page.8423 // backwards to the start of the page.
8173 const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits());8424 const next_page_vaddr = std.mem.alignBackward(u64, next_vaddr, page_align.toByteUnits());
81748425
8175 // If we're at the same vaddr we started at, then all we're worried about is the8426 // Check if the segment fits here. We apply `reserve_size_multiplier`, but only if
8176 // segment fitting here. However, if we've already changed our virtual address, then8427 // the segment is already known to be moving---making it easier to grow in-place is
8177 // we might as well try to reserve a bit *more* virtual address space while we're at8428 // the whole point of the multiplier!
8178 // it, because changing virtual address is quite disruptive (we need to re-flush a8429 {
8179 // lot of stuff!) and giving ourselves more space will make it less likely to happen8430 const target_size = if (vaddr == orig_vaddr) size else size * reserve_size_multiplier;
8180 // again.8431 if (vaddr + target_size <= next_page_vaddr) {
8181 const target_size = if (vaddr == orig_vaddr) size else size * 4;8432 break; // hooray, we fit here!
8182 if (vaddr + target_size <= next_page_vaddr) {8433 }
8183 break; // hooray, we fit here!8434 }
8435
8436 const next_ni = elf.phdrs.items[next_phndx].unwrap().?;
8437
8438 // This segment don't fit here, but before deciding how to proceed, we need to
8439 // consider any target-specific restrictions we are subject to.
8440 switch (elf.targetSegmentLoadAddressRestrictions()) {
8441 .none => {},
8442 .data_last => if (next_ni == elf.ni.data) {
8443 // We can't leapfrog over the data segment. Instead, that segment just needs
8444 // to be shifted forwards to make space for us, and we'll then `break` with
8445 // our current vaddr.
8446
8447 if (next_phndx + 1 < phdr.len) switch (elf.targetLoad(&phdr[next_phndx + 1].type)) {
8448 .NULL, .LOAD => unreachable, // data segment should be the last loadable segment
8449 else => {},
8450 };
8451
8452 const free_vaddr = vaddr + size * reserve_size_multiplier;
8453
8454 const next_align = page_align.max(next_ni.alignment(&elf.mf));
8455 const next_offset = elf.targetLoad(&next_ph.offset);
8456 const next_new_vaddr = next_align.forward(free_vaddr) + next_offset % next_align.toByteUnits();
8457
8458 // This logic for updating the data segment's vaddr is identical to how we
8459 // will update the vaddr of `phndx` when we break from the loop.
8460 elf.targetStore(&next_ph.vaddr, @intCast(next_new_vaddr));
8461 elf.targetStore(&next_ph.paddr, @intCast(next_new_vaddr));
8462 try next_ni.childrenMoved(elf.base.comp.gpa, &elf.mf);
8463
8464 break;
8465 },
8184 }8466 }
81858467
8186 // We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But8468 // We don't fit here, so shift ourselves forward (i.e. swap with `next_phndx`). But
...@@ -8192,8 +8474,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro...@@ -8192,8 +8474,7 @@ fn allocateSegmentLoadAddress(elf: *Elf, orig_phndx: u32) std.mem.Allocator.Erro
81928474
8193 // Now just swap the phdrs and update our `phndx`.8475 // Now just swap the phdrs and update our `phndx`.
8194 std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph);8476 std.mem.swap(@TypeOf(next_ph.*), &phdr[phndx], next_ph);
8195 const next_ni = elf.phdrs.items[next_phndx];8477 elf.phdrs.items[phndx] = .wrap(next_ni);
8196 elf.phdrs.items[phndx] = next_ni;
8197 elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx };8478 elf.nodes.items(.data)[@backingInt(next_ni)] = .{ .segment = phndx };
8198 elf.phdrs.items[next_phndx] = .wrap(segment_ni);8479 elf.phdrs.items[next_phndx] = .wrap(segment_ni);
8199 elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) };8480 elf.nodes.items(.data)[@backingInt(segment_ni)] = .{ .segment = @intCast(next_phndx) };
...@@ -8213,24 +8494,23 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo...@@ -8213,24 +8494,23 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo
8213 const trace = tracy.trace(@src());8494 const trace = tracy.trace(@src());
8214 defer trace.end();8495 defer trace.end();
82158496
8216 elf.mf.nodes_lock.lock();
8217 defer elf.mf.nodes_lock.unlock();
8218
8219 _, const size = ni.location(&elf.mf).resolve(&elf.mf);8497 _, const size = ni.location(&elf.mf).resolve(&elf.mf);
8220 switch (elf.getNode(ni)) {8498 switch (elf.getNode(ni)) {
8221 .archive => {8499 .archive, .archive_header => {},
8222 if (ni.last(&elf.mf).unwrap()) |last_ni| {8500 .archive_input_member => unreachable,
8223 if (last_ni.prev(&elf.mf).unwrap()) |prev_ni| {8501 .archive_elf_member_header => unreachable,
8224 if (prev_ni.hasNextMoved(&elf.mf)) return;8502 .elf => if (elf.archive) |*archive| {
8225 }8503 const member_ar_hdr: *std.elf.ar_hdr = @ptrCast(
8226 const offset, _ = last_ni.location(&elf.mf).resolve(&elf.mf);8504 archive.elf_member_header_ni.slice(&elf.mf),
8227 _ = std.mem.print(&elf.arHdrPtr(last_ni).ar_size, "{d:<10}", .{8505 );
8228 size - offset,8506 if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{size})) |size_str| {
8229 }) catch @panic("archive member too large");8507 @memset(member_ar_hdr.ar_size[size_str.len..], ' ');
8508 archive.elf_member_too_big = false;
8509 } else |err| switch (err) {
8510 error.NoSpaceLeft => archive.elf_member_too_big = true,
8230 }8511 }
8231 },8512 },
8232 .archive_header, .elf => {},8513 .ehdr => unreachable,
8233 .ehdr, .archive_elf_footer => unreachable,
8234 .shdr => {},8514 .shdr => {},
8235 .segment => |phndx| switch (elf.phdrSlice()) {8515 .segment => |phndx| switch (elf.phdrSlice()) {
8236 inline else => |phdr| {8516 inline else => |phdr| {
...@@ -8302,7 +8582,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo...@@ -8302,7 +8582,7 @@ fn flushResized(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!vo
8302 }8582 }
8303 },8583 },
8304 },8584 },
8305 .input_member, .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {},8585 .input_section, .copied_global, .nav, .uav, .lazy_code, .lazy_const_data => {},
8306 }8586 }
8307}8587}
83088588
...@@ -8310,12 +8590,11 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!...@@ -8310,12 +8590,11 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!
8310 const trace = tracy.trace(@src());8590 const trace = tracy.trace(@src());
8311 defer trace.end();8591 defer trace.end();
83128592
8313 elf.mf.nodes_lock.lock();
8314 defer elf.mf.nodes_lock.unlock();
8315
8316 switch (elf.getNode(ni)) {8593 switch (elf.getNode(ni)) {
8317 .archive,8594 .archive,
8318 .archive_elf_footer,8595 .archive_input_member,
8596 .archive_elf_member_header,
8597 .elf,
8319 .ehdr,8598 .ehdr,
8320 .shdr,8599 .shdr,
8321 .segment,8600 .segment,
...@@ -8327,54 +8606,32 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!...@@ -8327,54 +8606,32 @@ fn flushNextMoved(elf: *Elf, ni: MappedFile.Node.Index) std.mem.Allocator.Error!
8327 .lazy_code,8606 .lazy_code,
8328 .lazy_const_data,8607 .lazy_const_data,
8329 => unreachable,8608 => unreachable,
8330 .archive_header, .elf, .input_member => |_, tag| {8609
8331 const member_offset, const update_size = member_offset: {8610 .archive_header => {
8332 const offset, _ = ni.location(&elf.mf).resolve(&elf.mf);8611 const archive = &elf.archive.?;
8333 break :member_offset switch (tag) {8612
8334 else => unreachable,8613 // Because we can't just throw padding bytes in the middle of an archive file, we need
8335 .archive_header => .{ offset + std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr), true },8614 // the member name string table (the "//" member) to absorb all the padding bytes
8336 .elf, .input_member => .{ offset, !ni.prev(&elf.mf).unwrap().?.hasNextMoved(&elf.mf) },8615 // between it (in the `.archive_header` node) and the first actual member.
8337 };8616 const next_member_ni = ni.next(&elf.mf).unwrap() orelse {
8338 };8617 // I guess there are no link inputs yet? But there will be eventually!
8339 const member_size = if (ni.next(&elf.mf).unwrap()) |next_ni| member_size: {8618 return;
8340 const next_offset, _ = next_ni.location(&elf.mf).resolve(&elf.mf);
8341 const next_member_size = if (next_ni.next(&elf.mf).unwrap()) |next_next_ni| next_member_size: {
8342 const next_next_offset, _ = next_next_ni.location(&elf.mf).resolve(&elf.mf);
8343 const next_member_end = next_next_offset - @sizeOf(std.elf.ar_hdr);
8344 break :next_member_size next_member_end - next_offset;
8345 } else next_member_size: {
8346 _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf);
8347 const next_member_end = parent_size;
8348 break :next_member_size next_member_end - next_offset;
8349 };
8350 const ar_hdr = elf.arHdrPtr(next_ni);
8351 var name_buf: [16]u8 = undefined;
8352 _ = std.mem.print(&ar_hdr.ar_name, "{s:<16}", .{
8353 switch (elf.getNode(next_ni)) {
8354 else => unreachable,
8355 .elf => std.mem.print(&name_buf, "{s}_zcu.o/", .{elf.base.comp.root_name}),
8356 .input_member => |ii| std.mem.print(&name_buf, "{s}/", .{
8357 std.fs.path.basename(ii.path(elf).sub_path),
8358 }),
8359 } catch @panic("TODO: long archive member names"),
8360 }) catch @panic("TODO: long archive member names");
8361 ar_hdr.ar_date = "0 ".*;
8362 ar_hdr.ar_uid = "0 ".*;
8363 ar_hdr.ar_gid = "0 ".*;
8364 ar_hdr.ar_mode = "644 ".*;
8365 _ = std.mem.print(&ar_hdr.ar_size, "{d:<10}", .{next_member_size}) catch
8366 @panic("archive member too large");
8367 ar_hdr.ar_fmag = std.elf.ARFMAG.*;
8368 const member_end = next_offset - @sizeOf(std.elf.ar_hdr);
8369 break :member_size member_end - member_offset;
8370 } else member_size: {
8371 _, const parent_size = ni.parent(&elf.mf).unwrap().?.location(&elf.mf).resolve(&elf.mf);
8372 const member_end = parent_size;
8373 break :member_size member_end - member_offset;
8374 };8619 };
8375 if (update_size) _ = std.mem.print(&elf.arHdrPtr(ni).ar_size, "{d:<10}", .{8620 const next_member_offset: u64, _ = next_member_ni.location(&elf.mf).resolve(&elf.mf);
8376 member_size,8621 const strtab_member_offset = std.elf.ARMAG.len + @sizeOf(std.elf.ar_hdr);
8377 }) catch @panic("archive member too large");8622 assert(Alignment.@"2".check(next_member_offset));
8623 assert(Alignment.@"2".check(strtab_member_offset));
8624 const strtab_size = next_member_offset - strtab_member_offset;
8625
8626 const member_ar_hdr: *std.elf.ar_hdr = @ptrCast(
8627 archive.header_ni.slice(&elf.mf)[std.elf.ARMAG.len..][0..@sizeOf(std.elf.ar_hdr)],
8628 );
8629 if (std.mem.print(&member_ar_hdr.ar_size, "{d}", .{strtab_size})) |size_str| {
8630 @memset(member_ar_hdr.ar_size[size_str.len..], ' ');
8631 archive.strtab_member_too_big = false;
8632 } else |err| switch (err) {
8633 error.NoSpaceLeft => archive.strtab_member_too_big = true,
8634 }
8378 },8635 },
8379 }8636 }
8380}8637}
src/link/MappedFile.zig+405-236
...@@ -648,7 +648,10 @@ pub const Node = extern struct {...@@ -648,7 +648,10 @@ pub const Node = extern struct {
648 _, const current_size = ni.location(mf).resolve(mf);648 _, const current_size = ni.location(mf).resolve(mf);
649 if (current_size >= min_size) return;649 if (current_size >= min_size) return;
650 const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor);650 const new_size = ni.alignment(mf).forward(min_size +| min_size / growth_factor);
651 try mf.growNode(gpa, ni, new_size, .minimum);651 try mf.growNode(gpa, ni, new_size, .{
652 .exact_size = false,
653 .move_footers = true,
654 });
652 mf.updateWriters();655 mf.updateWriters();
653 }656 }
654657
...@@ -664,7 +667,10 @@ pub const Node = extern struct {...@@ -664,7 +667,10 @@ pub const Node = extern struct {
664 switch (std.math.order(size, old_size)) {667 switch (std.math.order(size, old_size)) {
665 .lt => try mf.shrinkLeafNode(gpa, ni, size),668 .lt => try mf.shrinkLeafNode(gpa, ni, size),
666 .eq => {}, // `old_size` must be well-aligned, so `size` is too669 .eq => {}, // `old_size` must be well-aligned, so `size` is too
667 .gt => try mf.growNode(gpa, ni, size, .exact),670 .gt => try mf.growNode(gpa, ni, size, .{
671 .exact_size = true,
672 .move_footers = false, // irrelevant, since we have no footers
673 }),
668 }674 }
669 mf.updateWriters();675 mf.updateWriters();
670 }676 }
...@@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct {...@@ -935,7 +941,10 @@ fn addNode(mf: *MappedFile, gpa: Allocator, opts: struct {
935941
936 try mf.realignNode(gpa, new_ni, opts.add_options.alignment);942 try mf.realignNode(gpa, new_ni, opts.add_options.alignment);
937 if (opts.add_options.size > 0) {943 if (opts.add_options.size > 0) {
938 try mf.growNode(gpa, new_ni, opts.add_options.size, .exact);944 try mf.growNode(gpa, new_ni, opts.add_options.size, .{
945 .exact_size = true,
946 .move_footers = false, // irrelevant, since we have no footers
947 });
939 }948 }
940 mf.updateWriters();949 mf.updateWriters();
941950
...@@ -1070,12 +1079,21 @@ fn shrinkLeafNode(...@@ -1070,12 +1079,21 @@ fn shrinkLeafNode(
1070 }1079 }
1071}1080}
10721081
1073const GrowMode = enum { exact, minimum };1082const GrowOptions = struct {
1083 /// If `true`, the node size must be set to exactly the given size.
1084 ///
1085 /// If `false`, the given size is a minimum, and the actual new node size may be larger.
1086 exact_size: bool,
1087 /// If `true`, footers within the resized node will be moved forwards to its new end.
1088 ///
1089 /// If `false`, footers will all remain at their current offsets (so the nodes are in a
1090 /// temporarily invalid state), and moving them is the responsibility of the *caller*.
1091 move_footers: bool,
1092};
10741093
1075/// Increases the size of a node. If `grow_mode` is `.exact`, the new size will be exactly `new_size`.1094/// Increases the size of a node.
1076/// If `grow_mode` is `.minimum`, the new size will be greater than or equal to `new_size`.
1077///1095///
1078/// Asserts that `new_size` is aligned to `ni.alignment(mf)` (even if `grow_mode` is `.minimum`!).1096/// Asserts that `new_size` is aligned to `ni.alignment(mf)`, even if `!grow_options.exact_size`.
1079///1097///
1080/// Asserts that `new_size` is greater than the current size of `ni`.1098/// Asserts that `new_size` is greater than the current size of `ni`.
1081fn growNode(1099fn growNode(
...@@ -1083,7 +1101,7 @@ fn growNode(...@@ -1083,7 +1101,7 @@ fn growNode(
1083 gpa: Allocator,1101 gpa: Allocator,
1084 ni: Node.Index,1102 ni: Node.Index,
1085 new_size: u64,1103 new_size: u64,
1086 grow_mode: GrowMode,1104 grow_options: GrowOptions,
1087) Error!void {1105) Error!void {
1088 mf.nodes_lock.assertUnlocked();1106 mf.nodes_lock.assertUnlocked();
10891107
...@@ -1098,7 +1116,7 @@ fn growNode(...@@ -1098,7 +1116,7 @@ fn growNode(
1098 const parent_ni = node.parent.unwrap() orelse {1116 const parent_ni = node.parent.unwrap() orelse {
1099 assert(ni == .root);1117 assert(ni == .root);
11001118
1101 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {1119 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1102 return;1120 return;
1103 }1121 }
11041122
...@@ -1120,21 +1138,23 @@ fn growNode(...@@ -1120,21 +1138,23 @@ fn growNode(
1120 };1138 };
1121 try mf.ensureTotalCapacityPrecise(@intCast(new_size));1139 try mf.ensureTotalCapacityPrecise(@intCast(new_size));
1122 try ni.setLocation(mf, gpa, old_offset, new_size);1140 try ni.setLocation(mf, gpa, old_offset, new_size);
1123 // We need to move any footers to be at the *new* end of the file.1141 if (grow_options.move_footers) {
1124 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {1142 // We need to move any footers to be at the *new* end of the file.
1125 const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);1143 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {
1126 const footers_size = old_size - old_footers_offset;1144 const old_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
1127 try mf.moveRange(1145 const footers_size = old_size - old_footers_offset;
1128 old_footers_offset,1146 try mf.moveRange(
1129 old_footers_offset + (new_size - old_size),1147 old_footers_offset,
1130 footers_size,1148 old_footers_offset + (new_size - old_size),
1131 );1149 footers_size,
1132 // Also update the footers' locations.1150 );
1133 var cur_ni = first_footer_ni;1151 // Also update the footers' locations.
1134 while (true) {1152 var cur_ni = first_footer_ni;
1135 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);1153 while (true) {
1136 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);1154 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
1137 cur_ni = cur_ni.next(mf).unwrap() orelse break;1155 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
1156 cur_ni = cur_ni.next(mf).unwrap() orelse break;
1157 }
1138 }1158 }
1139 }1159 }
1140 return;1160 return;
...@@ -1142,7 +1162,7 @@ fn growNode(...@@ -1142,7 +1162,7 @@ fn growNode(
11421162
1143 switch (node.flags.position) {1163 switch (node.flags.position) {
1144 .header => {1164 .header => {
1145 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {1165 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1146 return;1166 return;
1147 }1167 }
11481168
...@@ -1163,7 +1183,13 @@ fn growNode(...@@ -1163,7 +1183,13 @@ fn growNode(
1163 const old_headers_size = last_header_offset + last_header_size;1183 const old_headers_size = last_header_offset + last_header_size;
11641184
1165 // This is the first footer *inside* of `ni`.1185 // This is the first footer *inside* of `ni`.
1166 const first_sub_footer_oni = ni.firstFooter(mf);1186 const first_sub_footer_oni: Node.Index.Optional = footer: {
1187 if (!grow_options.move_footers) {
1188 // Pretend there are no footers so as to not move them.
1189 break :footer .none;
1190 }
1191 break :footer ni.firstFooter(mf);
1192 };
1167 const sub_footers_size = size: {1193 const sub_footers_size = size: {
1168 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;1194 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1169 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);1195 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
...@@ -1215,92 +1241,264 @@ fn growNode(...@@ -1215,92 +1241,264 @@ fn growNode(
1215 return;1241 return;
1216 },1242 },
1217 .floating => {1243 .floating => {
1218 try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_mode);1244 try mf.growFloatingNodeWithAlignment(gpa, ni, null, new_size, grow_options);
1219 },1245 },
1220 .footer => {1246 .footer => {
1221 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_mode)) {1247 if (try mf.growNodeViaInsertRange(gpa, ni, new_size, grow_options)) {
1222 return;1248 return;
1223 }1249 }
12241250
1225 try mf.ensureAdditionalFooterCapacity(gpa, parent_ni, new_size - old_size);1251 // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself
12261252 // a footer within its parent). We'll need this later in any case, so just find it now.
1227 const first_footer_ni: Node.Index = first_footer: {1253 const first_sub_footer_oni: Node.Index.Optional = footer: {
1228 var footer_ni = ni;1254 if (!grow_options.move_footers) {
1229 while (true) {1255 // Pretend there are no nested footers so as to not move them.
1230 const prev_ni = footer_ni.prev(mf).unwrap() orelse break;1256 break :footer .none;
1231 if (prev_ni.position(mf) != .footer) break;
1232 footer_ni = prev_ni;
1233 }1257 }
1234 break :first_footer footer_ni;1258 break :footer ni.firstFooter(mf);
1235 };1259 };
12361260
1237 // This is the first footer *inside* of `ni` (unrelated to the fact that `ni` is itself1261 // We have two different strategies for growing a footer node, with different advantages
1238 // a footer within its parent).1262 // and disadvantages; so first we must decide which to use.
1239 const first_sub_footer_oni = ni.firstFooter(mf);1263 const strat: union(enum) {
1240 const sub_footers_size = size: {1264 /// Expand into pre-footer padding space in the parent node (growing the parent if
1241 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;1265 /// necessary). This strategy has the benefit that it can reclaim padding bytes in
1242 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);1266 /// the parent, but it has the disadvantage that it requires moving this node's
1243 break :size old_size - first_sub_footer_offset;1267 /// existing content backwards in the file, which may be expensive (particularly
1268 /// since the src and dest ranges are likely to overlap).
1269 grow_backwards,
1270
1271 /// Grow the parent node with `GrowOptions.move_footers` set to `false`, and
1272 /// implicitly grow ourselves into the newly available space. This usually requires
1273 /// a lot less moving of bytes, but never reclaims unused space before the parent's
1274 /// footers, and is sometimes straight-up impossible.
1275 grow_parent_at_end: struct {
1276 add_size: u64,
1277 exact_size: bool,
1278 },
1279 } = strat: {
1280 // If this node is small, the move overhead is trivial, so prefer `.grow_backwards`
1281 // to avoid unnecessary growth of the parent node.
1282 if (old_size <= mf.flags.block_size.toByteUnits() * 2) {
1283 break :strat .grow_backwards;
1284 }
1285
1286 // It may also be worth doing `.grow_backwards` if the parent has a *lot* of space
1287 // we could grow into. More specifically, if "free space we can grow into" makes up
1288 // a significant proportion of the parent's total size, then that implies the parent
1289 // has quite poor utilization of space, *and* that we can significantly improve that
1290 // statistic by growing into that space.
1291 if (old_size + mf.availableFooterCapacity(parent_ni) >= new_size) {
1292 break :strat .grow_backwards;
1293 }
1294
1295 if (grow_options.exact_size) {
1296 const add_size = new_size - old_size;
1297 if (parent_ni.alignment(mf).check(add_size)) {
1298 break :strat .{ .grow_parent_at_end = .{
1299 .add_size = add_size,
1300 .exact_size = true,
1301 } };
1302 } else {
1303 // We *can't* ask the parent to grow by this much, so we have no choice.
1304 break :strat .grow_backwards;
1305 }
1306 }
1307
1308 if (parent_ni.alignment(mf).compare(.lt, node.flags.alignment)) {
1309 // Because the parent's alignment is less than our own, if we gave them the
1310 // freedom to pick a size, they might choose one which results in *us* having a
1311 // size incompatible with our alignment. Therefore, to prevent that, we need to
1312 // request an *exact* size from the parent in this case.
1313 break :strat .{ .grow_parent_at_end = .{
1314 .add_size = new_size - old_size,
1315 .exact_size = true,
1316 } };
1317 }
1318
1319 // The parent's alignment is greater than or equal to our own, so we only need to
1320 // give the parent a *minimum* size (although we need to ensure it matches their
1321 // alignment since it could be greater than our own).
1322 break :strat .{ .grow_parent_at_end = .{
1323 .add_size = parent_ni.alignment(mf).forward(new_size - old_size),
1324 .exact_size = false,
1325 } };
1244 };1326 };
12451327
1246 _, const parent_size = parent_ni.location(mf).resolve(mf);1328 switch (strat) {
1329 .grow_backwards => {
1330 // First, we might need to grow the parent to make enough space.
1331 {
1332 const available_size = mf.availableFooterCapacity(parent_ni);
1333 if (old_size + available_size < new_size) {
1334 _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf);
1335 const min_parent_size = old_parent_size + (new_size - old_size - available_size);
1336 const new_parent_size = parent_ni.alignment(mf).forward(
1337 min_parent_size +| min_parent_size / growth_factor,
1338 );
1339 try mf.growNode(gpa, parent_ni, new_parent_size, .{
1340 .exact_size = false,
1341 .move_footers = true,
1342 });
1343 assert(old_size + mf.availableFooterCapacity(parent_ni) >= new_size);
1344 }
1345 }
12471346
1248 const old_footers_size = parent_size - first_footer_ni.location(mf).resolve(mf)[0];1347 // Now we need to grow! To do that, we must move `ni` itself, and every footer
1249 const new_footers_size = old_footers_size - old_size + new_size;1348 // before it in `parent_ni`, backwards. Unlike header nodes, `ni` is included in
12501349 // the shift, because the bytes we're adding need to go at the *end* of `ni`
1251 // Shift ourselves, and any footer before us, backwards. Unlike header nodes, this node1350 // rather than its start.
1252 // itself needs to shift its contents, because our offset was shifted backwards by1351
1253 // `new_size - old_size`, and the added bytes should go at the end of this footer node.1352 // This is the same as `parent_ni.firstFooter(mf)`, it's just more efficient to
1254 // However, if we *contain* any footer nodes, they need to stay at the end of `ni`, so1353 // start at `ni` than to start at `parent_ni.last(mf)`.
1255 // we *shouldn't* shift *that* data.1354 const first_parent_footer_ni: Node.Index = first_footer: {
1256 const old_footers_start = parent_size - old_footers_size;1355 var footer_ni = ni;
1257 const new_footers_start = parent_size - new_footers_size;1356 while (true) {
1258 const end_offset = node.location().resolve(mf)[0] + old_size;1357 const prev_ni = footer_ni.prev(mf).unwrap() orelse break;
1259 const parent_file_offset = parent_ni.fileLocation(mf, false).offset;1358 if (prev_ni.position(mf) != .footer) break;
1260 try mf.moveRange(1359 footer_ni = prev_ni;
1261 parent_file_offset + old_footers_start,1360 }
1262 parent_file_offset + new_footers_start,1361 break :first_footer footer_ni;
1263 end_offset - old_footers_start - sub_footers_size,1362 };
1264 );
12651363
1266 // Update our own offset and size:1364 const shift = new_size - old_size;
1267 try ni.setLocation(mf, gpa, end_offset - new_size, new_size);
12681365
1269 // Any footers inside of us have had their offsets changed due to us growing:1366 // Update our own offset and size:
1270 if (first_sub_footer_oni.unwrap()) |first_sub_footer_ni| {1367 try ni.setLocation(
1271 var cur_ni = first_sub_footer_ni;
1272 while (true) {
1273 const old_sub_footer_offset, const sub_footer_size = cur_ni.location(mf).resolve(mf);
1274 try cur_ni.setLocation(
1275 mf,1368 mf,
1276 gpa,1369 gpa,
1277 old_sub_footer_offset + (new_size - old_size),1370 node.location().resolve(mf)[0] - shift,
1278 sub_footer_size,1371 new_size,
1279 );1372 );
1280 cur_ni = cur_ni.next(mf).unwrap() orelse break;
1281 }
1282 }
12831373
1284 // Finally, update the offsets of every footer before us:1374 // Any footers *inside* of `ni` have had their offsets changed, because they are
1285 if (node.prev.unwrap()) |prev_ni| {1375 // now positioned at the *new* end of `ni`:
1286 var maybe_footer_ni = prev_ni;1376 {
1287 while (true) {1377 var footer_oni = first_sub_footer_oni;
1288 switch (maybe_footer_ni.position(mf)) {1378 while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) {
1289 .header, .floating => break,1379 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1290 .footer => {},1380 try footer_ni.setLocation(mf, gpa, old_footer_offset + shift, footer_size);
1381 }
1382 }
1383
1384 // Any footers *before* `ni` (in `parent_ni`) have been shifted backwards. We'll
1385 // also be moving their actual bytes in a moment, so track whether they have
1386 // content (if nothing does then we'll be able to skip the `moveRange`). That
1387 // flag is initially whether `ni` has content because we're shifting our own
1388 // bytes backwards too.
1389 var moved_has_content: bool = node.flags.has_content;
1390 {
1391 var footer_ni = first_parent_footer_ni;
1392 while (footer_ni != ni) : (footer_ni = footer_ni.next(mf).unwrap().?) {
1393 moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
1394 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1395 try footer_ni.setLocation(mf, gpa, old_footer_offset - shift, footer_size);
1396 }
1291 }1397 }
1292 const moved_footer_offset, const moved_footer_size = maybe_footer_ni.location(mf).resolve(mf);1398
1293 try maybe_footer_ni.setLocation(1399 if (moved_has_content) {
1400 // We moved at least one thing containing initialized bytes, so we need to
1401 // move the actual data. However, we should *not* move the bytes of any
1402 // nested footers inside of `ni`, because they've been "moved" to the end
1403 // of our new size, which is the same file location as before.
1404 const sub_footers_size = size: {
1405 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1406 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
1407 break :size new_size - first_sub_footer_offset;
1408 };
1409 const new_offset: u64, _ = node.location().resolve(mf);
1410 const new_footers_offset: u64, _ = first_parent_footer_ni.location(mf).resolve(mf);
1411 const parent_file_offset = parent_ni.fileLocation(mf, false).offset;
1412 try mf.moveRange(
1413 parent_file_offset + new_footers_offset + shift,
1414 parent_file_offset + new_footers_offset,
1415 (new_offset - new_footers_offset) + // accounts for every footer before `ni`
1416 (old_size - sub_footers_size), // accounts for `ni` itself, excluding nested footers
1417 );
1418 }
1419 },
1420 .grow_parent_at_end => |grow_parent| {
1421 _, const old_parent_size: u64 = parent_ni.location(mf).resolve(mf);
1422 try mf.growNode(gpa, parent_ni, old_parent_size + grow_parent.add_size, .{
1423 .exact_size = grow_parent.exact_size,
1424 .move_footers = false,
1425 });
1426 _, const new_parent_size: u64 = parent_ni.location(mf).resolve(mf);
1427 const shift = new_parent_size - old_parent_size;
1428
1429 // Here's what we have left to do:
1430 //
1431 // * Increase our own size by `shift` to absorb the added space.
1432 //
1433 // * If there are any footers *inside* `ni`, increase their offsets by `shift`.
1434 //
1435 // * If there are any footers *after* `ni` (inside `parent_ni`), increase their
1436 // offsets by `shift`.
1437 //
1438 // * Do a `moveRange` corresponding to those offset changes. This is a single
1439 // range which starts at the footers *inside* `ni`.
1440
1441 const actual_new_size = old_size + shift;
1442 if (grow_options.exact_size) {
1443 assert(actual_new_size == new_size);
1444 }
1445
1446 try ni.setLocation(
1294 mf,1447 mf,
1295 gpa,1448 gpa,
1296 moved_footer_offset + old_size - new_size,1449 node.location().resolve(mf)[0],
1297 moved_footer_size,1450 actual_new_size,
1298 );1451 );
1299 maybe_footer_ni = maybe_footer_ni.prev(mf).unwrap() orelse break;
1300 }
1301 }
13021452
1303 return;1453 // This will track whether any node with a changed offset actually contains
1454 // initialized bytes. If not, there'll be no need to call `moveRange`.
1455 var moved_has_content: bool = false;
1456
1457 // Set any nested footers' offsets (and include them in `moved_has_content`).
1458 {
1459 var footer_oni = first_sub_footer_oni;
1460 while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) {
1461 assert(footer_ni.position(mf) == .footer);
1462 moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
1463 const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf);
1464 try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size);
1465 }
1466 }
1467
1468 // Now set offsets for footers after `ni` inside of `parent_ni`.
1469 {
1470 var footer_oni = ni.next(mf);
1471 while (footer_oni.unwrap()) |footer_ni| : (footer_oni = footer_ni.next(mf)) {
1472 assert(footer_ni.position(mf) == .footer);
1473 moved_has_content = moved_has_content or footer_ni.get(mf).flags.has_content;
1474 const footer_old_offset: u64, const footer_size: u64 = footer_ni.location(mf).resolve(mf);
1475 try footer_ni.setLocation(mf, gpa, footer_old_offset + shift, footer_size);
1476 }
1477 }
1478
1479 if (moved_has_content) {
1480 // We moved at least one footer containing initialized bytes, so we need to
1481 // move the actual data. Compute how big the footers inside `ni` are...
1482 const sub_footers_size: u64 = size: {
1483 const first_sub_footer_ni = first_sub_footer_oni.unwrap() orelse break :size 0;
1484 const first_sub_footer_offset, _ = first_sub_footer_ni.location(mf).resolve(mf);
1485 // `actual_new_size` is used here since we already updated the nested footers' offsets above.
1486 break :size actual_new_size - first_sub_footer_offset;
1487 };
1488 // ...and how big the footers *after* `ni`, inside `parent_ni`, are...
1489 const post_footers_size: u64 = old_parent_size - (old_offset + old_size);
1490 // ...and move them both.
1491 const parent_file_off = parent_ni.fileLocation(mf, false).offset;
1492 const total_move_size = sub_footers_size + post_footers_size;
1493 assert(total_move_size != 0);
1494 try mf.moveRange(
1495 parent_file_off + old_parent_size - total_move_size,
1496 parent_file_off + new_parent_size - total_move_size,
1497 total_move_size,
1498 );
1499 }
1500 },
1501 }
1304 },1502 },
1305 }1503 }
1306}1504}
...@@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment(...@@ -1320,7 +1518,7 @@ fn growFloatingNodeWithAlignment(
1320 ni: Node.Index,1518 ni: Node.Index,
1321 new_alignment: ?Alignment,1519 new_alignment: ?Alignment,
1322 new_size: u64,1520 new_size: u64,
1323 grow_mode: GrowMode,1521 grow_options: GrowOptions,
1324) Error!void {1522) Error!void {
1325 mf.nodes_lock.assertUnlocked();1523 mf.nodes_lock.assertUnlocked();
13261524
...@@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment(...@@ -1347,27 +1545,29 @@ fn growFloatingNodeWithAlignment(
1347 }1545 }
1348 // Great, we can grow this node without changing its offset or moving any siblings.1546 // Great, we can grow this node without changing its offset or moving any siblings.
1349 try ni.setLocation(mf, gpa, old_offset, new_size);1547 try ni.setLocation(mf, gpa, old_offset, new_size);
1350 // If we have any footers, we need to move them to the end of our new size, and update their1548 if (grow_options.move_footers) {
1351 // offsets accordingly.1549 // If we have any footers, we need to move them to the end of our new size, and update
1352 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {1550 // their offsets accordingly.
1353 var cur_ni = first_footer_ni;1551 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {
1354 var footers_have_content = false;1552 var cur_ni = first_footer_ni;
1355 while (true) {1553 var footers_have_content = false;
1356 footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;1554 while (true) {
1357 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);1555 footers_have_content = footers_have_content or cur_ni.get(mf).flags.has_content;
1358 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);1556 const old_footer_offset, const footer_size = cur_ni.location(mf).resolve(mf);
1359 cur_ni = cur_ni.next(mf).unwrap() orelse break;1557 try cur_ni.setLocation(mf, gpa, old_footer_offset + (new_size - old_size), footer_size);
1360 }1558 cur_ni = cur_ni.next(mf).unwrap() orelse break;
1361 if (footers_have_content) {1559 }
1362 const parent_file_off = parent_ni.fileLocation(mf, false).offset;1560 if (footers_have_content) {
1363 // This gets the *new* offset because we already updated the offsets above.1561 const parent_file_off = parent_ni.fileLocation(mf, false).offset;
1364 const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);1562 // This gets the *new* offset because we already updated the offsets above.
1365 const footers_size = new_size - new_footers_offset;1563 const new_footers_offset, _ = first_footer_ni.location(mf).resolve(mf);
1366 try mf.moveRange(1564 const footers_size = new_size - new_footers_offset;
1367 parent_file_off + old_offset + old_size - footers_size,1565 try mf.moveRange(
1368 parent_file_off + old_offset + new_size - footers_size,1566 parent_file_off + old_offset + old_size - footers_size,
1369 footers_size,1567 parent_file_off + old_offset + new_size - footers_size,
1370 );1568 footers_size,
1569 );
1570 }
1371 }1571 }
1372 }1572 }
1373 return;1573 return;
...@@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment(...@@ -1444,11 +1644,14 @@ fn growFloatingNodeWithAlignment(
1444 // that, let's first try the Linux "insert range" fast path. We didn't try it before now1644 // that, let's first try the Linux "insert range" fast path. We didn't try it before now
1445 // because it would have been more efficient to just move ourselves into existing space.1645 // because it would have been more efficient to just move ourselves into existing space.
1446 //1646 //
1447 // If we were given a custom alignment, we cannot pass `grow_mode` directly into the1647 // If we were given a custom alignment, we need to set `GrowOptions.exact_size` for the
1448 // "insert range" path, because that function is unaware of `new_alignment`.1648 // "insert range" path, because that function is unaware of `new_alignment`.
1449 const sub_grow_mode: GrowMode = if (new_alignment == null) grow_mode else .exact;1649 const insert_range_grow_options: GrowOptions = .{
1650 .exact_size = grow_options.exact_size or new_alignment != null,
1651 .move_footers = grow_options.move_footers,
1652 };
1450 if (alignment.check(old_offset) and1653 if (alignment.check(old_offset) and
1451 try mf.growNodeViaInsertRange(gpa, ni, new_size, sub_grow_mode))1654 try mf.growNodeViaInsertRange(gpa, ni, new_size, insert_range_grow_options))
1452 {1655 {
1453 // The Linux fast path did our job for us!1656 // The Linux fast path did our job for us!
1454 return;1657 return;
...@@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment(...@@ -1458,7 +1661,10 @@ fn growFloatingNodeWithAlignment(
1458 const new_parent_size = parent_ni.alignment(mf).forward(1661 const new_parent_size = parent_ni.alignment(mf).forward(
1459 min_parent_size +| min_parent_size / growth_factor,1662 min_parent_size +| min_parent_size / growth_factor,
1460 );1663 );
1461 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);1664 try mf.growNode(gpa, parent_ni, new_parent_size, .{
1665 .exact_size = false,
1666 .move_footers = true,
1667 });
1462 }1668 }
14631669
1464 break :new_loc .{1670 break :new_loc .{
...@@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment(...@@ -1471,6 +1677,10 @@ fn growFloatingNodeWithAlignment(
14711677
1472 // Footers need to move to a different place than the rest of our content.1678 // Footers need to move to a different place than the rest of our content.
1473 const footers_size: u64, const footers_have_content: bool = footers: {1679 const footers_size: u64, const footers_have_content: bool = footers: {
1680 if (!grow_options.move_footers) {
1681 // Pretend there are no footers so as to not move them.
1682 break :footers .{ 0, false };
1683 }
1474 const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {1684 const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
1475 break :footers .{ 0, false };1685 break :footers .{ 0, false };
1476 };1686 };
...@@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment(...@@ -1525,15 +1735,14 @@ fn growFloatingNodeWithAlignment(
1525/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false`1735/// If this strategy is inapplicable or unsuitable for this operation, this function returns `false`
1526/// without changing any nodes' locations or invalidating any slices.1736/// without changing any nodes' locations or invalidating any slices.
1527///1737///
1528/// Otherwise, this function grows `ni` to `new_size`, updates the location of `ni` and every node1738/// Otherwise, this function grows `ni` to `new_size` (maybe larger if `!grow_options.exact_size`),
1529/// whose offset has changed, and returns `true`. Like in `growNode`, if `grow_mode` is `.minimum`,1739/// updates the location of `ni` and every node whose offset has changed, and returns `true`.
1530/// the actual new size of `ni` may be greater than `new_size`.
1531fn growNodeViaInsertRange(1740fn growNodeViaInsertRange(
1532 mf: *MappedFile,1741 mf: *MappedFile,
1533 gpa: Allocator,1742 gpa: Allocator,
1534 ni: Node.Index,1743 ni: Node.Index,
1535 new_size: u64,1744 new_size: u64,
1536 grow_mode: GrowMode,1745 grow_options: GrowOptions,
1537) Error!bool {1746) Error!bool {
1538 if (!is_linux or mf.flags.fallocate_insert_range_unsupported) {1747 if (!is_linux or mf.flags.fallocate_insert_range_unsupported) {
1539 return false;1748 return false;
...@@ -1541,43 +1750,22 @@ fn growNodeViaInsertRange(...@@ -1541,43 +1750,22 @@ fn growNodeViaInsertRange(
15411750
1542 _, const old_size = ni.location(mf).resolve(mf);1751 _, const old_size = ni.location(mf).resolve(mf);
15431752
1544 // We don't compute the size of the range yet, because depending on `grow_mode` we might want to1753 // We don't compute the size of the range yet, because depending on `grow_options` we might want
1545 // bump it based on our sibling and parent nodes' alignments. However, we can do an early check1754 // to bump it based on our sibling and parent nodes' alignments. However, we can do an early
1546 // for cases where we should obviously exit.1755 // check for cases where we should obviously exit.
1547 const requested_range_size = new_size - old_size;
1548 if (!mf.flags.block_size.check(requested_range_size)) {
1549 // The requested size isn't exactly aligned.
1550 switch (grow_mode) {
1551 .exact => return false,
1552 .minimum => {
1553 // We can still choose to allow it by increasing the size a bit, but we shouldn't do
1554 // that if it would *significantly* increase the requested size.
1555 const block_size = mf.flags.block_size.toByteUnits();
1556 if (requested_range_size < block_size * 2) {
1557 // Bumping this size up to the next block boundary would be a quite significant
1558 // increase; let's not do it.
1559 return false;
1560 }
1561 },
1562 }
1563 }
1564 // If `grow_mode` is exact, we will use exactly this size, but if it is `.minimum`, we may bump
1565 // the size a little more.
1566 const min_range_size: u64 = s: {1756 const min_range_size: u64 = s: {
1567 const exact_size = new_size - old_size;1757 const requested_size = new_size - old_size;
1568 if (mf.flags.block_size.check(exact_size)) {1758 if (mf.flags.block_size.check(requested_size)) {
1569 break :s exact_size;1759 break :s requested_size;
1570 }
1571 switch (grow_mode) {
1572 .exact => return false,
1573 .minimum => if (exact_size >= mf.flags.block_size.toByteUnits() * 2) {
1574 // We're growing by at least a few blocks, so allow ourselves to bump the size
1575 // slightly to give it the needed alignment.
1576 break :s mf.flags.block_size.forward(exact_size);
1577 } else {
1578 return false;
1579 },
1580 }1760 }
1761 if (!grow_options.exact_size and
1762 requested_size >= mf.flags.block_size.toByteUnits() * 2)
1763 {
1764 // We're growing by at least a few blocks, so allow ourselves to bump the size
1765 // slightly to give it the needed alignment.
1766 break :s mf.flags.block_size.forward(requested_size);
1767 }
1768 return false;
1581 };1769 };
1582 assert(min_range_size > 0);1770 assert(min_range_size > 0);
1583 assert(mf.flags.block_size.check(min_range_size));1771 assert(mf.flags.block_size.check(min_range_size));
...@@ -1592,14 +1780,17 @@ fn growNodeViaInsertRange(...@@ -1592,14 +1780,17 @@ fn growNodeViaInsertRange(
1592 }1780 }
1593 break :range_file_offset range_file_offset;1781 break :range_file_offset range_file_offset;
1594 };1782 };
1595 const first_footer_oni = ni.firstFooter(mf);1783 const pre_footer_oni: Node.Index.Optional, const footers_size: u64 = footers: {
1596 const footers_size: u64 = if (first_footer_oni.unwrap()) |first_footer_ni| size: {1784 if (!grow_options.move_footers) {
1785 // Pretend there are no footers so as to not move them.
1786 break :footers .{ .wrap(last_ni), 0 };
1787 }
1788 const first_footer_ni = ni.firstFooter(mf).unwrap() orelse {
1789 break :footers .{ .wrap(last_ni), 0 };
1790 };
1597 const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);1791 const first_footer_offset, _ = first_footer_ni.location(mf).resolve(mf);
1598 break :size old_size - first_footer_offset;1792 break :footers .{ first_footer_ni.prev(mf), old_size - first_footer_offset };
1599 } else 0;1793 };
1600 const pre_footer_oni: Node.Index.Optional = if (first_footer_oni.unwrap()) |first_footer_ni| pre_footer: {
1601 break :pre_footer first_footer_ni.prev(mf);
1602 } else .wrap(last_ni);
1603 const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: {1794 const pre_footer_end: u64 = if (pre_footer_oni.unwrap()) |pre_footer_ni| end: {
1604 const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf);1795 const pre_footer_off, const pre_footer_size = pre_footer_ni.location(mf).resolve(mf);
1605 break :end pre_footer_off + pre_footer_size;1796 break :end pre_footer_off + pre_footer_size;
...@@ -1649,22 +1840,18 @@ fn growNodeViaInsertRange(...@@ -1649,22 +1840,18 @@ fn growNodeViaInsertRange(
1649 }1840 }
1650 // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment1841 // Traversal done. We didn't hit `max_moved_nodes`, so now we can use the computed alignment
1651 // requirement to figure out whether we're actually going to insert a range.1842 // requirement to figure out whether we're actually going to insert a range.
1652 if (need_range_align.check(requested_range_size)) {1843 if (need_range_align.check(min_range_size)) {
1653 break :range_size requested_range_size;1844 break :range_size min_range_size;
1654 }1845 }
1655 // Perhaps we're allowed to grow by more than `requested_range_size`?1846 // Perhaps we're allowed to grow by more than `min_range_size`?
1656 switch (grow_mode) {1847 const candidate_range_size = need_range_align.forward(min_range_size);
1657 .exact => return false,1848 if (!grow_options.exact_size and
1658 .minimum => {1849 // Allow growing by up to 50% more than was requested.
1659 const candidate_range_size = need_range_align.forward(min_range_size);1850 candidate_range_size <= min_range_size +| min_range_size / 2)
1660 // Allow growing by up to 50% more than was requested.1851 {
1661 if (candidate_range_size <= requested_range_size +| requested_range_size / 2) {1852 break :range_size candidate_range_size;
1662 break :range_size candidate_range_size;
1663 } else {
1664 return false;
1665 }
1666 },
1667 }1853 }
1854 return false;
1668 };1855 };
16691856
1670 // This `range_size` is compatible with everyone's alignment requirements, and we won't move too1857 // This `range_size` is compatible with everyone's alignment requirements, and we won't move too
...@@ -1742,13 +1929,15 @@ fn growNodeViaInsertRange(...@@ -1742,13 +1929,15 @@ fn growNodeViaInsertRange(
1742 cur_ni = cur_ni.parent(mf).unwrap() orelse break;1929 cur_ni = cur_ni.parent(mf).unwrap() orelse break;
1743 }1930 }
17441931
1745 // The only thing left is to update the offsets of any footers inside of `ni`.1932 if (grow_options.move_footers) {
1746 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {1933 // The only thing left is to update the offsets of any footers inside of `ni`.
1747 var footer_ni = first_footer_ni;1934 if (ni.firstFooter(mf).unwrap()) |first_footer_ni| {
1748 while (true) {1935 var footer_ni = first_footer_ni;
1749 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);1936 while (true) {
1750 try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);1937 const old_footer_offset, const footer_size = footer_ni.location(mf).resolve(mf);
1751 footer_ni = footer_ni.next(mf).unwrap() orelse break;1938 try footer_ni.setLocation(mf, gpa, old_footer_offset + range_size, footer_size);
1939 footer_ni = footer_ni.next(mf).unwrap() orelse break;
1940 }
1752 }1941 }
1753 }1942 }
17541943
...@@ -1802,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity(...@@ -1802,7 +1991,10 @@ fn ensureAdditionalHeaderCapacity(
1802 const new_parent_size = parent_ni.alignment(mf).forward(1991 const new_parent_size = parent_ni.alignment(mf).forward(
1803 min_parent_size +| min_parent_size / growth_factor,1992 min_parent_size +| min_parent_size / growth_factor,
1804 );1993 );
1805 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);1994 try mf.growNode(gpa, parent_ni, new_parent_size, .{
1995 .exact_size = false,
1996 .move_footers = true,
1997 });
1806 }1998 }
1807 return;1999 return;
1808 };2000 };
...@@ -1884,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity(...@@ -1884,7 +2076,10 @@ fn ensureAdditionalHeaderCapacity(
1884 const new_parent_size = parent_ni.alignment(mf).forward(2076 const new_parent_size = parent_ni.alignment(mf).forward(
1885 min_parent_size +| min_parent_size / growth_factor,2077 min_parent_size +| min_parent_size / growth_factor,
1886 );2078 );
1887 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);2079 try mf.growNode(gpa, parent_ni, new_parent_size, .{
2080 .exact_size = false,
2081 .move_footers = true,
2082 });
1888 }2083 }
18892084
1890 if (moving_has_content) {2085 if (moving_has_content) {
...@@ -1914,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity(...@@ -1914,48 +2109,27 @@ fn ensureAdditionalHeaderCapacity(
1914 }2109 }
1915}2110}
19162111
1917/// Ensures that `parent_ni` has at least `extra_capacity` padding bytes preceding its current2112/// Returns how many padding bytes `parent_ni` currently has directly preceding its footers, which
1918/// footers, so that the footers can grow into that space.2113/// footers can therefore grow into.
1919fn ensureAdditionalFooterCapacity(2114fn availableFooterCapacity(mf: *const MappedFile, parent_ni: Node.Index) u64 {
1920 mf: *MappedFile,
1921 gpa: Allocator,
1922 parent_ni: Node.Index,
1923 extra_capacity: u64,
1924) Error!void {
1925 // This is way easier than the header case, because we don't need to actually move anything; we
1926 // just need to expand the parent if there isn't space, and that will add padding after the
1927 // parent's floating children, which is exactly where we want it.
1928
1929 const first_footer_oni = parent_ni.firstFooter(mf);2115 const first_footer_oni = parent_ni.firstFooter(mf);
19302116
1931 _, const parent_size = parent_ni.location(mf).resolve(mf);2117 const before_footers_oni: Node.Index.Optional, const footers_off: u64 = footers: {
19322118 const first_footer_ni = first_footer_oni.unwrap() orelse {
1933 const footers_size: u64 = footers_size: {2119 _, const parent_size = parent_ni.location(mf).resolve(mf);
1934 const first_footer_ni = first_footer_oni.unwrap() orelse break :footers_size 0;2120 break :footers .{ parent_ni.last(mf), parent_size };
2121 };
1935 const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);2122 const first_footer_off, _ = first_footer_ni.location(mf).resolve(mf);
1936 break :footers_size parent_size - first_footer_off;2123 break :footers .{ first_footer_ni.prev(mf), first_footer_off };
1937 };2124 };
19382125
1939 const header_and_floating_end: u64 = end: {2126 const header_and_floating_end: u64 = end: {
1940 const before_footers_oni = if (first_footer_oni.unwrap()) |first_footer_ni| before_footers: {
1941 break :before_footers first_footer_ni.prev(mf);
1942 } else before_footers: {
1943 break :before_footers parent_ni.last(mf);
1944 };
1945 const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0;2127 const before_footers_ni = before_footers_oni.unwrap() orelse break :end 0;
1946 const offset, const size = before_footers_ni.location(mf).resolve(mf);2128 const offset, const size = before_footers_ni.location(mf).resolve(mf);
1947 break :end offset + size;2129 break :end offset + size;
1948 };2130 };
19492131
1950 assert(header_and_floating_end + footers_size <= parent_size);2132 return footers_off - header_and_floating_end;
1951
1952 const min_parent_size = header_and_floating_end + footers_size + extra_capacity;
1953 if (parent_size < min_parent_size) {
1954 const new_parent_size = parent_ni.alignment(mf).forward(
1955 min_parent_size +| min_parent_size / growth_factor,
1956 );
1957 try mf.growNode(gpa, parent_ni, new_parent_size, .minimum);
1958 }
1959}2133}
19602134
1961fn removeNodesFromChildList(2135fn removeNodesFromChildList(
...@@ -2049,7 +2223,7 @@ fn realignNode(...@@ -2049,7 +2223,7 @@ fn realignNode(
2049 mf: *MappedFile,2223 mf: *MappedFile,
2050 gpa: Allocator,2224 gpa: Allocator,
2051 ni: Node.Index,2225 ni: Node.Index,
2052 new_alignment: Alignment,2226 new_align: Alignment,
2053) Error!void {2227) Error!void {
2054 mf.nodes_lock.assertUnlocked();2228 mf.nodes_lock.assertUnlocked();
20552229
...@@ -2057,30 +2231,25 @@ fn realignNode(...@@ -2057,30 +2231,25 @@ fn realignNode(
20572231
2058 if (ni == .root or ni.position(mf) != .floating) {2232 if (ni == .root or ni.position(mf) != .floating) {
2059 // Only this node's size is aligned, not its offset.2233 // Only this node's size is aligned, not its offset.
2060 if (!new_alignment.check(old_size)) {2234 if (!new_align.check(old_size)) {
2061 assert(new_alignment.compare(.gt, ni.alignment(mf)));2235 assert(new_align.compare(.gt, ni.alignment(mf)));
2062 try mf.growNode(2236 try mf.growNode(gpa, ni, new_align.forward(old_size), .{
2063 gpa,2237 .exact_size = true, // because `growNode` is not aware that the size needs to match `new_align`
2064 ni,2238 .move_footers = true,
2065 new_alignment.forward(old_size),2239 });
2066 .exact, // because `growNode` is not aware that the size needs to match `new_alignment`
2067 );
2068 }2240 }
2069 } else {2241 } else {
2070 // This is a floating node, so its size and offset are both aligned.2242 // This is a floating node, so its size and offset are both aligned.
2071 if (!new_alignment.check(old_offset) or !new_alignment.check(old_size)) {2243 if (!new_align.check(old_offset) or !new_align.check(old_size)) {
2072 assert(new_alignment.compare(.gt, ni.alignment(mf)));2244 assert(new_align.compare(.gt, ni.alignment(mf)));
2073 try mf.growFloatingNodeWithAlignment(2245 try mf.growFloatingNodeWithAlignment(gpa, ni, new_align, new_align.forward(old_size), .{
2074 gpa,2246 .exact_size = false,
2075 ni,2247 .move_footers = true,
2076 new_alignment,2248 });
2077 new_alignment.forward(old_size),
2078 .minimum,
2079 );
2080 }2249 }
2081 }2250 }
20822251
2083 ni.get(mf).flags.alignment = new_alignment;2252 ni.get(mf).flags.alignment = new_align;
2084}2253}
20852254
2086fn updateWriters(mf: *MappedFile) void {2255fn updateWriters(mf: *MappedFile) void {