authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-10-03 09:22:43+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-10-16 19:33:04+02:00
log1b70ad622bd6dbc776563656a6aaee94d69c66ea
tree91d198693858844aa4a089491f20fae354e3f4c8
parent860beda55fd94fc56fce20c79aa6ddadce9baf39

elf: port zld's allocation mechanism


2 files changed, 380 insertions(+), 52 deletions(-)

src/link/Elf.zig+351-49
......@@ -1247,17 +1247,16 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
12471247 try self.sortSections();
12481248 try self.updateSectionSizes();
12491249
1250 try self.allocateSections();
1251 self.allocateAtoms();
1252 self.allocateLinkerDefinedSymbols();
1253
12501254 // Dump the state for easy debugging.
12511255 // State can be dumped via `--debug-log link_state`.
12521256 if (build_options.enable_logging) {
12531257 state_log.debug("{}", .{self.dumpState()});
12541258 }
12551259
1256 // Allocate atoms parsed from input object files, followed by allocating
1257 // linker-defined synthetic symbols.
1258 try self.allocateObjects();
1259 self.allocateLinkerDefinedSymbols();
1260
12611260 // Look for entry address in objects if not set by the incremental compiler.
12621261 if (self.entry_addr == null) {
12631262 const entry: ?[]const u8 = entry: {
......@@ -1757,34 +1756,6 @@ fn scanRelocs(self: *Elf) !void {
17571756 }
17581757}
17591758
1760fn allocateObjects(self: *Elf) !void {
1761 for (self.objects.items) |index| {
1762 const object = self.file(index).?.object;
1763
1764 for (object.atoms.items) |atom_index| {
1765 const atom_ptr = self.atom(atom_index) orelse continue;
1766 if (!atom_ptr.flags.alive or atom_ptr.flags.allocated) continue;
1767 try atom_ptr.allocate(self);
1768 }
1769
1770 for (object.locals()) |local_index| {
1771 const local = self.symbol(local_index);
1772 const atom_ptr = local.atom(self) orelse continue;
1773 if (!atom_ptr.flags.alive) continue;
1774 local.value = local.elfSym(self).st_value + atom_ptr.value;
1775 }
1776
1777 for (object.globals()) |global_index| {
1778 const global = self.symbol(global_index);
1779 const atom_ptr = global.atom(self) orelse continue;
1780 if (!atom_ptr.flags.alive) continue;
1781 if (global.file_index == index) {
1782 global.value = global.elfSym(self).st_value + atom_ptr.value;
1783 }
1784 }
1785 }
1786}
1787
17881759fn writeObjects(self: *Elf) !void {
17891760 const gpa = self.base.allocator;
17901761
......@@ -3398,14 +3369,9 @@ fn allocateLinkerDefinedSymbols(self: *Elf) void {
33983369 // _end
33993370 {
34003371 const end_symbol = self.symbol(self.end_index.?);
3401 end_symbol.value = 0;
3402 for (self.shdrs.items, 0..) |*shdr, shndx| {
3403 if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue;
3404 const phdr_index = self.phdr_to_shdr_table.get(@intCast(shndx)).?;
3405 const phdr = self.phdrs.items[phdr_index];
3406 const value = phdr.p_vaddr + phdr.p_memsz;
3407 if (end_symbol.value < value) {
3408 end_symbol.value = value;
3372 for (self.shdrs.items, 0..) |shdr, shndx| {
3373 if (shdr.sh_flags & elf.SHF_ALLOC != 0) {
3374 end_symbol.value = shdr.sh_addr + shdr.sh_size;
34093375 end_symbol.output_section_index = @intCast(shndx);
34103376 }
34113377 }
......@@ -3440,7 +3406,7 @@ fn initSections(self: *Elf) !void {
34403406 const atom_ptr = self.atom(atom_index) orelse continue;
34413407 if (!atom_ptr.flags.alive) continue;
34423408 const shdr = atom_ptr.inputShdr(self);
3443 atom_ptr.output_section_index = try object.getOutputSectionIndex(self, shdr);
3409 atom_ptr.output_section_index = try object.initOutputSection(self, shdr);
34443410 }
34453411 }
34463412
......@@ -3605,11 +3571,323 @@ fn updateSectionSizes(self: *Elf) !void {
36053571 if (self.got_section_index) |_| {
36063572 try self.got.updateStrtab(self);
36073573 }
3608 try self.growNonAllocSection(index, self.strtab.buffer.items.len, 1, false);
3574 self.shdrs.items[index].sh_size = self.strtab.buffer.items.len;
3575 // try self.growNonAllocSection(index, self.strtab.buffer.items.len, 1, false);
36093576 }
36103577
36113578 if (self.shstrtab_section_index) |index| {
3612 try self.growNonAllocSection(index, self.shstrtab.buffer.items.len, 1, false);
3579 self.shdrs.items[index].sh_size = self.shstrtab.buffer.items.len;
3580 // try self.growNonAllocSection(index, self.shstrtab.buffer.items.len, 1, false);
3581 }
3582}
3583
3584fn initPhdrs(self: *Elf) !void {
3585 // Add PHDR phdr
3586 const phdr_index = try self.addPhdr(.{
3587 .type = elf.PT_PHDR,
3588 .flags = elf.PF_R,
3589 .@"align" = @alignOf(elf.Elf64_Phdr),
3590 .addr = self.calcImageBase() + @sizeOf(elf.Elf64_Ehdr),
3591 .offset = @sizeOf(elf.Elf64_Ehdr),
3592 });
3593
3594 // Add INTERP phdr if required
3595 // if (self.interp_sect_index) |index| {
3596 // const shdr = self.sections.items(.shdr)[index];
3597 // _ = try self.addPhdr(.{
3598 // .type = elf.PT_INTERP,
3599 // .flags = elf.PF_R,
3600 // .@"align" = 1,
3601 // .offset = shdr.sh_offset,
3602 // .addr = shdr.sh_addr,
3603 // .filesz = shdr.sh_size,
3604 // .memsz = shdr.sh_size,
3605 // });
3606 // }
3607
3608 // Add LOAD phdrs
3609 const slice = self.shdrs.items;
3610 {
3611 var last_phdr: ?u16 = null;
3612 var shndx: usize = 0;
3613 while (shndx < slice.len) {
3614 const shdr = &slice[shndx];
3615 if (!shdrIsAlloc(shdr) or shdrIsTbss(shdr)) {
3616 shndx += 1;
3617 continue;
3618 }
3619 last_phdr = try self.addPhdr(.{
3620 .type = elf.PT_LOAD,
3621 .flags = shdrToPhdrFlags(shdr.sh_flags),
3622 .@"align" = @max(self.page_size, shdr.sh_addralign),
3623 .offset = if (last_phdr == null) 0 else shdr.sh_offset,
3624 .addr = if (last_phdr == null) self.calcImageBase() else shdr.sh_addr,
3625 });
3626 const p_flags = self.phdrs.items[last_phdr.?].p_flags;
3627 try self.addShdrToPhdr(last_phdr.?, shdr);
3628 shndx += 1;
3629
3630 while (shndx < slice.len) : (shndx += 1) {
3631 const next = &slice[shndx];
3632 if (shdrIsTbss(next)) continue;
3633 if (p_flags == shdrToPhdrFlags(next.sh_flags)) {
3634 if (shdrIsBss(next) or next.sh_offset - shdr.sh_offset == next.sh_addr - shdr.sh_addr) {
3635 try self.addShdrToPhdr(last_phdr.?, next);
3636 continue;
3637 }
3638 }
3639 break;
3640 }
3641 }
3642 }
3643
3644 // Add TLS phdr
3645 {
3646 var shndx: usize = 0;
3647 outer: while (shndx < slice.len) {
3648 const shdr = &slice[shndx];
3649 if (!shdrIsTls(shdr)) {
3650 shndx += 1;
3651 continue;
3652 }
3653 self.phdr_tls_index = try self.addPhdr(.{
3654 .type = elf.PT_TLS,
3655 .flags = elf.PF_R,
3656 .@"align" = shdr.sh_addralign,
3657 .offset = shdr.sh_offset,
3658 .addr = shdr.sh_addr,
3659 });
3660 try self.addShdrToPhdr(self.phdr_tls_index.?, shdr);
3661 shndx += 1;
3662
3663 while (shndx < slice.len) : (shndx += 1) {
3664 const next = &slice[shndx];
3665 if (!shdrIsTls(next)) continue :outer;
3666 try self.addShdrToPhdr(self.phdr_tls_index.?, next);
3667 }
3668 }
3669 }
3670
3671 // Add DYNAMIC phdr
3672 // if (self.dynamic_sect_index) |index| {
3673 // const shdr = self.sections.items(.shdr)[index];
3674 // _ = try self.addPhdr(.{
3675 // .type = elf.PT_DYNAMIC,
3676 // .flags = elf.PF_R | elf.PF_W,
3677 // .@"align" = shdr.sh_addralign,
3678 // .offset = shdr.sh_offset,
3679 // .addr = shdr.sh_addr,
3680 // .memsz = shdr.sh_size,
3681 // .filesz = shdr.sh_size,
3682 // });
3683 // }
3684
3685 // Add PT_GNU_EH_FRAME phdr if required.
3686 if (self.eh_frame_hdr_section_index) |index| {
3687 const shdr = self.shdrs.items[index];
3688 _ = try self.addPhdr(.{
3689 .type = elf.PT_GNU_EH_FRAME,
3690 .flags = elf.PF_R,
3691 .@"align" = shdr.sh_addralign,
3692 .offset = shdr.sh_offset,
3693 .addr = shdr.sh_addr,
3694 .memsz = shdr.sh_size,
3695 .filesz = shdr.sh_size,
3696 });
3697 }
3698
3699 // Add PT_GNU_STACK phdr that controls some stack attributes that apparently may or may not
3700 // be respected by the OS.
3701 _ = try self.addPhdr(.{
3702 .type = elf.PT_GNU_STACK,
3703 .flags = elf.PF_W | elf.PF_R,
3704 .memsz = self.base.options.stack_size_override orelse 0,
3705 .@"align" = 1,
3706 });
3707
3708 // Backpatch size of the PHDR phdr
3709 {
3710 const phdr = &self.phdrs.items[phdr_index];
3711 const size = @sizeOf(elf.Elf64_Phdr) * self.phdrs.items.len;
3712 phdr.p_filesz = size;
3713 phdr.p_memsz = size;
3714 }
3715}
3716
3717fn addShdrToPhdr(self: *Elf, phdr_index: u16, shdr: *const elf.Elf64_Shdr) !void {
3718 const phdr = &self.phdrs.items[phdr_index];
3719 phdr.p_align = @max(phdr.p_align, shdr.sh_addralign);
3720 if (shdr.sh_type != elf.SHT_NOBITS) {
3721 phdr.p_filesz = shdr.sh_addr + shdr.sh_size - phdr.p_vaddr;
3722 }
3723 phdr.p_memsz = shdr.sh_addr + shdr.sh_size - phdr.p_vaddr;
3724}
3725
3726fn shdrToPhdrFlags(sh_flags: u64) u32 {
3727 const write = sh_flags & elf.SHF_WRITE != 0;
3728 const exec = sh_flags & elf.SHF_EXECINSTR != 0;
3729 var out_flags: u32 = elf.PF_R;
3730 if (write) out_flags |= elf.PF_W;
3731 if (exec) out_flags |= elf.PF_X;
3732 return out_flags;
3733}
3734
3735inline fn shdrIsAlloc(shdr: *const elf.Elf64_Shdr) bool {
3736 return shdr.sh_flags & elf.SHF_ALLOC != 0;
3737}
3738
3739inline fn shdrIsBss(shdr: *const elf.Elf64_Shdr) bool {
3740 return shdrIsZerofill(shdr) and !shdrIsTls(shdr);
3741}
3742
3743inline fn shdrIsTbss(shdr: *const elf.Elf64_Shdr) bool {
3744 return shdrIsZerofill(shdr) and shdrIsTls(shdr);
3745}
3746
3747inline fn shdrIsZerofill(shdr: *const elf.Elf64_Shdr) bool {
3748 return shdr.sh_type == elf.SHT_NOBITS;
3749}
3750
3751pub inline fn shdrIsTls(shdr: *const elf.Elf64_Shdr) bool {
3752 return shdr.sh_flags & elf.SHF_TLS != 0;
3753}
3754
3755fn allocateSectionsInMemory(self: *Elf, base_offset: u64) !void {
3756 // We use this struct to track maximum alignment of all TLS sections.
3757 // According to https://github.com/rui314/mold/commit/bd46edf3f0fe9e1a787ea453c4657d535622e61f in mold,
3758 // in-file offsets have to be aligned against the start of TLS program header.
3759 // If that's not ensured, then in a multi-threaded context, TLS variables across a shared object
3760 // boundary may not get correctly loaded at an aligned address.
3761 const Align = struct {
3762 tls_start_align: u64 = 1,
3763 first_tls_index: ?usize = null,
3764
3765 inline fn isFirstTlsShdr(this: @This(), other: usize) bool {
3766 if (this.first_tls_index) |index| return index == other;
3767 return false;
3768 }
3769
3770 inline fn @"align"(this: @This(), index: usize, sh_addralign: u64, addr: u64) u64 {
3771 const alignment = if (this.isFirstTlsShdr(index)) this.tls_start_align else sh_addralign;
3772 return mem.alignForward(u64, addr, alignment);
3773 }
3774 };
3775
3776 var alignment = Align{};
3777 for (self.shdrs.items, 0..) |*shdr, i| {
3778 if (shdr.sh_type == elf.SHT_NULL) continue;
3779 if (!shdrIsTls(shdr)) continue;
3780 if (alignment.first_tls_index == null) alignment.first_tls_index = i;
3781 alignment.tls_start_align = @max(alignment.tls_start_align, shdr.sh_addralign);
3782 }
3783
3784 var addr = self.calcImageBase() + base_offset;
3785 var i: usize = 0;
3786 while (i < self.shdrs.items.len) : (i += 1) {
3787 const shdr = &self.shdrs.items[i];
3788 if (shdr.sh_type == elf.SHT_NULL) continue;
3789 if (!shdrIsAlloc(shdr)) continue;
3790 if (i > 0) {
3791 const prev_shdr = self.shdrs.items[i - 1];
3792 if (shdrToPhdrFlags(shdr.sh_flags) != shdrToPhdrFlags(prev_shdr.sh_flags)) {
3793 // We need to advance by page size
3794 addr += self.page_size;
3795 }
3796 }
3797 if (shdrIsTbss(shdr)) {
3798 // .tbss is a little special as it's used only by the loader meaning it doesn't
3799 // need to be actually mmap'ed at runtime. We still need to correctly increment
3800 // the addresses of every TLS zerofill section tho. Thus, we hack it so that
3801 // we increment the start address like normal, however, after we are done,
3802 // the next ALLOC section will get its start address allocated within the same
3803 // range as the .tbss sections. We will get something like this:
3804 //
3805 // ...
3806 // .tbss 0x10
3807 // .tcommon 0x20
3808 // .data 0x10
3809 // ...
3810 var tbss_addr = addr;
3811 while (i < self.shdrs.items.len and shdrIsTbss(&self.shdrs.items[i])) : (i += 1) {
3812 const tbss_shdr = &self.shdrs.items[i];
3813 tbss_addr = alignment.@"align"(i, tbss_shdr.sh_addralign, tbss_addr);
3814 tbss_shdr.sh_addr = tbss_addr;
3815 tbss_addr += tbss_shdr.sh_size;
3816 }
3817 i -= 1;
3818 continue;
3819 }
3820
3821 addr = alignment.@"align"(i, shdr.sh_addralign, addr);
3822 shdr.sh_addr = addr;
3823 addr += shdr.sh_size;
3824 }
3825}
3826
3827fn allocateSectionsInFile(self: *Elf, base_offset: u64) void {
3828 var offset = base_offset;
3829 var i: usize = 0;
3830 while (i < self.shdrs.items.len) {
3831 const first = &self.shdrs.items[i];
3832 defer if (!shdrIsAlloc(first) or shdrIsZerofill(first)) {
3833 i += 1;
3834 };
3835
3836 if (first.sh_type == elf.SHT_NULL) continue;
3837
3838 // Non-alloc sections don't need congruency with their allocated virtual memory addresses
3839 if (!shdrIsAlloc(first)) {
3840 first.sh_offset = mem.alignForward(u64, offset, first.sh_addralign);
3841 offset = first.sh_offset + first.sh_size;
3842 continue;
3843 }
3844 // Skip any zerofill section
3845 if (shdrIsZerofill(first)) continue;
3846
3847 // Set the offset to a value that is congruent with the section's allocated virtual memory address
3848 if (first.sh_addralign > self.page_size) {
3849 offset = mem.alignForward(u64, offset, first.sh_addralign);
3850 } else {
3851 const val = mem.alignBackward(u64, offset, self.page_size) + @rem(first.sh_addr, self.page_size);
3852 offset = if (offset <= val) val else val + self.page_size;
3853 }
3854
3855 while (true) {
3856 const prev = &self.shdrs.items[i];
3857 prev.sh_offset = offset + prev.sh_addr - first.sh_addr;
3858 i += 1;
3859
3860 const next = &self.shdrs.items[i];
3861 if (i >= self.shdrs.items.len or !shdrIsAlloc(next) or shdrIsZerofill(next)) break;
3862 if (next.sh_addr < first.sh_addr) break;
3863
3864 const gap = next.sh_addr - prev.sh_addr - prev.sh_size;
3865 if (gap >= self.page_size) break;
3866 }
3867
3868 const prev = &self.shdrs.items[i - 1];
3869 offset = prev.sh_offset + prev.sh_size;
3870
3871 // Skip any zerofill section
3872 while (i < self.shdrs.items.len and shdrIsAlloc(&self.shdrs.items[i]) and shdrIsZerofill(&self.shdrs.items[i])) : (i += 1) {}
3873 }
3874}
3875
3876fn allocateSections(self: *Elf) !void {
3877 while (true) {
3878 const nphdrs = self.phdrs.items.len;
3879 const base_offset: u64 = @sizeOf(elf.Elf64_Ehdr) + nphdrs * @sizeOf(elf.Elf64_Phdr);
3880 try self.allocateSectionsInMemory(base_offset);
3881 self.allocateSectionsInFile(base_offset);
3882 self.phdrs.clearRetainingCapacity();
3883 try self.initPhdrs();
3884 if (nphdrs == self.phdrs.items.len) break;
3885 }
3886}
3887
3888fn allocateAtoms(self: *Elf) void {
3889 for (self.objects.items) |index| {
3890 self.file(index).?.object.allocateAtoms(self);
36133891 }
36143892}
36153893
......@@ -3649,12 +3927,13 @@ fn updateSymtabSize(self: *Elf) !void {
36493927 .p32 => @sizeOf(elf.Elf32_Sym),
36503928 .p64 => @sizeOf(elf.Elf64_Sym),
36513929 };
3652 const sym_align: u16 = switch (self.ptr_width) {
3653 .p32 => @alignOf(elf.Elf32_Sym),
3654 .p64 => @alignOf(elf.Elf64_Sym),
3655 };
3930 // const sym_align: u16 = switch (self.ptr_width) {
3931 // .p32 => @alignOf(elf.Elf32_Sym),
3932 // .p64 => @alignOf(elf.Elf64_Sym),
3933 // };
36563934 const needed_size = (sizes.nlocals + sizes.nglobals + 1) * sym_size;
3657 try self.growNonAllocSection(self.symtab_section_index.?, needed_size, sym_align, false);
3935 shdr.sh_size = needed_size;
3936 // try self.growNonAllocSection(self.symtab_section_index.?, needed_size, sym_align, false);
36583937}
36593938
36603939fn writeSyntheticSections(self: *Elf) !void {
......@@ -4084,6 +4363,29 @@ pub fn isDynLib(self: Elf) bool {
40844363 return self.base.options.output_mode == .Lib and self.base.options.link_mode == .Dynamic;
40854364}
40864365
4366fn addPhdr(self: *Elf, opts: struct {
4367 type: u32 = 0,
4368 flags: u32 = 0,
4369 @"align": u64 = 0,
4370 offset: u64 = 0,
4371 addr: u64 = 0,
4372 filesz: u64 = 0,
4373 memsz: u64 = 0,
4374}) !u16 {
4375 const index = @as(u16, @intCast(self.phdrs.items.len));
4376 try self.phdrs.append(self.base.allocator, .{
4377 .p_type = opts.type,
4378 .p_flags = opts.flags,
4379 .p_offset = opts.offset,
4380 .p_vaddr = opts.addr,
4381 .p_paddr = opts.addr,
4382 .p_filesz = opts.filesz,
4383 .p_memsz = opts.memsz,
4384 .p_align = opts.@"align",
4385 });
4386 return index;
4387}
4388
40874389pub const AddSectionOpts = struct {
40884390 name: [:0]const u8,
40894391 type: u32 = elf.SHT_NULL,
src/link/Elf/Object.zig+29-3
......@@ -193,7 +193,7 @@ fn addAtom(
193193 }
194194}
195195
196pub fn getOutputSectionIndex(self: Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) error{OutOfMemory}!u16 {
196pub fn initOutputSection(self: Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) error{OutOfMemory}!u16 {
197197 const name = blk: {
198198 const name = self.strings.getAssumeExists(shdr.sh_name);
199199 if (shdr.sh_flags & elf.SHF_MERGE != 0) break :blk name;
......@@ -614,6 +614,32 @@ pub fn updateSectionSizes(self: Object, elf_file: *Elf) void {
614614 }
615615}
616616
617pub fn allocateAtoms(self: Object, elf_file: *Elf) void {
618 for (self.atoms.items) |atom_index| {
619 const atom = elf_file.atom(atom_index) orelse continue;
620 if (!atom.flags.alive) continue;
621 const shdr = elf_file.shdrs.items[atom.output_section_index];
622 atom.value += shdr.sh_addr;
623 }
624
625 for (self.locals()) |local_index| {
626 const local = elf_file.symbol(local_index);
627 const atom = local.atom(elf_file) orelse continue;
628 if (!atom.flags.alive) continue;
629 local.value += atom.value;
630 local.output_section_index = atom.output_section_index;
631 }
632
633 for (self.globals()) |global_index| {
634 const global = elf_file.symbol(global_index);
635 const atom = global.atom(elf_file) orelse continue;
636 if (!atom.flags.alive) continue;
637 if (global.file(elf_file).?.index() != self.index) continue;
638 global.value += atom.value;
639 global.output_section_index = atom.output_section_index;
640 }
641}
642
617643pub fn updateSymtabSize(self: *Object, elf_file: *Elf) void {
618644 for (self.locals()) |local_index| {
619645 const local = elf_file.symbol(local_index);
......@@ -664,12 +690,12 @@ pub fn writeSymtab(self: *Object, elf_file: *Elf, ctx: anytype) void {
664690 }
665691}
666692
667pub fn locals(self: *Object) []const Symbol.Index {
693pub fn locals(self: Object) []const Symbol.Index {
668694 const end = self.first_global orelse self.symbols.items.len;
669695 return self.symbols.items[0..end];
670696}
671697
672pub fn globals(self: *Object) []const Symbol.Index {
698pub fn globals(self: Object) []const Symbol.Index {
673699 const start = self.first_global orelse self.symbols.items.len;
674700 return self.symbols.items[start..];
675701}