authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-09-28 14:29:35+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-09-28 14:29:35+02:00
log1063035be6b5886d34b3ccd62680bbb52cb97a90
tree6ea55330ba299808321bd760fa1a7d3a6b0e7fec
parente0ef61d46d0e8a7142cb8c6ac80a6ef39ac8374e
parentdf285949f78661732031972599f720771a725241
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17304 from ziglang/elf-grow-vm-2

elf: grow segments in virtual memory if they exceed allocated capacity

6 files changed, 267 insertions(+), 106 deletions(-)

src/link/Elf.zig+186-79
......@@ -1,4 +1,5 @@
11base: link.File,
2
23dwarf: ?Dwarf = null,
34
45ptr_width: PtrWidth,
......@@ -103,6 +104,7 @@ phdr_table_dirty: bool = false,
103104shdr_table_dirty: bool = false,
104105shstrtab_dirty: bool = false,
105106strtab_dirty: bool = false,
107got_dirty: bool = false,
106108
107109debug_strtab_dirty: bool = false,
108110debug_abbrev_section_dirty: bool = false,
......@@ -411,21 +413,31 @@ fn findFreeSpace(self: *Elf, object_size: u64, min_alignment: u64) u64 {
411413const AllocateSegmentOpts = struct {
412414 size: u64,
413415 alignment: u64,
414 addr: ?u64 = null, // TODO find free VM space
416 addr: ?u64 = null,
415417 flags: u32 = elf.PF_R,
416418};
417419
418420pub fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}!u16 {
421 const gpa = self.base.allocator;
419422 const index = @as(u16, @intCast(self.phdrs.items.len));
420 try self.phdrs.ensureUnusedCapacity(self.base.allocator, 1);
423 try self.phdrs.ensureUnusedCapacity(gpa, 1);
421424 const off = self.findFreeSpace(opts.size, opts.alignment);
422 // Memory is always allocated in sequence.
423 // TODO is this correct? Or should we implement something similar to `findFreeSpace`?
424 // How would that impact HCS?
425 // Currently, we automatically allocate memory in sequence by finding the largest
426 // allocated virtual address and going from there.
427 // TODO we want to keep machine code segment in the furthest memory range among all
428 // segments as it is most likely to grow.
425429 const addr = opts.addr orelse blk: {
426 assert(self.phdr_table_load_index != null);
427 const phdr = &self.phdrs.items[index - 1];
428 break :blk mem.alignForward(u64, phdr.p_vaddr + phdr.p_memsz, opts.alignment);
430 const reserved_capacity = self.calcImageBase() * 4;
431 // Calculate largest VM address
432 const count = self.phdrs.items.len;
433 var addresses = std.ArrayList(u64).init(gpa);
434 defer addresses.deinit();
435 try addresses.ensureTotalCapacityPrecise(count);
436 for (self.phdrs.items) |phdr| {
437 addresses.appendAssumeCapacity(phdr.p_vaddr + reserved_capacity);
438 }
439 mem.sort(u64, addresses.items, {}, std.sort.asc(u64));
440 break :blk mem.alignForward(u64, addresses.items[count - 1], opts.alignment);
429441 };
430442 log.debug("allocating phdr({d})({c}{c}{c}) from 0x{x} to 0x{x} (0x{x} - 0x{x})", .{
431443 index,
......@@ -530,6 +542,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
530542 .p64 => false,
531543 };
532544 const ptr_size: u8 = self.ptrWidthBytes();
545 const is_linux = self.base.options.target.os.tag == .linux;
546 const large_addrspace = self.base.options.target.ptrBitWidth() >= 32;
533547 const image_base = self.calcImageBase();
534548
535549 if (self.phdr_table_index == null) {
......@@ -577,13 +591,10 @@ pub fn populateMissingMetadata(self: *Elf) !void {
577591 }
578592
579593 if (self.phdr_got_index == null) {
580 // TODO instead of hard coding the vaddr, make a function to find a vaddr to put things at.
581 // we'll need to re-use that function anyway, in case the GOT grows and overlaps something
582 // else in virtual memory.
583 const addr: u64 = if (self.base.options.target.ptrBitWidth() >= 32) 0x4000000 else 0x8000;
594 const addr: u64 = if (large_addrspace) 0x4000000 else 0x8000;
584595 // We really only need ptr alignment but since we are using PROGBITS, linux requires
585596 // page align.
586 const alignment = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
597 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
587598 self.phdr_got_index = try self.allocateSegment(.{
588599 .addr = addr,
589600 .size = @as(u64, ptr_size) * self.base.options.symbol_count_hint,
......@@ -593,10 +604,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
593604 }
594605
595606 if (self.phdr_load_ro_index == null) {
596 // TODO Same as for GOT
597 const addr: u64 = if (self.base.options.target.ptrBitWidth() >= 32) 0xc000000 else 0xa000;
598 // Same reason as for GOT
599 const alignment = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
607 const addr: u64 = if (large_addrspace) 0xc000000 else 0xa000;
608 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
600609 self.phdr_load_ro_index = try self.allocateSegment(.{
601610 .addr = addr,
602611 .size = 1024,
......@@ -606,10 +615,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
606615 }
607616
608617 if (self.phdr_load_rw_index == null) {
609 // TODO Same as for GOT
610 const addr: u64 = if (self.base.options.target.ptrBitWidth() >= 32) 0x10000000 else 0xc000;
611 // Same reason as for GOT
612 const alignment = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
618 const addr: u64 = if (large_addrspace) 0x10000000 else 0xc000;
619 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
613620 self.phdr_load_rw_index = try self.allocateSegment(.{
614621 .addr = addr,
615622 .size = 1024,
......@@ -619,9 +626,8 @@ pub fn populateMissingMetadata(self: *Elf) !void {
619626 }
620627
621628 if (self.phdr_load_zerofill_index == null) {
622 // TODO Same as for GOT
623 const addr: u64 = if (self.base.options.target.ptrBitWidth() >= 32) 0x14000000 else 0xf000;
624 const alignment = if (self.base.options.target.os.tag == .linux) self.page_size else @as(u16, ptr_size);
629 const addr: u64 = if (large_addrspace) 0x14000000 else 0xf000;
630 const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
625631 self.phdr_load_zerofill_index = try self.allocateSegment(.{
626632 .addr = addr,
627633 .size = 0,
......@@ -803,7 +809,7 @@ pub fn populateMissingMetadata(self: *Elf) !void {
803809 }
804810
805811 if (self.base.options.module) |module| {
806 if (self.zig_module_index == null) {
812 if (self.zig_module_index == null and !self.base.options.use_llvm) {
807813 const index = @as(File.Index, @intCast(try self.files.addOne(gpa)));
808814 self.files.set(index, .{ .zig_module = .{
809815 .index = index,
......@@ -858,7 +864,30 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {
858864 }
859865
860866 const mem_capacity = self.allocatedVirtualSize(phdr.p_vaddr);
861 assert(needed_size <= mem_capacity); // TODO grow section in virtual memory
867 if (needed_size > mem_capacity) {
868 // We are exceeding our allocated VM capacity so we need to shift everything in memory
869 // and grow.
870 {
871 const dirty_addr = phdr.p_vaddr + phdr.p_memsz;
872 self.got_dirty = for (self.got.entries.items) |entry| {
873 if (self.symbol(entry.symbol_index).value >= dirty_addr) break true;
874 } else false;
875
876 // TODO mark relocs dirty
877 }
878 try self.growSegment(shdr_index, needed_size);
879
880 if (self.zig_module_index != null) {
881 // TODO self-hosted backends cannot yet handle this condition correctly as the linker
882 // cannot update emitted virtual addresses of symbols already committed to the final file.
883 var err = try self.addErrorWithNotes(2);
884 try err.addMsg(self, "fatal linker error: cannot expand load segment phdr({d}) in virtual memory", .{
885 phdr_index,
886 });
887 try err.addNote(self, "TODO: emit relocations to memory locations in self-hosted backends", .{});
888 try err.addNote(self, "as a workaround, try increasing pre-allocated virtual memory of each segment", .{});
889 }
890 }
862891
863892 shdr.sh_size = needed_size;
864893 phdr.p_memsz = needed_size;
......@@ -870,6 +899,62 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {
870899 self.markDirty(shdr_index, phdr_index);
871900}
872901
902fn growSegment(self: *Elf, shndx: u16, needed_size: u64) !void {
903 const phdr_index = self.phdr_to_shdr_table.get(shndx).?;
904 const phdr = &self.phdrs.items[phdr_index];
905 const increased_size = padToIdeal(needed_size);
906 const end_addr = phdr.p_vaddr + phdr.p_memsz;
907 const old_aligned_end = phdr.p_vaddr + mem.alignForward(u64, phdr.p_memsz, phdr.p_align);
908 const new_aligned_end = phdr.p_vaddr + mem.alignForward(u64, increased_size, phdr.p_align);
909 const diff = new_aligned_end - old_aligned_end;
910 log.debug("growing phdr({d}) in memory by {x}", .{ phdr_index, diff });
911
912 // Update symbols and atoms.
913 var files = std.ArrayList(File.Index).init(self.base.allocator);
914 defer files.deinit();
915 try files.ensureTotalCapacityPrecise(self.objects.items.len + 1);
916
917 if (self.zig_module_index) |index| files.appendAssumeCapacity(index);
918 files.appendSliceAssumeCapacity(self.objects.items);
919
920 for (files.items) |index| {
921 const file_ptr = self.file(index).?;
922
923 for (file_ptr.locals()) |sym_index| {
924 const sym = self.symbol(sym_index);
925 const atom_ptr = sym.atom(self) orelse continue;
926 if (!atom_ptr.flags.alive or !atom_ptr.flags.allocated) continue;
927 if (sym.value >= end_addr) sym.value += diff;
928 }
929
930 for (file_ptr.globals()) |sym_index| {
931 const sym = self.symbol(sym_index);
932 if (sym.file_index != index) continue;
933 const atom_ptr = sym.atom(self) orelse continue;
934 if (!atom_ptr.flags.alive or !atom_ptr.flags.allocated) continue;
935 if (sym.value >= end_addr) sym.value += diff;
936 }
937
938 for (file_ptr.atoms()) |atom_index| {
939 const atom_ptr = self.atom(atom_index) orelse continue;
940 if (!atom_ptr.flags.alive or !atom_ptr.flags.allocated) continue;
941 if (atom_ptr.value >= end_addr) atom_ptr.value += diff;
942 }
943 }
944
945 // Finally, update section headers.
946 for (self.shdrs.items, 0..) |*other_shdr, other_shndx| {
947 if (other_shdr.sh_flags & elf.SHF_ALLOC == 0) continue;
948 if (other_shndx == shndx) continue;
949 const other_phdr_index = self.phdr_to_shdr_table.get(@intCast(other_shndx)) orelse continue;
950 const other_phdr = &self.phdrs.items[other_phdr_index];
951 if (other_phdr.p_vaddr < end_addr) continue;
952 other_shdr.sh_addr += diff;
953 other_phdr.p_vaddr += diff;
954 other_phdr.p_paddr += diff;
955 }
956}
957
873958pub fn growNonAllocSection(
874959 self: *Elf,
875960 shdr_index: u16,
......@@ -1143,7 +1228,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
11431228 if (self.zig_module_index) |index| {
11441229 for (self.file(index).?.zig_module.atoms.keys()) |atom_index| {
11451230 const atom_ptr = self.atom(atom_index).?;
1146 if (!atom_ptr.alive) continue;
1231 if (!atom_ptr.flags.alive) continue;
11471232 const shdr = &self.shdrs.items[atom_ptr.outputShndx().?];
11481233 const file_offset = shdr.sh_offset + atom_ptr.value - shdr.sh_addr;
11491234 const size = math.cast(usize, atom_ptr.size) orelse return error.Overflow;
......@@ -1157,6 +1242,15 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
11571242 }
11581243 try self.writeObjects();
11591244
1245 if (self.got_dirty) {
1246 const shdr = &self.shdrs.items[self.got_section_index.?];
1247 var buffer = try std.ArrayList(u8).initCapacity(gpa, self.got.size(self));
1248 defer buffer.deinit();
1249 try self.got.writeAllEntries(self, buffer.writer());
1250 try self.base.file.?.pwriteAll(buffer.items, shdr.sh_offset);
1251 self.got_dirty = false;
1252 }
1253
11601254 // Look for entry address in objects if not set by the incremental compiler.
11611255 if (self.entry_addr == null) {
11621256 const entry: ?[]const u8 = entry: {
......@@ -1537,7 +1631,7 @@ fn resolveSymbols(self: *Elf) error{Overflow}!void {
15371631 for (try object.comdatGroupMembers(cg.shndx)) |shndx| {
15381632 const atom_index = object.atoms.items[shndx];
15391633 if (self.atom(atom_index)) |atom_ptr| {
1540 atom_ptr.alive = false;
1634 atom_ptr.flags.alive = false;
15411635 // atom_ptr.markFdesDead(self);
15421636 }
15431637 }
......@@ -1645,25 +1739,26 @@ fn scanRelocs(self: *Elf) !void {
16451739fn allocateObjects(self: *Elf) !void {
16461740 for (self.objects.items) |index| {
16471741 const object = self.file(index).?.object;
1742
16481743 for (object.atoms.items) |atom_index| {
16491744 const atom_ptr = self.atom(atom_index) orelse continue;
1650 if (!atom_ptr.alive) continue;
1745 if (!atom_ptr.flags.alive or atom_ptr.flags.allocated) continue;
16511746 try atom_ptr.allocate(self);
16521747 }
16531748
16541749 for (object.locals()) |local_index| {
16551750 const local = self.symbol(local_index);
16561751 const atom_ptr = local.atom(self) orelse continue;
1657 if (!atom_ptr.alive) continue;
1658 local.value += atom_ptr.value;
1752 if (!atom_ptr.flags.alive) continue;
1753 local.value = local.elfSym(self).st_value + atom_ptr.value;
16591754 }
16601755
16611756 for (object.globals()) |global_index| {
16621757 const global = self.symbol(global_index);
16631758 const atom_ptr = global.atom(self) orelse continue;
1664 if (!atom_ptr.alive) continue;
1759 if (!atom_ptr.flags.alive) continue;
16651760 if (global.file_index == index) {
1666 global.value += atom_ptr.value;
1761 global.value = global.elfSym(self).st_value + atom_ptr.value;
16671762 }
16681763 }
16691764 }
......@@ -1676,7 +1771,7 @@ fn writeObjects(self: *Elf) !void {
16761771 const object = self.file(index).?.object;
16771772 for (object.atoms.items) |atom_index| {
16781773 const atom_ptr = self.atom(atom_index) orelse continue;
1679 if (!atom_ptr.alive) continue;
1774 if (!atom_ptr.flags.alive) continue;
16801775
16811776 const shdr = &self.shdrs.items[atom_ptr.outputShndx().?];
16821777 if (shdr.sh_type == elf.SHT_NOBITS) continue;
......@@ -2650,7 +2745,7 @@ fn updateDeclCode(
26502745 atom_ptr.output_section_index = shdr_index;
26512746
26522747 sym.name_offset = try self.strtab.insert(gpa, decl_name);
2653 atom_ptr.alive = true;
2748 atom_ptr.flags.alive = true;
26542749 atom_ptr.name_offset = sym.name_offset;
26552750 esym.st_name = sym.name_offset;
26562751 esym.st_info |= stt_bits;
......@@ -2681,11 +2776,6 @@ fn updateDeclCode(
26812776 } else {
26822777 try atom_ptr.allocate(self);
26832778 errdefer self.freeDeclMetadata(sym_index);
2684 log.debug("allocated atom for {s} at 0x{x} to 0x{x}", .{
2685 decl_name,
2686 atom_ptr.value,
2687 atom_ptr.value + atom_ptr.size,
2688 });
26892779
26902780 sym.value = atom_ptr.value;
26912781 esym.st_value = atom_ptr.value;
......@@ -2873,7 +2963,6 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol.
28732963 defer gpa.free(name);
28742964 break :blk try self.strtab.insert(gpa, name);
28752965 };
2876 const name = self.strtab.get(name_str_index).?;
28772966
28782967 const src = if (sym.ty.getOwnerDeclOrNull(mod)) |owner_decl|
28792968 mod.declPtr(owner_decl).srcLoc(mod)
......@@ -2913,7 +3002,7 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol.
29133002 local_esym.st_info |= elf.STT_OBJECT;
29143003 local_esym.st_size = code.len;
29153004 const atom_ptr = local_sym.atom(self).?;
2916 atom_ptr.alive = true;
3005 atom_ptr.flags.alive = true;
29173006 atom_ptr.name_offset = name_str_index;
29183007 atom_ptr.alignment = required_alignment;
29193008 atom_ptr.size = code.len;
......@@ -2922,12 +3011,6 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol.
29223011 try atom_ptr.allocate(self);
29233012 errdefer self.freeDeclMetadata(symbol_index);
29243013
2925 log.debug("allocated atom for {s} at 0x{x} to 0x{x}", .{
2926 name,
2927 atom_ptr.value,
2928 atom_ptr.value + atom_ptr.size,
2929 });
2930
29313014 local_sym.value = atom_ptr.value;
29323015 local_esym.st_value = atom_ptr.value;
29333016
......@@ -2961,7 +3044,6 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module
29613044 defer gpa.free(name);
29623045 break :blk try self.strtab.insert(gpa, name);
29633046 };
2964 const name = self.strtab.get(name_str_index).?;
29653047
29663048 const zig_module = self.file(self.zig_module_index.?).?.zig_module;
29673049 const sym_index = try zig_module.addAtom(self);
......@@ -2992,7 +3074,7 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module
29923074 local_esym.st_info |= elf.STT_OBJECT;
29933075 local_esym.st_size = code.len;
29943076 const atom_ptr = local_sym.atom(self).?;
2995 atom_ptr.alive = true;
3077 atom_ptr.flags.alive = true;
29963078 atom_ptr.name_offset = name_str_index;
29973079 atom_ptr.alignment = required_alignment;
29983080 atom_ptr.size = code.len;
......@@ -3001,8 +3083,6 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module
30013083 try atom_ptr.allocate(self);
30023084 errdefer self.freeDeclMetadata(sym_index);
30033085
3004 log.debug("allocated atom for {s} at 0x{x} to 0x{x}", .{ name, atom_ptr.value, atom_ptr.value + atom_ptr.size });
3005
30063086 local_sym.value = atom_ptr.value;
30073087 local_esym.st_value = atom_ptr.value;
30083088
......@@ -3931,6 +4011,50 @@ pub fn comdatGroupOwner(self: *Elf, index: ComdatGroupOwner.Index) *ComdatGroupO
39314011 return &self.comdat_groups_owners.items[index];
39324012}
39334013
4014const ErrorWithNotes = struct {
4015 /// Allocated index in misc_errors array.
4016 index: usize,
4017
4018 /// Next available note slot.
4019 note_slot: usize = 0,
4020
4021 pub fn addMsg(
4022 err: ErrorWithNotes,
4023 elf_file: *Elf,
4024 comptime format: []const u8,
4025 args: anytype,
4026 ) error{OutOfMemory}!void {
4027 const gpa = elf_file.base.allocator;
4028 const err_msg = &elf_file.misc_errors.items[err.index];
4029 err_msg.msg = try std.fmt.allocPrint(gpa, format, args);
4030 }
4031
4032 pub fn addNote(
4033 err: *ErrorWithNotes,
4034 elf_file: *Elf,
4035 comptime format: []const u8,
4036 args: anytype,
4037 ) error{OutOfMemory}!void {
4038 const gpa = elf_file.base.allocator;
4039 const err_msg = &elf_file.misc_errors.items[err.index];
4040 assert(err.note_slot < err_msg.notes.len);
4041 err_msg.notes[err.note_slot] = .{ .msg = try std.fmt.allocPrint(gpa, format, args) };
4042 err.note_slot += 1;
4043 }
4044};
4045
4046fn addErrorWithNotes(self: *Elf, note_count: usize) error{OutOfMemory}!ErrorWithNotes {
4047 try self.misc_errors.ensureUnusedCapacity(self.base.allocator, 1);
4048 return self.addErrorWithNotesAssumeCapacity(note_count);
4049}
4050
4051fn addErrorWithNotesAssumeCapacity(self: *Elf, note_count: usize) error{OutOfMemory}!ErrorWithNotes {
4052 const index = self.misc_errors.items.len;
4053 const err = self.misc_errors.addOneAssumeCapacity();
4054 err.* = .{ .msg = undefined, .notes = try self.base.allocator.alloc(link.File.ErrorMsg, note_count) };
4055 return .{ .index = index };
4056}
4057
39344058fn reportUndefined(self: *Elf, undefs: anytype) !void {
39354059 const gpa = self.base.allocator;
39364060 const max_notes = 4;
......@@ -3941,33 +4065,22 @@ fn reportUndefined(self: *Elf, undefs: anytype) !void {
39414065 while (it.next()) |entry| {
39424066 const undef_index = entry.key_ptr.*;
39434067 const atoms = entry.value_ptr.*.items;
3944 const nnotes = @min(atoms.len, max_notes);
4068 const natoms = @min(atoms.len, max_notes);
4069 const nnotes = natoms + @intFromBool(atoms.len > max_notes);
39454070
3946 var notes = try std.ArrayList(link.File.ErrorMsg).initCapacity(gpa, max_notes + 1);
3947 defer notes.deinit();
4071 var err = try self.addErrorWithNotesAssumeCapacity(nnotes);
4072 try err.addMsg(self, "undefined symbol: {s}", .{self.symbol(undef_index).name(self)});
39484073
3949 for (atoms[0..nnotes]) |atom_index| {
4074 for (atoms[0..natoms]) |atom_index| {
39504075 const atom_ptr = self.atom(atom_index).?;
39514076 const file_ptr = self.file(atom_ptr.file_index).?;
3952 const note = try std.fmt.allocPrint(gpa, "referenced by {s}:{s}", .{
3953 file_ptr.fmtPath(),
3954 atom_ptr.name(self),
3955 });
3956 notes.appendAssumeCapacity(.{ .msg = note });
4077 try err.addNote(self, "referenced by {s}:{s}", .{ file_ptr.fmtPath(), atom_ptr.name(self) });
39574078 }
39584079
39594080 if (atoms.len > max_notes) {
39604081 const remaining = atoms.len - max_notes;
3961 const note = try std.fmt.allocPrint(gpa, "referenced {d} more times", .{remaining});
3962 notes.appendAssumeCapacity(.{ .msg = note });
4082 try err.addNote(self, "referenced {d} more times", .{remaining});
39634083 }
3964
3965 var err_msg = link.File.ErrorMsg{
3966 .msg = try std.fmt.allocPrint(gpa, "undefined symbol: {s}", .{self.symbol(undef_index).name(self)}),
3967 };
3968 err_msg.notes = try notes.toOwnedSlice();
3969
3970 self.misc_errors.appendAssumeCapacity(err_msg);
39714084 }
39724085}
39734086
......@@ -4003,15 +4116,9 @@ fn reportParseError(
40034116 comptime format: []const u8,
40044117 args: anytype,
40054118) error{OutOfMemory}!void {
4006 const gpa = self.base.allocator;
4007 try self.misc_errors.ensureUnusedCapacity(gpa, 1);
4008 var notes = try gpa.alloc(link.File.ErrorMsg, 1);
4009 errdefer gpa.free(notes);
4010 notes[0] = .{ .msg = try std.fmt.allocPrint(gpa, "while parsing {s}", .{path}) };
4011 self.misc_errors.appendAssumeCapacity(.{
4012 .msg = try std.fmt.allocPrint(gpa, format, args),
4013 .notes = notes,
4014 });
4119 var err = try self.addErrorWithNotes(1);
4120 try err.addMsg(self, format, args);
4121 try err.addNote(self, "while parsing {s}", .{path});
40154122}
40164123
40174124fn fmtShdrs(self: *Elf) std.fmt.Formatter(formatShdrs) {
src/link/Elf/Atom.zig+37-16
......@@ -25,11 +25,8 @@ relocs_section_index: Index = 0,
2525/// Index of this atom in the linker's atoms table.
2626atom_index: Index = 0,
2727
28/// Specifies whether this atom is alive or has been garbage collected.
29alive: bool = false,
30
31/// Specifies if the atom has been visited during garbage collection.
32visited: bool = false,
28/// Flags we use for state tracking.
29flags: Flags = .{},
3330
3431/// Start index of FDEs referencing this atom.
3532fde_start: u32 = 0,
......@@ -48,8 +45,12 @@ pub fn name(self: Atom, elf_file: *Elf) []const u8 {
4845 return elf_file.strtab.getAssumeExists(self.name_offset);
4946}
5047
48pub fn file(self: Atom, elf_file: *Elf) ?File {
49 return elf_file.file(self.file_index);
50}
51
5152pub fn inputShdr(self: Atom, elf_file: *Elf) elf.Elf64_Shdr {
52 const object = elf_file.file(self.file_index).?.object;
53 const object = self.file(elf_file).?.object;
5354 return object.shdrs.items[self.input_section_index];
5455}
5556
......@@ -59,7 +60,7 @@ pub fn outputShndx(self: Atom) ?u16 {
5960}
6061
6162pub fn codeInObject(self: Atom, elf_file: *Elf) error{Overflow}![]const u8 {
62 const object = elf_file.file(self.file_index).?.object;
63 const object = self.file(elf_file).?.object;
6364 return object.shdrContents(self.input_section_index);
6465}
6566
......@@ -91,7 +92,7 @@ pub fn codeInObjectUncompressAlloc(self: Atom, elf_file: *Elf) ![]u8 {
9192}
9293
9394pub fn priority(self: Atom, elf_file: *Elf) u64 {
94 const index = elf_file.file(self.file_index).?.index();
95 const index = self.file(elf_file).?.index();
9596 return (@as(u64, @intCast(index)) << 32) | @as(u64, @intCast(self.input_section_index));
9697}
9798
......@@ -178,6 +179,13 @@ pub fn allocate(self: *Atom, elf_file: *Elf) !void {
178179 }
179180 };
180181
182 log.debug("allocated atom({d}) : '{s}' at 0x{x} to 0x{x}", .{
183 self.atom_index,
184 self.name(elf_file),
185 self.value,
186 self.value + self.size,
187 });
188
181189 const expand_section = if (atom_placement) |placement_index|
182190 elf_file.atom(placement_index).?.next_index == 0
183191 else
......@@ -222,6 +230,8 @@ pub fn allocate(self: *Atom, elf_file: *Elf) !void {
222230 if (free_list_removal) |i| {
223231 _ = free_list.swapRemove(i);
224232 }
233
234 self.flags.allocated = true;
225235}
226236
227237pub fn shrink(self: *Atom, elf_file: *Elf) void {
......@@ -238,7 +248,7 @@ pub fn free(self: *Atom, elf_file: *Elf) void {
238248 log.debug("freeAtom {d} ({s})", .{ self.atom_index, self.name(elf_file) });
239249
240250 const gpa = elf_file.base.allocator;
241 const zig_module = elf_file.file(self.file_index).?.zig_module;
251 const zig_module = self.file(elf_file).?.zig_module;
242252 const shndx = self.outputShndx().?;
243253 const meta = elf_file.last_atom_and_free_list_table.getPtr(shndx).?;
244254 const free_list = &meta.free_list;
......@@ -294,7 +304,7 @@ pub fn free(self: *Atom, elf_file: *Elf) void {
294304}
295305
296306pub fn relocs(self: Atom, elf_file: *Elf) error{Overflow}![]align(1) const elf.Elf64_Rela {
297 return switch (elf_file.file(self.file_index).?) {
307 return switch (self.file(elf_file).?) {
298308 .zig_module => |x| x.relocs.items[self.relocs_section_index].items,
299309 .object => |x| x.getRelocs(self.relocs_section_index),
300310 else => unreachable,
......@@ -303,7 +313,7 @@ pub fn relocs(self: Atom, elf_file: *Elf) error{Overflow}![]align(1) const elf.E
303313
304314pub fn addReloc(self: Atom, elf_file: *Elf, reloc: elf.Elf64_Rela) !void {
305315 const gpa = elf_file.base.allocator;
306 const file_ptr = elf_file.file(self.file_index).?;
316 const file_ptr = self.file(elf_file).?;
307317 assert(file_ptr == .zig_module);
308318 const zig_module = file_ptr.zig_module;
309319 const rels = &zig_module.relocs.items[self.relocs_section_index];
......@@ -311,14 +321,14 @@ pub fn addReloc(self: Atom, elf_file: *Elf, reloc: elf.Elf64_Rela) !void {
311321}
312322
313323pub fn freeRelocs(self: Atom, elf_file: *Elf) void {
314 const file_ptr = elf_file.file(self.file_index).?;
324 const file_ptr = self.file(elf_file).?;
315325 assert(file_ptr == .zig_module);
316326 const zig_module = file_ptr.zig_module;
317327 zig_module.relocs.items[self.relocs_section_index].clearRetainingCapacity();
318328}
319329
320330pub fn scanRelocs(self: Atom, elf_file: *Elf, undefs: anytype) !void {
321 const file_ptr = elf_file.file(self.file_index).?;
331 const file_ptr = self.file(elf_file).?;
322332 const rels = try self.relocs(elf_file);
323333 var i: usize = 0;
324334 while (i < rels.len) : (i += 1) {
......@@ -392,7 +402,7 @@ fn reportUndefined(
392402 rel: elf.Elf64_Rela,
393403 undefs: anytype,
394404) !void {
395 const rel_esym = switch (elf_file.file(self.file_index).?) {
405 const rel_esym = switch (self.file(elf_file).?) {
396406 .zig_module => |x| x.elfSym(rel.r_sym()).*,
397407 .object => |x| x.symtab[rel.r_sym()],
398408 else => unreachable,
......@@ -416,7 +426,7 @@ fn reportUndefined(
416426pub fn resolveRelocs(self: Atom, elf_file: *Elf, code: []u8) !void {
417427 relocs_log.debug("0x{x}: {s}", .{ self.value, self.name(elf_file) });
418428
419 const file_ptr = elf_file.file(self.file_index).?;
429 const file_ptr = self.file(elf_file).?;
420430 var stream = std.io.fixedBufferStream(code);
421431 const cwriter = stream.writer();
422432
......@@ -619,7 +629,7 @@ fn format2(
619629 // try writer.writeAll(" }");
620630 // }
621631 const gc_sections = if (elf_file.base.options.gc_sections) |gc_sections| gc_sections else false;
622 if (gc_sections and !atom.alive) {
632 if (gc_sections and !atom.flags.alive) {
623633 try writer.writeAll(" : [*]");
624634 }
625635}
......@@ -629,6 +639,17 @@ fn format2(
629639// future.
630640pub const Index = u16;
631641
642pub const Flags = packed struct {
643 /// Specifies whether this atom is alive or has been garbage collected.
644 alive: bool = false,
645
646 /// Specifies if the atom has been visited during garbage collection.
647 visited: bool = false,
648
649 /// Specifies whether this atom has been allocated in the output section.
650 allocated: bool = false,
651};
652
632653const x86_64 = struct {
633654 pub fn relaxGotpcrelx(code: []u8) !void {
634655 const old_inst = disassemble(code) orelse return error.RelaxFail;
src/link/Elf/Object.zig+6-6
......@@ -180,7 +180,7 @@ fn addAtom(
180180 atom.file_index = self.index;
181181 atom.input_section_index = shndx;
182182 atom.output_section_index = try self.getOutputSectionIndex(elf_file, shdr);
183 atom.alive = true;
183 atom.flags.alive = true;
184184 self.atoms.items[shndx] = atom_index;
185185
186186 if (shdr.sh_flags & elf.SHF_COMPRESSED != 0) {
......@@ -424,7 +424,7 @@ fn filterRelocs(
424424pub fn scanRelocs(self: *Object, elf_file: *Elf, undefs: anytype) !void {
425425 for (self.atoms.items) |atom_index| {
426426 const atom = elf_file.atom(atom_index) orelse continue;
427 if (!atom.alive) continue;
427 if (!atom.flags.alive) continue;
428428 const shdr = atom.inputShdr(elf_file);
429429 if (shdr.sh_flags & elf.SHF_ALLOC == 0) continue;
430430 if (shdr.sh_type == elf.SHT_NOBITS) continue;
......@@ -458,7 +458,7 @@ pub fn resolveSymbols(self: *Object, elf_file: *Elf) void {
458458 if (esym.st_shndx != elf.SHN_ABS and esym.st_shndx != elf.SHN_COMMON) {
459459 const atom_index = self.atoms.items[esym.st_shndx];
460460 const atom = elf_file.atom(atom_index) orelse continue;
461 if (!atom.alive) continue;
461 if (!atom.flags.alive) continue;
462462 }
463463
464464 const global = elf_file.symbol(index);
......@@ -553,7 +553,7 @@ pub fn checkDuplicates(self: *Object, elf_file: *Elf) void {
553553 if (this_sym.st_shndx != elf.SHN_ABS) {
554554 const atom_index = self.atoms.items[this_sym.st_shndx];
555555 const atom = elf_file.atom(atom_index) orelse continue;
556 if (!atom.alive) continue;
556 if (!atom.flags.alive) continue;
557557 }
558558
559559 elf_file.base.fatal("multiple definition: {}: {}: {s}", .{
......@@ -628,7 +628,7 @@ pub fn convertCommonSymbols(self: *Object, elf_file: *Elf) !void {
628628pub fn updateSymtabSize(self: *Object, elf_file: *Elf) void {
629629 for (self.locals()) |local_index| {
630630 const local = elf_file.symbol(local_index);
631 if (local.atom(elf_file)) |atom| if (!atom.alive) continue;
631 if (local.atom(elf_file)) |atom| if (!atom.flags.alive) continue;
632632 const esym = local.elfSym(elf_file);
633633 switch (esym.st_type()) {
634634 elf.STT_SECTION, elf.STT_NOTYPE => continue,
......@@ -641,7 +641,7 @@ pub fn updateSymtabSize(self: *Object, elf_file: *Elf) void {
641641 for (self.globals()) |global_index| {
642642 const global = elf_file.symbol(global_index);
643643 if (global.file(elf_file)) |file| if (file.index() != self.index) continue;
644 if (global.atom(elf_file)) |atom| if (!atom.alive) continue;
644 if (global.atom(elf_file)) |atom| if (!atom.flags.alive) continue;
645645 global.flags.output_symtab = true;
646646 if (global.isLocal()) {
647647 self.output_symtab_size.nlocals += 1;
src/link/Elf/ZigModule.zig+7-5
......@@ -53,17 +53,19 @@ pub fn addAtom(self: *ZigModule, elf_file: *Elf) !Symbol.Index {
5353 const gpa = elf_file.base.allocator;
5454
5555 const atom_index = try elf_file.addAtom();
56 const symbol_index = try elf_file.addSymbol();
57 const esym_index = try self.addLocalEsym(gpa);
58
5659 try self.atoms.putNoClobber(gpa, atom_index, {});
60 try self.local_symbols.append(gpa, symbol_index);
61
5762 const atom_ptr = elf_file.atom(atom_index).?;
5863 atom_ptr.file_index = self.index;
5964
60 const symbol_index = try elf_file.addSymbol();
61 try self.local_symbols.append(gpa, symbol_index);
6265 const symbol_ptr = elf_file.symbol(symbol_index);
6366 symbol_ptr.file_index = self.index;
6467 symbol_ptr.atom_index = atom_index;
6568
66 const esym_index = try self.addLocalEsym(gpa);
6769 const esym = &self.local_esyms.items[esym_index];
6870 esym.st_shndx = atom_index;
6971 symbol_ptr.esym_index = esym_index;
......@@ -86,7 +88,7 @@ pub fn resolveSymbols(self: *ZigModule, elf_file: *Elf) void {
8688 if (esym.st_shndx != elf.SHN_ABS and esym.st_shndx != elf.SHN_COMMON) {
8789 const atom_index = esym.st_shndx;
8890 const atom = elf_file.atom(atom_index) orelse continue;
89 if (!atom.alive) continue;
91 if (!atom.flags.alive) continue;
9092 }
9193
9294 const global = elf_file.symbol(index);
......@@ -141,7 +143,7 @@ pub fn claimUnresolved(self: *ZigModule, elf_file: *Elf) void {
141143pub fn scanRelocs(self: *ZigModule, elf_file: *Elf, undefs: anytype) !void {
142144 for (self.atoms.keys()) |atom_index| {
143145 const atom = elf_file.atom(atom_index) orelse continue;
144 if (!atom.alive) continue;
146 if (!atom.flags.alive) continue;
145147 try atom.scanRelocs(elf_file, undefs);
146148 }
147149}
src/link/Elf/file.zig+16
......@@ -89,6 +89,21 @@ pub const File = union(enum) {
8989 }
9090 }
9191
92 pub fn atoms(file: File) []const Atom.Index {
93 return switch (file) {
94 .linker_defined => unreachable,
95 .zig_module => |x| x.atoms.keys(),
96 .object => |x| x.atoms.items,
97 };
98 }
99
100 pub fn locals(file: File) []const Symbol.Index {
101 return switch (file) {
102 .linker_defined => unreachable,
103 inline else => |x| x.locals(),
104 };
105 }
106
92107 pub fn globals(file: File) []const Symbol.Index {
93108 return switch (file) {
94109 inline else => |x| x.globals(),
......@@ -110,6 +125,7 @@ const std = @import("std");
110125const elf = std.elf;
111126
112127const Allocator = std.mem.Allocator;
128const Atom = @import("Atom.zig");
113129const Elf = @import("../Elf.zig");
114130const LinkerDefined = @import("LinkerDefined.zig");
115131const Object = @import("Object.zig");
src/link/Elf/synthetic_sections.zig+15
......@@ -169,6 +169,21 @@ pub const GotSection = struct {
169169 }
170170 }
171171
172 pub fn writeAllEntries(got: GotSection, elf_file: *Elf, writer: anytype) !void {
173 assert(!got.dirty);
174 const entry_size: u16 = elf_file.archPtrWidthBytes();
175 const endian = elf_file.base.options.target.cpu.arch.endian();
176 for (got.entries.items) |entry| {
177 const value = elf_file.symbol(entry.symbol_index).value;
178 switch (entry_size) {
179 2 => try writer.writeInt(u16, @intCast(value), endian),
180 4 => try writer.writeInt(u32, @intCast(value), endian),
181 8 => try writer.writeInt(u64, @intCast(value), endian),
182 else => unreachable,
183 }
184 }
185 }
186
172187 // pub fn write(got: GotSection, elf_file: *Elf, writer: anytype) !void {
173188 // const is_shared = elf_file.options.output_mode == .lib;
174189 // const apply_relocs = elf_file.options.apply_dynamic_relocs;