Commit 7be983ac9217a596b7f35e7ef4c49fda0270e10b | zig

author	Jakub Konka <kubkon@jakubkonka.com> 2023-10-13 14:54:52+02:00
committer	Jakub Konka <kubkon@jakubkonka.com> 2023-10-16 19:33:05+02:00
log	7be983ac9217a596b7f35e7ef4c49fda0270e10b
tree	358dd3cf5de8c7c412febd9d77c487f9d7c3114e
parent	9b6337ab06e90b23c7d494b49fb8a728ac9d75e2

elf: create new synthetic section ZigGotSection

10 files changed, 341 insertions(+), 122 deletions(-)

src
- arch
  - aarch64
    CodeGen.zig
  - arm
    CodeGen.zig
  - riscv64
    CodeGen.zig
  - sparc64
    CodeGen.zig
  - x86_64
    CodeGen.zig
- codegen.zig
- link
  - Elf.zig
  - Elf
    Atom.zig
    Symbol.zig
    synthetic_sections.zig

src/arch/aarch64/CodeGen.zig+2-2

...	...	@@ -4318,8 +4318,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4318	4318	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
4319	4319	const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl);
4320	4320	const sym = elf_file.symbol(sym_index);
4321		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
4322		const got_addr = @as(u32, @intCast(sym.gotAddress(elf_file)));
	4321	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
	4322	const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file)));
4323	4323	try self.genSetReg(Type.usize, .x30, .{ .memory = got_addr });
4324	4324	} else if (self.bin_file.cast(link.File.MachO)) \|macho_file\| {
4325	4325	const atom = try macho_file.getOrCreateAtomForDecl(func.owner_decl);

src/arch/arm/CodeGen.zig+2-2

...	...	@@ -4304,8 +4304,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
4304	4304	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
4305	4305	const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl);
4306	4306	const sym = elf_file.symbol(sym_index);
4307		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
4308		const got_addr = @as(u32, @intCast(sym.gotAddress(elf_file)));
	4307	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
	4308	const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file)));
4309	4309	try self.genSetReg(Type.usize, .lr, .{ .memory = got_addr });
4310	4310	} else if (self.bin_file.cast(link.File.MachO)) \|_\| {
4311	4311	unreachable; // unsupported architecture for MachO

src/arch/riscv64/CodeGen.zig+2-2

...	...	@@ -1754,8 +1754,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1754	1754	.func => \|func\| {
1755	1755	const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl);
1756	1756	const sym = elf_file.symbol(sym_index);
1757		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
1758		const got_addr = @as(u32, @intCast(sym.gotAddress(elf_file)));
	1757	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
	1758	const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file)));
1759	1759	try self.genSetReg(Type.usize, .ra, .{ .memory = got_addr });
1760	1760	_ = try self.addInst(.{
1761	1761	.tag = .jalr,

src/arch/sparc64/CodeGen.zig+2-2

...	...	@@ -1349,8 +1349,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier
1349	1349	const got_addr = if (self.bin_file.cast(link.File.Elf)) \|elf_file\| blk: {
1350	1350	const sym_index = try elf_file.getOrCreateMetadataForDecl(func.owner_decl);
1351	1351	const sym = elf_file.symbol(sym_index);
1352		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
1353		break :blk @as(u32, @intCast(sym.gotAddress(elf_file)));
	1352	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
	1353	break :blk @as(u32, @intCast(sym.zigGotAddress(elf_file)));
1354	1354	} else unreachable;
1355	1355
1356	1356	try self.genSetReg(Type.usize, .o7, .{ .memory = got_addr });

src/arch/x86_64/CodeGen.zig+2-4

...	...	@@ -9189,8 +9189,7 @@ fn genCall(self: *Self, info: union(enum) {
9189	9189	if (self.bin_file.cast(link.File.Elf)) \|elf_file\| {
9190	9190	const sym_index = try elf_file.getOrCreateMetadataForDecl(owner_decl);
9191	9191	const sym = elf_file.symbol(sym_index);
9192		sym.flags.needs_got = true;
9193		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
	9192	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
9194	9193	_ = try self.addInst(.{
9195	9194	.tag = .call,
9196	9195	.ops = .direct_got_reloc,
...	...	@@ -11637,8 +11636,7 @@ fn genLazySymbolRef(
11637	11636	const sym_index = elf_file.getOrCreateMetadataForLazySymbol(lazy_sym) catch \|err\|
11638	11637	return self.fail("{s} creating lazy symbol", .{@errorName(err)});
11639	11638	const sym = elf_file.symbol(sym_index);
11640		sym.flags.needs_got = true;
11641		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
	11639	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
11642	11640	const reloc = Mir.Reloc{
11643	11641	.atom_index = try self.owner.getSymbolIndex(self),
11644	11642	.sym_index = sym.esym_index,

src/codegen.zig+2-3

...	...	@@ -889,9 +889,8 @@ fn genDeclRef(
889	889	if (bin_file.cast(link.File.Elf)) \|elf_file\| {
890	890	const sym_index = try elf_file.getOrCreateMetadataForDecl(decl_index);
891	891	const sym = elf_file.symbol(sym_index);
892		sym.flags.needs_got = true;
893		_ = try sym.getOrCreateGotEntry(sym_index, elf_file);
894		return GenResult.mcv(.{ .memory = sym.gotAddress(elf_file) });
	892	_ = try sym.getOrCreateZigGotEntry(sym_index, elf_file);
	893	return GenResult.mcv(.{ .memory = sym.zigGotAddress(elf_file) });
895	894	} else if (bin_file.cast(link.File.MachO)) \|macho_file\| {
896	895	const atom_index = try macho_file.getOrCreateAtomForDecl(decl_index);
897	896	const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?;

src/link/Elf.zig+95-81

...	...	@@ -33,16 +33,16 @@ phdrs: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{},
33	33
34	34	/// Tracked loadable segments during incremental linking.
35	35	/// The index into the program headers of a PT_LOAD program header with Read and Execute flags
36		phdr_load_re_zig_index: ?u16 = null,
	36	phdr_zig_load_re_index: ?u16 = null,
37	37	/// The index into the program headers of the global offset table.
38	38	/// It needs PT_LOAD and Read flags.
39		phdr_got_zig_index: ?u16 = null,
	39	phdr_zig_got_index: ?u16 = null,
40	40	/// The index into the program headers of a PT_LOAD program header with Read flag
41		phdr_load_ro_zig_index: ?u16 = null,
	41	phdr_zig_load_ro_index: ?u16 = null,
42	42	/// The index into the program headers of a PT_LOAD program header with Write flag
43		phdr_load_rw_zig_index: ?u16 = null,
	43	phdr_zig_load_rw_index: ?u16 = null,
44	44	/// The index into the program headers of a PT_LOAD program header with zerofill data.
45		phdr_load_zerofill_zig_index: ?u16 = null,
	45	phdr_zig_load_zerofill_index: ?u16 = null,
46	46
47	47	/// Special program headers
48	48	/// PT_PHDR
...	...	@@ -99,13 +99,15 @@ plt_got: PltGotSection = .{},
99	99	copy_rel: CopyRelSection = .{},
100	100	/// .rela.plt section
101	101	rela_plt: std.ArrayListUnmanaged(elf.Elf64_Rela) = .{},
	102	/// .zig.got section
	103	zig_got: ZigGotSection = .{},
102	104
103	105	/// Tracked section headers with incremental updates to Zig module
104		text_zig_section_index: ?u16 = null,
105		rodata_zig_section_index: ?u16 = null,
106		data_zig_section_index: ?u16 = null,
107		bss_zig_section_index: ?u16 = null,
108		got_zig_section_index: ?u16 = null,
	106	zig_text_section_index: ?u16 = null,
	107	zig_rodata_section_index: ?u16 = null,
	108	zig_data_section_index: ?u16 = null,
	109	zig_bss_section_index: ?u16 = null,
	110	zig_got_section_index: ?u16 = null,
109	111	debug_info_section_index: ?u16 = null,
110	112	debug_abbrev_section_index: ?u16 = null,
111	113	debug_str_section_index: ?u16 = null,
...	...	@@ -479,7 +481,7 @@ pub fn lowerAnonDecl(self: *Elf, decl_val: InternPool.Index, src_loc: Module.Src
479	481	const tv = TypedValue{ .ty = ty, .val = val };
480	482	const name = try std.fmt.allocPrint(gpa, "__anon_{d}", .{@intFromEnum(decl_val)});
481	483	defer gpa.free(name);
482		const res = self.lowerConst(name, tv, self.rodata_zig_section_index.?, src_loc) catch \|err\| switch (err) {
	484	const res = self.lowerConst(name, tv, self.zig_rodata_section_index.?, src_loc) catch \|err\| switch (err) {
483	485	else => {
484	486	// TODO improve error message
485	487	const em = try Module.ErrorMsg.create(gpa, src_loc, "lowerAnonDecl failed with error: {s}", .{
...	...	@@ -687,8 +689,8 @@ pub fn initMetadata(self: *Elf) !void {
687	689	const ptr_bit_width = self.base.options.target.ptrBitWidth();
688	690	const is_linux = self.base.options.target.os.tag == .linux;
689	691
690		if (self.phdr_load_re_zig_index == null) {
691		self.phdr_load_re_zig_index = try self.allocateSegment(.{
	692	if (self.phdr_zig_load_re_index == null) {
	693	self.phdr_zig_load_re_index = try self.allocateSegment(.{
692	694	.addr = if (ptr_bit_width >= 32) 0x8000000 else 0x8000,
693	695	.memsz = self.base.options.program_code_size_hint,
694	696	.filesz = self.base.options.program_code_size_hint,
...	...	@@ -697,11 +699,11 @@ pub fn initMetadata(self: *Elf) !void {
697	699	});
698	700	}
699	701
700		if (self.phdr_got_zig_index == null) {
	702	if (self.phdr_zig_got_index == null) {
701	703	// We really only need ptr alignment but since we are using PROGBITS, linux requires
702	704	// page align.
703	705	const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
704		self.phdr_got_zig_index = try self.allocateSegment(.{
	706	self.phdr_zig_got_index = try self.allocateSegment(.{
705	707	.addr = if (ptr_bit_width >= 32) 0x4000000 else 0x4000,
706	708	.memsz = @as(u64, ptr_size) * self.base.options.symbol_count_hint,
707	709	.filesz = @as(u64, ptr_size) * self.base.options.symbol_count_hint,
...	...	@@ -710,9 +712,9 @@ pub fn initMetadata(self: *Elf) !void {
710	712	});
711	713	}
712	714
713		if (self.phdr_load_ro_zig_index == null) {
	715	if (self.phdr_zig_load_ro_index == null) {
714	716	const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
715		self.phdr_load_ro_zig_index = try self.allocateSegment(.{
	717	self.phdr_zig_load_ro_index = try self.allocateSegment(.{
716	718	.addr = if (ptr_bit_width >= 32) 0xc000000 else 0xa000,
717	719	.memsz = 1024,
718	720	.filesz = 1024,
...	...	@@ -721,9 +723,9 @@ pub fn initMetadata(self: *Elf) !void {
721	723	});
722	724	}
723	725
724		if (self.phdr_load_rw_zig_index == null) {
	726	if (self.phdr_zig_load_rw_index == null) {
725	727	const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
726		self.phdr_load_rw_zig_index = try self.allocateSegment(.{
	728	self.phdr_zig_load_rw_index = try self.allocateSegment(.{
727	729	.addr = if (ptr_bit_width >= 32) 0x10000000 else 0xc000,
728	730	.memsz = 1024,
729	731	.filesz = 1024,
...	...	@@ -732,64 +734,64 @@ pub fn initMetadata(self: *Elf) !void {
732	734	});
733	735	}
734	736
735		if (self.phdr_load_zerofill_zig_index == null) {
	737	if (self.phdr_zig_load_zerofill_index == null) {
736	738	const alignment = if (is_linux) self.page_size else @as(u16, ptr_size);
737		self.phdr_load_zerofill_zig_index = try self.allocateSegment(.{
	739	self.phdr_zig_load_zerofill_index = try self.allocateSegment(.{
738	740	.addr = if (ptr_bit_width >= 32) 0x14000000 else 0xf000,
739	741	.memsz = 0,
740	742	.filesz = 0,
741	743	.alignment = alignment,
742	744	.flags = elf.PF_R \| elf.PF_W,
743	745	});
744		const phdr = &self.phdrs.items[self.phdr_load_zerofill_zig_index.?];
745		phdr.p_offset = self.phdrs.items[self.phdr_load_rw_zig_index.?].p_offset; // .bss overlaps .data
	746	const phdr = &self.phdrs.items[self.phdr_zig_load_zerofill_index.?];
	747	phdr.p_offset = self.phdrs.items[self.phdr_zig_load_rw_index.?].p_offset; // .bss overlaps .data
746	748	phdr.p_memsz = 1024;
747	749	}
748	750
749		if (self.text_zig_section_index == null) {
750		self.text_zig_section_index = try self.allocateAllocSection(.{
751		.name = ".text.zig",
752		.phdr_index = self.phdr_load_re_zig_index.?,
	751	if (self.zig_text_section_index == null) {
	752	self.zig_text_section_index = try self.allocateAllocSection(.{
	753	.name = ".zig.text",
	754	.phdr_index = self.phdr_zig_load_re_index.?,
753	755	.flags = elf.SHF_ALLOC \| elf.SHF_EXECINSTR,
754	756	});
755		try self.last_atom_and_free_list_table.putNoClobber(gpa, self.text_zig_section_index.?, .{});
	757	try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_text_section_index.?, .{});
756	758	}
757	759
758		if (self.got_zig_section_index == null) {
759		self.got_zig_section_index = try self.allocateAllocSection(.{
760		.name = ".got.zig",
761		.phdr_index = self.phdr_got_zig_index.?,
	760	if (self.zig_got_section_index == null) {
	761	self.zig_got_section_index = try self.allocateAllocSection(.{
	762	.name = ".zig.got",
	763	.phdr_index = self.phdr_zig_got_index.?,
762	764	.alignment = ptr_size,
763	765	});
764	766	}
765	767
766		if (self.rodata_zig_section_index == null) {
767		self.rodata_zig_section_index = try self.allocateAllocSection(.{
768		.name = ".rodata.zig",
769		.phdr_index = self.phdr_load_ro_zig_index.?,
	768	if (self.zig_rodata_section_index == null) {
	769	self.zig_rodata_section_index = try self.allocateAllocSection(.{
	770	.name = ".zig.rodata",
	771	.phdr_index = self.phdr_zig_load_ro_index.?,
770	772	});
771		try self.last_atom_and_free_list_table.putNoClobber(gpa, self.rodata_zig_section_index.?, .{});
	773	try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_rodata_section_index.?, .{});
772	774	}
773	775
774		if (self.data_zig_section_index == null) {
775		self.data_zig_section_index = try self.allocateAllocSection(.{
776		.name = ".data.zig",
777		.phdr_index = self.phdr_load_rw_zig_index.?,
	776	if (self.zig_data_section_index == null) {
	777	self.zig_data_section_index = try self.allocateAllocSection(.{
	778	.name = ".zig.data",
	779	.phdr_index = self.phdr_zig_load_rw_index.?,
778	780	.alignment = ptr_size,
779	781	.flags = elf.SHF_ALLOC \| elf.SHF_WRITE,
780	782	});
781		try self.last_atom_and_free_list_table.putNoClobber(gpa, self.data_zig_section_index.?, .{});
	783	try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_data_section_index.?, .{});
782	784	}
783	785
784		if (self.bss_zig_section_index == null) {
785		self.bss_zig_section_index = try self.allocateAllocSection(.{
	786	if (self.zig_bss_section_index == null) {
	787	self.zig_bss_section_index = try self.allocateAllocSection(.{
786	788	.name = ".bss.zig",
787		.phdr_index = self.phdr_load_zerofill_zig_index.?,
	789	.phdr_index = self.phdr_zig_load_zerofill_index.?,
788	790	.alignment = ptr_size,
789	791	.flags = elf.SHF_ALLOC \| elf.SHF_WRITE,
790	792	.type = elf.SHT_NOBITS,
791	793	});
792		try self.last_atom_and_free_list_table.putNoClobber(gpa, self.bss_zig_section_index.?, .{});
	794	try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_bss_section_index.?, .{});
793	795	}
794	796
795	797	if (self.dwarf) \|*dw\| {
...	...	@@ -875,7 +877,7 @@ pub fn growAllocSection(self: *Elf, shdr_index: u16, needed_size: u64) !void {
875	877	}
876	878
877	879	const mem_capacity = self.allocatedVirtualSize(phdr.p_vaddr);
878		if (needed_size <= mem_capacity) {
	880	if (needed_size > mem_capacity) {
879	881	var err = try self.addErrorWithNotes(2);
880	882	try err.addMsg(self, "fatal linker error: cannot expand load segment phdr({d}) in virtual memory", .{
881	883	phdr_index,
...	...	@@ -1544,8 +1546,8 @@ pub fn flushModule(self: Elf, comp: Compilation, prog_node: *std.Progress.Node
1544	1546	if (self.zig_module_index) \|index\| {
1545	1547	// .bss.zig always overlaps .data.zig in file offset, but is zero-sized in file so it doesn't
1546	1548	// get mapped by the loader
1547		if (self.data_zig_section_index) \|data_shndx\| blk: {
1548		const bss_shndx = self.bss_zig_section_index orelse break :blk;
	1549	if (self.zig_data_section_index) \|data_shndx\| blk: {
	1550	const bss_shndx = self.zig_bss_section_index orelse break :blk;
1549	1551	const data_phndx = self.phdr_to_shdr_table.get(data_shndx).?;
1550	1552	const bss_phndx = self.phdr_to_shdr_table.get(bss_shndx).?;
1551	1553	self.shdrs.items[bss_shndx].sh_offset = self.shdrs.items[data_shndx].sh_offset;
...	...	@@ -1580,7 +1582,7 @@ pub fn flushModule(self: Elf, comp: Compilation, prog_node: *std.Progress.Node
1580	1582	if (self.debug_info_header_dirty) {
1581	1583	// Currently only one compilation unit is supported, so the address range is simply
1582	1584	// identical to the main program header virtual address and memory size.
1583		const text_phdr = &self.phdrs.items[self.phdr_load_re_zig_index.?];
	1585	const text_phdr = &self.phdrs.items[self.phdr_zig_load_re_index.?];
1584	1586	const low_pc = text_phdr.p_vaddr;
1585	1587	const high_pc = text_phdr.p_vaddr + text_phdr.p_memsz;
1586	1588	try dw.writeDbgInfoHeader(self.base.options.module.?, low_pc, high_pc);
...	...	@@ -1590,7 +1592,7 @@ pub fn flushModule(self: Elf, comp: Compilation, prog_node: *std.Progress.Node
1590	1592	if (self.debug_aranges_section_dirty) {
1591	1593	// Currently only one compilation unit is supported, so the address range is simply
1592	1594	// identical to the main program header virtual address and memory size.
1593		const text_phdr = &self.phdrs.items[self.phdr_load_re_zig_index.?];
	1595	const text_phdr = &self.phdrs.items[self.phdr_zig_load_re_index.?];
1594	1596	try dw.writeDbgAranges(text_phdr.p_vaddr, text_phdr.p_memsz);
1595	1597	self.debug_aranges_section_dirty = false;
1596	1598	}
...	...	@@ -2999,24 +3001,24 @@ fn getDeclShdrIndex(self: *Elf, decl_index: Module.Decl.Index, code: []const u8)
2999	3001	const decl = mod.declPtr(decl_index);
3000	3002	const shdr_index = switch (decl.ty.zigTypeTag(mod)) {
3001	3003	// TODO: what if this is a function pointer?
3002		.Fn => self.text_zig_section_index.?,
	3004	.Fn => self.zig_text_section_index.?,
3003	3005	else => blk: {
3004	3006	if (decl.getOwnedVariable(mod)) \|variable\| {
3005		if (variable.is_const) break :blk self.rodata_zig_section_index.?;
	3007	if (variable.is_const) break :blk self.zig_rodata_section_index.?;
3006	3008	if (variable.init.toValue().isUndefDeep(mod)) {
3007	3009	const mode = self.base.options.optimize_mode;
3008		if (mode == .Debug or mode == .ReleaseSafe) break :blk self.data_zig_section_index.?;
3009		break :blk self.bss_zig_section_index.?;
	3010	if (mode == .Debug or mode == .ReleaseSafe) break :blk self.zig_data_section_index.?;
	3011	break :blk self.zig_bss_section_index.?;
3010	3012	}
3011	3013	// TODO I blatantly copied the logic from the Wasm linker, but is there a less
3012	3014	// intrusive check for all zeroes than this?
3013	3015	const is_all_zeroes = for (code) \|byte\| {
3014	3016	if (byte != 0) break false;
3015	3017	} else true;
3016		if (is_all_zeroes) break :blk self.bss_zig_section_index.?;
3017		break :blk self.data_zig_section_index.?;
	3018	if (is_all_zeroes) break :blk self.zig_bss_section_index.?;
	3019	break :blk self.zig_data_section_index.?;
3018	3020	}
3019		break :blk self.rodata_zig_section_index.?;
	3021	break :blk self.zig_rodata_section_index.?;
3020	3022	},
3021	3023	};
3022	3024	return shdr_index;
...	...	@@ -3070,8 +3072,9 @@ fn updateDeclCode(
3070	3072	esym.st_value = atom_ptr.value;
3071	3073
3072	3074	log.debug(" (writing new offset table entry)", .{});
3073		// const extra = sym.extra(self).?;
3074		// try self.got.writeEntry(self, extra.got);
	3075	assert(sym.flags.has_zig_got);
	3076	const extra = sym.extra(self).?;
	3077	try self.zig_got.writeOne(self, extra.zig_got);
3075	3078	}
3076	3079	} else if (code.len < old_size) {
3077	3080	atom_ptr.shrink(self);
...	...	@@ -3083,10 +3086,8 @@ fn updateDeclCode(
3083	3086	sym.value = atom_ptr.value;
3084	3087	esym.st_value = atom_ptr.value;
3085	3088
3086		sym.flags.needs_got = true;
3087		const gop = try sym.getOrCreateGotEntry(sym_index, self);
3088		_ = gop;
3089		// try self.got.writeEntry(self, gop.index);
	3089	const gop = try sym.getOrCreateZigGotEntry(sym_index, self);
	3090	try self.zig_got.writeOne(self, gop.index);
3090	3091	}
3091	3092
3092	3093	if (self.base.child_pid) \|pid\| {
...	...	@@ -3296,8 +3297,8 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol.
3296	3297	};
3297	3298
3298	3299	const output_section_index = switch (sym.kind) {
3299		.code => self.text_zig_section_index.?,
3300		.const_data => self.rodata_zig_section_index.?,
	3300	.code => self.zig_text_section_index.?,
	3301	.const_data => self.zig_rodata_section_index.?,
3301	3302	};
3302	3303	const local_sym = self.symbol(symbol_index);
3303	3304	const phdr_index = self.phdr_to_shdr_table.get(output_section_index).?;
...	...	@@ -3320,10 +3321,8 @@ fn updateLazySymbol(self: *Elf, sym: link.File.LazySymbol, symbol_index: Symbol.
3320	3321	local_sym.value = atom_ptr.value;
3321	3322	local_esym.st_value = atom_ptr.value;
3322	3323
3323		local_sym.flags.needs_got = true;
3324		const gop = try local_sym.getOrCreateGotEntry(symbol_index, self);
3325		_ = gop;
3326		// try self.got.writeEntry(self, gop.index);
	3324	const gop = try local_sym.getOrCreateZigGotEntry(symbol_index, self);
	3325	try self.zig_got.writeOne(self, gop.index);
3327	3326
3328	3327	const section_offset = atom_ptr.value - self.phdrs.items[phdr_index].p_vaddr;
3329	3328	const file_offset = self.shdrs.items[output_section_index].sh_offset + section_offset;
...	...	@@ -3343,7 +3342,7 @@ pub fn lowerUnnamedConst(self: *Elf, typed_value: TypedValue, decl_index: Module
3343	3342	const index = unnamed_consts.items.len;
3344	3343	const name = try std.fmt.allocPrint(gpa, "__unnamed_{s}_{d}", .{ decl_name, index });
3345	3344	defer gpa.free(name);
3346		const sym_index = switch (try self.lowerConst(name, typed_value, self.rodata_zig_section_index.?, decl.srcLoc(mod))) {
	3345	const sym_index = switch (try self.lowerConst(name, typed_value, self.zig_rodata_section_index.?, decl.srcLoc(mod))) {
3347	3346	.ok => \|sym_index\| sym_index,
3348	3347	.fail => \|em\| {
3349	3348	decl.analysis = .codegen_failure;
...	...	@@ -4183,11 +4182,11 @@ fn sortSections(self: *Elf) !void {
4183	4182	&self.copy_rel_section_index,
4184	4183	&self.versym_section_index,
4185	4184	&self.verneed_section_index,
4186		&self.text_zig_section_index,
4187		&self.got_zig_section_index,
4188		&self.rodata_zig_section_index,
4189		&self.data_zig_section_index,
4190		&self.bss_zig_section_index,
	4185	&self.zig_text_section_index,
	4186	&self.zig_got_section_index,
	4187	&self.zig_rodata_section_index,
	4188	&self.zig_data_section_index,
	4189	&self.zig_bss_section_index,
4191	4190	&self.debug_str_section_index,
4192	4191	&self.debug_info_section_index,
4193	4192	&self.debug_abbrev_section_index,
...	...	@@ -4340,6 +4339,9 @@ fn updateSectionSizes(self: *Elf) !void {
4340	4339	if (self.strtab_section_index) \|index\| {
4341	4340	// TODO I don't really this here but we need it to add symbol names from GOT and other synthetic
4342	4341	// sections into .strtab for easier debugging.
	4342	if (self.zig_got_section_index) \|_\| {
	4343	try self.zig_got.updateStrtab(self);
	4344	}
4343	4345	if (self.got_section_index) \|_\| {
4344	4346	try self.got.updateStrtab(self);
4345	4347	}
...	...	@@ -4719,6 +4721,11 @@ fn updateSymtabSize(self: *Elf) !void {
4719	4721	sizes.nglobals += shared_object.output_symtab_size.nglobals;
4720	4722	}
4721	4723
	4724	if (self.zig_got_section_index) \|_\| {
	4725	self.zig_got.updateSymtabSize(self);
	4726	sizes.nlocals += self.zig_got.output_symtab_size.nlocals;
	4727	}
	4728
4722	4729	if (self.got_section_index) \|_\| {
4723	4730	self.got.updateSymtabSize(self);
4724	4731	sizes.nlocals += self.got.output_symtab_size.nlocals;
...	...	@@ -4930,6 +4937,11 @@ fn writeSymtab(self: *Elf) !void {
4930	4937	ctx.iglobal += shared_object.output_symtab_size.nglobals;
4931	4938	}
4932	4939
	4940	if (self.zig_got_section_index) \|_\| {
	4941	try self.zig_got.writeSymtab(self, ctx);
	4942	ctx.ilocal += self.zig_got.output_symtab_size.nlocals;
	4943	}
	4944
4933	4945	if (self.got_section_index) \|_\| {
4934	4946	try self.got.writeSymtab(self, ctx);
4935	4947	ctx.ilocal += self.got.output_symtab_size.nlocals;
...	...	@@ -5289,11 +5301,11 @@ pub fn isDynLib(self: Elf) bool {
5289	5301
5290	5302	pub fn isZigSection(self: Elf, shndx: u16) bool {
5291	5303	inline for (&[_]?u16{
5292		self.text_zig_section_index,
5293		self.rodata_zig_section_index,
5294		self.data_zig_section_index,
5295		self.bss_zig_section_index,
5296		self.got_zig_section_index,
	5304	self.zig_text_section_index,
	5305	self.zig_rodata_section_index,
	5306	self.zig_data_section_index,
	5307	self.zig_bss_section_index,
	5308	self.zig_got_section_index,
5297	5309	}) \|maybe_index\| {
5298	5310	if (maybe_index) \|index\| {
5299	5311	if (index == shndx) return true;
...	...	@@ -5851,6 +5863,7 @@ fn fmtDumpState(
5851	5863	try writer.print("{}\n", .{linker_defined.fmtSymtab(self)});
5852	5864	}
5853	5865	try writer.print("{}\n", .{self.got.fmt(self)});
	5866	try writer.print("{}\n", .{self.zig_got.fmt(self)});
5854	5867	try writer.writeAll("Output shdrs\n");
5855	5868	for (self.shdrs.items, 0..) \|shdr, shndx\| {
5856	5869	try writer.print("shdr({d}) : phdr({?d}) : {}\n", .{
...	...	@@ -6030,4 +6043,5 @@ const Type = @import("../type.zig").Type;
6030	6043	const TypedValue = @import("../TypedValue.zig");
6031	6044	const Value = @import("../value.zig").Value;
6032	6045	const VerneedSection = synthetic_sections.VerneedSection;
	6046	const ZigGotSection = synthetic_sections.ZigGotSection;
6033	6047	const ZigModule = @import("Elf/ZigModule.zig");

src/link/Elf/Atom.zig+16-2

...	...	@@ -367,8 +367,16 @@ pub fn scanRelocs(self: Atom, elf_file: *Elf, code: ?[]const u8, undefs: anytype
367	367	try self.scanReloc(symbol, rel, dynAbsRelocAction(symbol, elf_file), elf_file);
368	368	},
369	369
	370	// TODO I have temporarily repurposed those for handling .zig.got indirection
	371	// but we should probably claim unused custom values for incremental linking
	372	// that get rewritten to standard relocs when lowering to a relocatable object
	373	// file.
370	374	elf.R_X86_64_GOT32,
371	375	elf.R_X86_64_GOT64,
	376	=> {
	377	assert(symbol.flags.has_zig_got);
	378	},
	379
372	380	elf.R_X86_64_GOTPC32,
373	381	elf.R_X86_64_GOTPC64,
374	382	elf.R_X86_64_GOTPCREL,
...	...	@@ -736,6 +744,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) !void {
736	744	null;
737	745	break :blk if (shndx) \|index\| @as(i64, @intCast(elf_file.shdrs.items[index].sh_addr)) else 0;
738	746	};
	747	// Address of the .zig.got table entry if any.
	748	const ZIG_GOT = @as(i64, @intCast(target.zigGotAddress(elf_file)));
739	749	// Relative offset to the start of the global offset table.
740	750	const G = @as(i64, @intCast(target.gotAddress(elf_file))) - GOT;
741	751	// // Address of the thread pointer.
...	...	@@ -796,8 +806,12 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) !void {
796	806	elf.R_X86_64_32 => try cwriter.writeIntLittle(u32, @as(u32, @truncate(@as(u64, @intCast(S + A))))),
797	807	elf.R_X86_64_32S => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A))),
798	808
799		elf.R_X86_64_GOT32 => try cwriter.writeIntLittle(u32, @as(u32, @intCast(G + GOT + A))),
800		elf.R_X86_64_GOT64 => try cwriter.writeIntLittle(u64, @as(u64, @intCast(G + GOT + A))),
	809	// TODO I have temporarily repurposed those for handling .zig.got indirection
	810	// but we should probably claim unused custom values for incremental linking
	811	// that get rewritten to standard relocs when lowering to a relocatable object
	812	// file.
	813	elf.R_X86_64_GOT32 => try cwriter.writeIntLittle(u32, @as(u32, @intCast(ZIG_GOT + A))),
	814	elf.R_X86_64_GOT64 => try cwriter.writeIntLittle(u64, @as(u64, @intCast(ZIG_GOT + A))),
801	815
802	816	elf.R_X86_64_TPOFF32 => try cwriter.writeIntLittle(i32, @as(i32, @truncate(S + A - TP))),
803	817	elf.R_X86_64_TPOFF64 => try cwriter.writeIntLittle(i64, S + A - TP),

src/link/Elf/Symbol.zig+22-13

...	...	@@ -137,19 +137,6 @@ pub fn copyRelAddress(symbol: Symbol, elf_file: *Elf) u64 {
137	137	return shdr.sh_addr + symbol.value;
138	138	}
139	139
140		const GetOrCreateGotEntryResult = struct {
141		found_existing: bool,
142		index: GotSection.Index,
143		};
144
145		pub fn getOrCreateGotEntry(symbol: Symbol, symbol_index: Index, elf_file: Elf) !GetOrCreateGotEntryResult {
146		assert(symbol.flags.needs_got);
147		if (symbol.flags.has_got) return .{ .found_existing = true, .index = symbol.extra(elf_file).?.got };
148		const index = try elf_file.got.addGotSymbol(symbol_index, elf_file);
149		symbol.flags.has_got = true;
150		return .{ .found_existing = false, .index = index };
151		}
152
153	140	pub fn tlsGdAddress(symbol: Symbol, elf_file: *Elf) u64 {
154	141	if (!symbol.flags.has_tlsgd) return 0;
155	142	const extras = symbol.extra(elf_file).?;
...	...	@@ -171,6 +158,23 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) u64 {
171	158	return entry.address(elf_file);
172	159	}
173	160
	161	const GetOrCreateZigGotEntryResult = struct {
	162	found_existing: bool,
	163	index: ZigGotSection.Index,
	164	};
	165
	166	pub fn getOrCreateZigGotEntry(symbol: Symbol, symbol_index: Index, elf_file: Elf) !GetOrCreateZigGotEntryResult {
	167	if (symbol.flags.has_zig_got) return .{ .found_existing = true, .index = symbol.extra(elf_file).?.zig_got };
	168	const index = try elf_file.zig_got.addSymbol(symbol_index, elf_file);
	169	return .{ .found_existing = false, .index = index };
	170	}
	171
	172	pub fn zigGotAddress(symbol: Symbol, elf_file: *Elf) u64 {
	173	if (!symbol.flags.has_zig_got) return 0;
	174	const extras = symbol.extra(elf_file).?;
	175	return elf_file.zig_got.entryAddress(extras.zig_got, elf_file);
	176	}
	177
174	178	pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 {
175	179	const file_ptr = symbol.file(elf_file) orelse return 0;
176	180	assert(file_ptr == .shared_object);
...	...	@@ -367,6 +371,9 @@ pub const Flags = packed struct {
367	371	/// Whether the symbol contains TLSDESC indirection.
368	372	needs_tlsdesc: bool = false,
369	373	has_tlsdesc: bool = false,
	374
	375	/// Whether the symbol contains .zig.got indirection.
	376	has_zig_got: bool = false,
370	377	};
371	378
372	379	pub const Extra = struct {
...	...	@@ -378,6 +385,7 @@ pub const Extra = struct {
378	385	tlsgd: u32 = 0,
379	386	gottp: u32 = 0,
380	387	tlsdesc: u32 = 0,
	388	zig_got: u32 = 0,
381	389	};
382	390
383	391	pub const Index = u32;
...	...	@@ -397,4 +405,5 @@ const Object = @import("Object.zig");
397	405	const PltSection = synthetic_sections.PltSection;
398	406	const SharedObject = @import("SharedObject.zig");
399	407	const Symbol = @This();
	408	const ZigGotSection = synthetic_sections.ZigGotSection;
400	409	const ZigModule = @import("ZigModule.zig");

src/link/Elf/synthetic_sections.zig+196-11

...	...	@@ -220,6 +220,191 @@ pub const DynamicSection = struct {
220	220	}
221	221	};
222	222
	223	pub const ZigGotSection = struct {
	224	entries: std.ArrayListUnmanaged(Symbol.Index) = .{},
	225	output_symtab_size: Elf.SymtabSize = .{},
	226	flags: Flags = .{},
	227
	228	const Flags = packed struct {
	229	needs_rela: bool = false, // TODO in prep for PIC/PIE and base relocations
	230	dirty: bool = false,
	231	};
	232
	233	pub const Index = u32;
	234
	235	pub fn deinit(zig_got: *ZigGotSection, allocator: Allocator) void {
	236	zig_got.entries.deinit(allocator);
	237	}
	238
	239	fn allocateEntry(zig_got: *ZigGotSection, allocator: Allocator) !Index {
	240	try zig_got.entries.ensureUnusedCapacity(allocator, 1);
	241	// TODO add free list
	242	const index = @as(Index, @intCast(zig_got.entries.items.len));
	243	_ = zig_got.entries.addOneAssumeCapacity();
	244	zig_got.flags.dirty = true;
	245	return index;
	246	}
	247
	248	pub fn addSymbol(zig_got: ZigGotSection, sym_index: Symbol.Index, elf_file: Elf) !Index {
	249	const index = try zig_got.allocateEntry(elf_file.base.allocator);
	250	const entry = &zig_got.entries.items[index];
	251	entry.* = sym_index;
	252	const symbol = elf_file.symbol(sym_index);
	253	symbol.flags.has_zig_got = true;
	254	if (symbol.extra(elf_file)) \|extra\| {
	255	var new_extra = extra;
	256	new_extra.zig_got = index;
	257	symbol.setExtra(new_extra, elf_file);
	258	} else try symbol.addExtra(.{ .zig_got = index }, elf_file);
	259	return index;
	260	}
	261
	262	pub fn entryOffset(zig_got: ZigGotSection, index: Index, elf_file: *Elf) u64 {
	263	_ = zig_got;
	264	const entry_size = elf_file.archPtrWidthBytes();
	265	const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?];
	266	return shdr.sh_offset + @as(u64, entry_size) * index;
	267	}
	268
	269	pub fn entryAddress(zig_got: ZigGotSection, index: Index, elf_file: *Elf) u64 {
	270	_ = zig_got;
	271	const entry_size = elf_file.archPtrWidthBytes();
	272	const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?];
	273	return shdr.sh_addr + @as(u64, entry_size) * index;
	274	}
	275
	276	pub fn size(zig_got: ZigGotSection, elf_file: *Elf) usize {
	277	return elf_file.archPtrWidthBytes() * zig_got.entries.items.len;
	278	}
	279
	280	pub fn writeOne(zig_got: ZigGotSection, elf_file: Elf, index: Index) !void {
	281	if (zig_got.flags.dirty) {
	282	const needed_size = zig_got.size(elf_file);
	283	try elf_file.growAllocSection(elf_file.zig_got_section_index.?, needed_size);
	284	zig_got.flags.dirty = false;
	285	}
	286	const entry_size: u16 = elf_file.archPtrWidthBytes();
	287	const endian = elf_file.base.options.target.cpu.arch.endian();
	288	const off = zig_got.entryOffset(index, elf_file);
	289	const vaddr = zig_got.entryAddress(index, elf_file);
	290	const entry = zig_got.entries.items[index];
	291	const value = elf_file.symbol(entry).value;
	292	switch (entry_size) {
	293	2 => {
	294	var buf: [2]u8 = undefined;
	295	std.mem.writeInt(u16, &buf, @as(u16, @intCast(value)), endian);
	296	try elf_file.base.file.?.pwriteAll(&buf, off);
	297	},
	298	4 => {
	299	var buf: [4]u8 = undefined;
	300	std.mem.writeInt(u32, &buf, @as(u32, @intCast(value)), endian);
	301	try elf_file.base.file.?.pwriteAll(&buf, off);
	302	},
	303	8 => {
	304	var buf: [8]u8 = undefined;
	305	std.mem.writeInt(u64, &buf, value, endian);
	306	try elf_file.base.file.?.pwriteAll(&buf, off);
	307
	308	if (elf_file.base.child_pid) \|pid\| {
	309	switch (builtin.os.tag) {
	310	.linux => {
	311	var local_vec: [1]std.os.iovec_const = .{.{
	312	.iov_base = &buf,
	313	.iov_len = buf.len,
	314	}};
	315	var remote_vec: [1]std.os.iovec_const = .{.{
	316	.iov_base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))),
	317	.iov_len = buf.len,
	318	}};
	319	const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0);
	320	switch (std.os.errno(rc)) {
	321	.SUCCESS => assert(rc == buf.len),
	322	else => \|errno\| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}),
	323	}
	324	},
	325	else => return error.HotSwapUnavailableOnHostOperatingSystem,
	326	}
	327	}
	328	},
	329	else => unreachable,
	330	}
	331	}
	332
	333	pub fn writeAll(zig_got: ZigGotSection, elf_file: *Elf, writer: anytype) !void {
	334	for (zig_got.entries.items) \|entry\| {
	335	const symbol = elf_file.symbol(entry);
	336	const value = symbol.address(.{ .plt = false }, elf_file);
	337	try writeInt(value, elf_file, writer);
	338	}
	339	}
	340
	341	pub fn updateSymtabSize(zig_got: ZigGotSection, elf_file: Elf) void {
	342	_ = elf_file;
	343	zig_got.output_symtab_size.nlocals = @as(u32, @intCast(zig_got.entries.items.len));
	344	}
	345
	346	pub fn updateStrtab(zig_got: ZigGotSection, elf_file: *Elf) !void {
	347	const gpa = elf_file.base.allocator;
	348	for (zig_got.entries.items) \|entry\| {
	349	const symbol_name = elf_file.symbol(entry).name(elf_file);
	350	const name = try std.fmt.allocPrint(gpa, "{s}$ziggot", .{symbol_name});
	351	defer gpa.free(name);
	352	_ = try elf_file.strtab.insert(gpa, name);
	353	}
	354	}
	355
	356	pub fn writeSymtab(zig_got: ZigGotSection, elf_file: *Elf, ctx: anytype) !void {
	357	const gpa = elf_file.base.allocator;
	358	for (zig_got.entries.items, ctx.ilocal.., 0..) \|entry, ilocal, index\| {
	359	const symbol = elf_file.symbol(entry);
	360	const symbol_name = symbol.name(elf_file);
	361	const name = try std.fmt.allocPrint(gpa, "{s}$ziggot", .{symbol_name});
	362	defer gpa.free(name);
	363	const st_name = try elf_file.strtab.insert(gpa, name);
	364	const st_value = zig_got.entryAddress(@intCast(index), elf_file);
	365	const st_size = elf_file.archPtrWidthBytes();
	366	ctx.symtab[ilocal] = .{
	367	.st_name = st_name,
	368	.st_info = elf.STT_OBJECT,
	369	.st_other = 0,
	370	.st_shndx = elf_file.zig_got_section_index.?,
	371	.st_value = st_value,
	372	.st_size = st_size,
	373	};
	374	}
	375	}
	376
	377	const FormatCtx = struct {
	378	zig_got: ZigGotSection,
	379	elf_file: *Elf,
	380	};
	381
	382	pub fn fmt(zig_got: ZigGotSection, elf_file: *Elf) std.fmt.Formatter(format2) {
	383	return .{ .data = .{ .zig_got = zig_got, .elf_file = elf_file } };
	384	}
	385
	386	pub fn format2(
	387	ctx: FormatCtx,
	388	comptime unused_fmt_string: []const u8,
	389	options: std.fmt.FormatOptions,
	390	writer: anytype,
	391	) !void {
	392	_ = options;
	393	_ = unused_fmt_string;
	394	try writer.writeAll(".zig.got\n");
	395	for (ctx.zig_got.entries.items, 0..) \|entry, index\| {
	396	const symbol = ctx.elf_file.symbol(entry);
	397	try writer.print(" {d}@0x{x} => {d}@0x{x} ({s})\n", .{
	398	index,
	399	ctx.zig_got.entryAddress(@intCast(index), ctx.elf_file),
	400	entry,
	401	symbol.address(.{}, ctx.elf_file),
	402	symbol.name(ctx.elf_file),
	403	});
	404	}
	405	}
	406	};
	407
223	408	pub const GotSection = struct {
224	409	entries: std.ArrayListUnmanaged(Entry) = .{},
225	410	output_symtab_size: Elf.SymtabSize = .{},
...	...	@@ -420,17 +605,6 @@ pub const GotSection = struct {
420	605	}
421	606	}
422	607
423		fn writeInt(value: anytype, elf_file: *Elf, writer: anytype) !void {
424		const entry_size = elf_file.archPtrWidthBytes();
425		const endian = elf_file.base.options.target.cpu.arch.endian();
426		switch (entry_size) {
427		2 => try writer.writeInt(u16, @intCast(value), endian),
428		4 => try writer.writeInt(u32, @intCast(value), endian),
429		8 => try writer.writeInt(u64, @intCast(value), endian),
430		else => unreachable,
431		}
432		}
433
434	608	pub fn addRela(got: GotSection, elf_file: *Elf) !void {
435	609	const is_dyn_lib = elf_file.isDynLib();
436	610	try elf_file.rela_dyn.ensureUnusedCapacity(elf_file.base.allocator, got.numRela(elf_file));
...	...	@@ -1327,6 +1501,17 @@ pub const VerneedSection = struct {
1327	1501	}
1328	1502	};
1329	1503
	1504	fn writeInt(value: anytype, elf_file: *Elf, writer: anytype) !void {
	1505	const entry_size = elf_file.archPtrWidthBytes();
	1506	const endian = elf_file.base.options.target.cpu.arch.endian();
	1507	switch (entry_size) {
	1508	2 => try writer.writeInt(u16, @intCast(value), endian),
	1509	4 => try writer.writeInt(u32, @intCast(value), endian),
	1510	8 => try writer.writeInt(u64, @intCast(value), endian),
	1511	else => unreachable,
	1512	}
	1513	}
	1514
1330	1515	const assert = std.debug.assert;
1331	1516	const builtin = @import("builtin");
1332	1517	const elf = std.elf;