authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-09-27 07:39:58+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-09-27 07:39:58+02:00
log7a43f45908fda0ad8513ab4d7692018cdf801150
tree6b0494c0892b5733ee9223adcbb5ea489c8081bb
parentd1e39b6914ba0f79fb25c5bba7375964d865a69a
parente30f396b732f7fcb68a701f1b385bb47eba48b89
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #17284 from ziglang/elf-tests

elf: link against musl libc, add ELF test harness, dynamically allocate misc SHF_ALLOC sections

5 files changed, 302 insertions(+), 33 deletions(-)

src/link.zig+19-2
......@@ -465,9 +465,10 @@ pub const File = struct {
465465 .Exe => {},
466466 }
467467 switch (base.tag) {
468 .coff, .elf, .macho, .plan9, .wasm => if (base.file) |f| {
468 .elf => if (base.file) |f| {
469469 if (build_options.only_c) unreachable;
470 if (base.intermediary_basename != null) {
470 const use_lld = build_options.have_llvm and base.options.use_lld;
471 if (base.intermediary_basename != null and use_lld) {
471472 // The file we have open is not the final file that we want to
472473 // make executable, so we don't have to close it.
473474 return;
......@@ -480,6 +481,22 @@ pub const File = struct {
480481 .linux => std.os.ptrace(std.os.linux.PTRACE.DETACH, pid, 0, 0) catch |err| {
481482 log.warn("ptrace failure: {s}", .{@errorName(err)});
482483 },
484 else => return error.HotSwapUnavailableOnHostOperatingSystem,
485 }
486 }
487 },
488 .coff, .macho, .plan9, .wasm => if (base.file) |f| {
489 if (build_options.only_c) unreachable;
490 if (base.intermediary_basename != null) {
491 // The file we have open is not the final file that we want to
492 // make executable, so we don't have to close it.
493 return;
494 }
495 f.close();
496 base.file = null;
497
498 if (base.child_pid) |pid| {
499 switch (builtin.os.tag) {
483500 .macos => base.cast(MachO).?.ptraceDetach(pid) catch |err| {
484501 log.warn("detaching failed with error: {s}", .{@errorName(err)});
485502 },
src/link/Elf.zig+90-23
......@@ -254,7 +254,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*Elf {
254254 .default_sym_version = default_sym_version,
255255 };
256256 const use_llvm = options.use_llvm;
257 if (use_llvm) {
257 if (use_llvm and options.module != null) {
258258 self.llvm_object = try LlvmObject.create(gpa, options);
259259 }
260260
......@@ -409,16 +409,24 @@ fn findFreeSpace(self: *Elf, object_size: u64, min_alignment: u64) u64 {
409409}
410410
411411const AllocateSegmentOpts = struct {
412 addr: u64, // TODO find free VM space
413412 size: u64,
414413 alignment: u64,
414 addr: ?u64 = null, // TODO find free VM space
415415 flags: u32 = elf.PF_R,
416416};
417417
418fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}!u16 {
418pub fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}!u16 {
419419 const index = @as(u16, @intCast(self.phdrs.items.len));
420420 try self.phdrs.ensureUnusedCapacity(self.base.allocator, 1);
421421 const off = self.findFreeSpace(opts.size, opts.alignment);
422 // Memory is always allocated in sequence.
423 // TODO is this correct? Or should we implement something similar to `findFreeSpace`?
424 // How would that impact HCS?
425 const addr = opts.addr orelse blk: {
426 assert(self.phdr_table_load_index != null);
427 const phdr = &self.phdrs.items[index - 1];
428 break :blk mem.alignForward(u64, phdr.p_vaddr + phdr.p_memsz, opts.alignment);
429 };
422430 log.debug("allocating phdr({d})({c}{c}{c}) from 0x{x} to 0x{x} (0x{x} - 0x{x})", .{
423431 index,
424432 if (opts.flags & elf.PF_R != 0) @as(u8, 'R') else '_',
......@@ -426,15 +434,15 @@ fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}!u16
426434 if (opts.flags & elf.PF_X != 0) @as(u8, 'X') else '_',
427435 off,
428436 off + opts.size,
429 opts.addr,
430 opts.addr + opts.size,
437 addr,
438 addr + opts.size,
431439 });
432440 self.phdrs.appendAssumeCapacity(.{
433441 .p_type = elf.PT_LOAD,
434442 .p_offset = off,
435443 .p_filesz = opts.size,
436 .p_vaddr = opts.addr,
437 .p_paddr = opts.addr,
444 .p_vaddr = addr,
445 .p_paddr = addr,
438446 .p_memsz = opts.size,
439447 .p_align = opts.alignment,
440448 .p_flags = opts.flags,
......@@ -446,12 +454,12 @@ fn allocateSegment(self: *Elf, opts: AllocateSegmentOpts) error{OutOfMemory}!u16
446454const AllocateAllocSectionOpts = struct {
447455 name: [:0]const u8,
448456 phdr_index: u16,
449 alignment: u16 = 1,
450 flags: u16 = elf.SHF_ALLOC,
457 alignment: u64 = 1,
458 flags: u64 = elf.SHF_ALLOC,
451459 type: u32 = elf.SHT_PROGBITS,
452460};
453461
454fn allocateAllocSection(self: *Elf, opts: AllocateAllocSectionOpts) error{OutOfMemory}!u16 {
462pub fn allocateAllocSection(self: *Elf, opts: AllocateAllocSectionOpts) error{OutOfMemory}!u16 {
455463 const gpa = self.base.allocator;
456464 const phdr = &self.phdrs.items[opts.phdr_index];
457465 const index = @as(u16, @intCast(self.shdrs.items.len));
......@@ -622,6 +630,7 @@ pub fn populateMissingMetadata(self: *Elf) !void {
622630 });
623631 const phdr = &self.phdrs.items[self.phdr_load_zerofill_index.?];
624632 phdr.p_offset = self.phdrs.items[self.phdr_load_rw_index.?].p_offset; // .bss overlaps .data
633 phdr.p_memsz = 1024;
625634 }
626635
627636 if (self.shstrtab_section_index == null) {
......@@ -965,6 +974,7 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
965974 defer arena_allocator.deinit();
966975 const arena = arena_allocator.allocator();
967976
977 const target = self.base.options.target;
968978 const directory = self.base.options.emit.?.directory; // Just an alias to make it shorter to type.
969979 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
970980
......@@ -993,19 +1003,77 @@ pub fn flushModule(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node
9931003 try positionals.append(.{ .path = key.status.success.object_path });
9941004 }
9951005
1006 // csu prelude
1007 var csu = try CsuObjects.init(arena, self.base.options, comp);
1008 if (csu.crt0) |v| try positionals.append(.{ .path = v });
1009 if (csu.crti) |v| try positionals.append(.{ .path = v });
1010 if (csu.crtbegin) |v| try positionals.append(.{ .path = v });
1011
1012 for (positionals.items) |obj| {
1013 const in_file = try std.fs.cwd().openFile(obj.path, .{});
1014 defer in_file.close();
1015 var parse_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
1016 self.parsePositional(in_file, obj.path, obj.must_link, &parse_ctx) catch |err|
1017 try self.handleAndReportParseError(obj.path, err, &parse_ctx);
1018 }
1019
1020 var system_libs = std.ArrayList(SystemLib).init(arena);
1021
1022 // libc dep
1023 self.error_flags.missing_libc = false;
1024 if (self.base.options.link_libc) {
1025 if (self.base.options.libc_installation != null) {
1026 @panic("TODO explicit libc_installation");
1027 } else if (target.isGnuLibC()) {
1028 try system_libs.ensureUnusedCapacity(glibc.libs.len + 1);
1029 for (glibc.libs) |lib| {
1030 const lib_path = try std.fmt.allocPrint(arena, "{s}{c}lib{s}.so.{d}", .{
1031 comp.glibc_so_files.?.dir_path, fs.path.sep, lib.name, lib.sover,
1032 });
1033 system_libs.appendAssumeCapacity(.{ .path = lib_path });
1034 }
1035 system_libs.appendAssumeCapacity(.{
1036 .path = try comp.get_libc_crt_file(arena, "libc_nonshared.a"),
1037 });
1038 } else if (target.isMusl()) {
1039 const path = try comp.get_libc_crt_file(arena, switch (self.base.options.link_mode) {
1040 .Static => "libc.a",
1041 .Dynamic => "libc.so",
1042 });
1043 try system_libs.append(.{ .path = path });
1044 } else {
1045 self.error_flags.missing_libc = true;
1046 }
1047 }
1048
1049 for (system_libs.items) |lib| {
1050 const in_file = try std.fs.cwd().openFile(lib.path, .{});
1051 defer in_file.close();
1052 var parse_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
1053 self.parseLibrary(in_file, lib, false, &parse_ctx) catch |err|
1054 try self.handleAndReportParseError(lib.path, err, &parse_ctx);
1055 }
1056
1057 // Finally, as the last input objects we add compiler_rt and CSU postlude (if any).
1058 positionals.clearRetainingCapacity();
1059
1060 // compiler-rt. Since compiler_rt exports symbols like `memset`, it needs
1061 // to be after the shared libraries, so they are picked up from the shared
1062 // libraries, not libcompiler_rt.
9961063 const compiler_rt_path: ?[]const u8 = blk: {
9971064 if (comp.compiler_rt_lib) |x| break :blk x.full_object_path;
9981065 if (comp.compiler_rt_obj) |x| break :blk x.full_object_path;
9991066 break :blk null;
10001067 };
1001 if (compiler_rt_path) |path| {
1002 try positionals.append(.{ .path = path });
1003 }
1068 if (compiler_rt_path) |path| try positionals.append(.{ .path = path });
1069
1070 // csu postlude
1071 if (csu.crtend) |v| try positionals.append(.{ .path = v });
1072 if (csu.crtn) |v| try positionals.append(.{ .path = v });
10041073
10051074 for (positionals.items) |obj| {
10061075 const in_file = try std.fs.cwd().openFile(obj.path, .{});
10071076 defer in_file.close();
1008
10091077 var parse_ctx: ParseErrorCtx = .{ .detected_cpu_arch = undefined };
10101078 self.parsePositional(in_file, obj.path, obj.must_link, &parse_ctx) catch |err|
10111079 try self.handleAndReportParseError(obj.path, err, &parse_ctx);
......@@ -1347,28 +1415,22 @@ fn parsePositional(
13471415 if (Object.isObject(in_file)) {
13481416 try self.parseObject(in_file, path, ctx);
13491417 } else {
1350 try self.parseLibrary(in_file, path, .{
1351 .path = null,
1352 .needed = false,
1353 .weak = false,
1354 }, must_link, ctx);
1418 try self.parseLibrary(in_file, .{ .path = path }, must_link, ctx);
13551419 }
13561420}
13571421
13581422fn parseLibrary(
13591423 self: *Elf,
13601424 in_file: std.fs.File,
1361 path: []const u8,
1362 lib: link.SystemLib,
1425 lib: SystemLib,
13631426 must_link: bool,
13641427 ctx: *ParseErrorCtx,
13651428) ParseError!void {
13661429 const tracy = trace(@src());
13671430 defer tracy.end();
1368 _ = lib;
13691431
13701432 if (Archive.isArchive(in_file)) {
1371 try self.parseArchive(in_file, path, must_link, ctx);
1433 try self.parseArchive(in_file, lib.path, must_link, ctx);
13721434 } else return error.UnknownFileType;
13731435}
13741436
......@@ -4150,6 +4212,11 @@ pub const null_sym = elf.Elf64_Sym{
41504212 .st_size = 0,
41514213};
41524214
4215const SystemLib = struct {
4216 needed: bool = false,
4217 path: []const u8,
4218};
4219
41534220const std = @import("std");
41544221const build_options = @import("build_options");
41554222const builtin = @import("builtin");
src/link/Elf/Object.zig+36-8
......@@ -136,7 +136,7 @@ fn initAtoms(self: *Object, elf_file: *Elf) !void {
136136 try self.comdat_groups.append(elf_file.base.allocator, comdat_group_index);
137137 },
138138
139 elf.SHT_SYMTAB_SHNDX => @panic("TODO"),
139 elf.SHT_SYMTAB_SHNDX => @panic("TODO SHT_SYMTAB_SHNDX"),
140140
141141 elf.SHT_NULL,
142142 elf.SHT_REL,
......@@ -166,14 +166,20 @@ fn initAtoms(self: *Object, elf_file: *Elf) !void {
166166 };
167167}
168168
169fn addAtom(self: *Object, shdr: elf.Elf64_Shdr, shndx: u16, name: [:0]const u8, elf_file: *Elf) !void {
169fn addAtom(
170 self: *Object,
171 shdr: elf.Elf64_Shdr,
172 shndx: u16,
173 name: [:0]const u8,
174 elf_file: *Elf,
175) error{ OutOfMemory, Overflow }!void {
170176 const atom_index = try elf_file.addAtom();
171177 const atom = elf_file.atom(atom_index).?;
172178 atom.atom_index = atom_index;
173179 atom.name_offset = try elf_file.strtab.insert(elf_file.base.allocator, name);
174180 atom.file_index = self.index;
175181 atom.input_section_index = shndx;
176 atom.output_section_index = self.getOutputSectionIndex(elf_file, shdr);
182 atom.output_section_index = try self.getOutputSectionIndex(elf_file, shdr);
177183 atom.alive = true;
178184 self.atoms.items[shndx] = atom_index;
179185
......@@ -188,7 +194,7 @@ fn addAtom(self: *Object, shdr: elf.Elf64_Shdr, shndx: u16, name: [:0]const u8,
188194 }
189195}
190196
191fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) u16 {
197fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) error{OutOfMemory}!u16 {
192198 const name = blk: {
193199 const name = self.strings.getAssumeExists(shdr.sh_name);
194200 // if (shdr.sh_flags & elf.SHF_MERGE != 0) break :blk name;
......@@ -223,10 +229,32 @@ fn getOutputSectionIndex(self: *Object, elf_file: *Elf, shdr: elf.Elf64_Shdr) u1
223229 else => flags,
224230 };
225231 };
226 _ = flags;
227 const out_shndx = elf_file.sectionByName(name) orelse {
228 log.err("{}: output section {s} not found", .{ self.fmtPath(), name });
229 @panic("TODO: missing output section!");
232 const out_shndx = elf_file.sectionByName(name) orelse blk: {
233 const is_alloc = flags & elf.SHF_ALLOC != 0;
234 const is_write = flags & elf.SHF_WRITE != 0;
235 const is_exec = flags & elf.SHF_EXECINSTR != 0;
236 const is_tls = flags & elf.SHF_TLS != 0;
237 if (!is_alloc or is_tls) {
238 log.err("{}: output section {s} not found", .{ self.fmtPath(), name });
239 @panic("TODO: missing output section!");
240 }
241 var phdr_flags: u32 = elf.PF_R;
242 if (is_write) phdr_flags |= elf.PF_W;
243 if (is_exec) phdr_flags |= elf.PF_X;
244 const phdr_index = try elf_file.allocateSegment(.{
245 .size = Elf.padToIdeal(shdr.sh_size),
246 .alignment = if (is_tls) shdr.sh_addralign else elf_file.page_size,
247 .flags = phdr_flags,
248 });
249 const shndx = try elf_file.allocateAllocSection(.{
250 .name = name,
251 .phdr_index = phdr_index,
252 .alignment = shdr.sh_addralign,
253 .flags = flags,
254 .type = @"type",
255 });
256 try elf_file.last_atom_and_free_list_table.putNoClobber(elf_file.base.allocator, shndx, .{});
257 break :blk shndx;
230258 };
231259 return out_shndx;
232260}
test/link.zig+6
......@@ -29,6 +29,12 @@ pub const cases = [_]Case{
2929 .import = @import("link/glibc_compat/build.zig"),
3030 },
3131
32 // Elf Cases
33 .{
34 .build_root = "test/link",
35 .import = @import("link/elf.zig"),
36 },
37
3238 // WASM Cases
3339 // https://github.com/ziglang/zig/issues/16938
3440 //.{
test/link/elf.zig created+151
......@@ -0,0 +1,151 @@
1//! Here we test our ELF linker for correctness and functionality.
2//! Currently, we support linking x86_64 Linux, but in the future we
3//! will progressively relax those to exercise more combinations.
4
5pub fn build(b: *Build) void {
6 const elf_step = b.step("test-elf", "Run ELF tests");
7 b.default_step = elf_step;
8
9 const musl_target = CrossTarget{
10 .cpu_arch = .x86_64, // TODO relax this once ELF linker is able to handle other archs
11 .os_tag = .linux,
12 .abi = .musl,
13 };
14
15 // Exercise linker with self-hosted backend (no LLVM)
16 elf_step.dependOn(testLinkingZig(b, .{ .use_llvm = false }));
17
18 // Exercise linker with LLVM backend
19 elf_step.dependOn(testEmptyObject(b, .{ .target = musl_target }));
20 elf_step.dependOn(testLinkingC(b, .{ .target = musl_target }));
21 elf_step.dependOn(testLinkingZig(b, .{}));
22}
23
24fn testEmptyObject(b: *Build, opts: Options) *Step {
25 const test_step = addTestStep(b, "empty-object", opts);
26
27 const exe = addExecutable(b, opts);
28 addCSourceBytes(exe, "int main() { return 0; }");
29 addCSourceBytes(exe, "");
30 exe.is_linking_libc = true;
31
32 const run = addRunArtifact(exe);
33 run.expectExitCode(0);
34 test_step.dependOn(&run.step);
35
36 return test_step;
37}
38
39fn testLinkingC(b: *Build, opts: Options) *Step {
40 const test_step = addTestStep(b, "linking-c-static", opts);
41
42 const exe = addExecutable(b, opts);
43 addCSourceBytes(exe,
44 \\#include <stdio.h>
45 \\int main() {
46 \\ printf("Hello World!\n");
47 \\ return 0;
48 \\}
49 );
50 exe.is_linking_libc = true;
51
52 const run = addRunArtifact(exe);
53 run.expectStdOutEqual("Hello World!\n");
54 test_step.dependOn(&run.step);
55
56 const check = exe.checkObject();
57 check.checkStart();
58 check.checkExact("header");
59 check.checkExact("type EXEC");
60 check.checkStart();
61 check.checkExact("section headers");
62 check.checkNotPresent("name .dynamic");
63 test_step.dependOn(&check.step);
64
65 return test_step;
66}
67
68fn testLinkingZig(b: *Build, opts: Options) *Step {
69 const test_step = addTestStep(b, "linking-zig-static", opts);
70
71 const exe = addExecutable(b, opts);
72 addZigSourceBytes(exe,
73 \\pub fn main() void {
74 \\ @import("std").debug.print("Hello World!\n", .{});
75 \\}
76 );
77
78 const run = addRunArtifact(exe);
79 run.expectStdErrEqual("Hello World!\n");
80 test_step.dependOn(&run.step);
81
82 const check = exe.checkObject();
83 check.checkStart();
84 check.checkExact("header");
85 check.checkExact("type EXEC");
86 check.checkStart();
87 check.checkExact("section headers");
88 check.checkNotPresent("name .dynamic");
89 test_step.dependOn(&check.step);
90
91 return test_step;
92}
93
94const Options = struct {
95 target: CrossTarget = .{ .cpu_arch = .x86_64, .os_tag = .linux },
96 optimize: std.builtin.OptimizeMode = .Debug,
97 use_llvm: bool = true,
98};
99
100fn addTestStep(b: *Build, comptime prefix: []const u8, opts: Options) *Step {
101 const target = opts.target.zigTriple(b.allocator) catch @panic("OOM");
102 const optimize = @tagName(opts.optimize);
103 const use_llvm = if (opts.use_llvm) "llvm" else "no-llvm";
104 const name = std.fmt.allocPrint(b.allocator, "test-elf-" ++ prefix ++ "-{s}-{s}-{s}", .{
105 target,
106 optimize,
107 use_llvm,
108 }) catch @panic("OOM");
109 return b.step(name, "");
110}
111
112fn addExecutable(b: *Build, opts: Options) *Compile {
113 return b.addExecutable(.{
114 .name = "test",
115 .target = opts.target,
116 .optimize = opts.optimize,
117 .single_threaded = true, // TODO temp until we teach linker how to handle TLS
118 .use_llvm = opts.use_llvm,
119 .use_lld = false,
120 });
121}
122
123fn addRunArtifact(comp: *Compile) *Run {
124 const b = comp.step.owner;
125 const run = b.addRunArtifact(comp);
126 run.skip_foreign_checks = true;
127 return run;
128}
129
130fn addZigSourceBytes(comp: *Compile, bytes: []const u8) void {
131 const b = comp.step.owner;
132 const file = WriteFile.create(b).add("a.zig", bytes);
133 file.addStepDependencies(&comp.step);
134 comp.root_src = file;
135}
136
137fn addCSourceBytes(comp: *Compile, bytes: []const u8) void {
138 const b = comp.step.owner;
139 const file = WriteFile.create(b).add("a.c", bytes);
140 comp.addCSourceFile(.{ .file = file, .flags = &.{} });
141}
142
143const std = @import("std");
144
145const Build = std.Build;
146const Compile = Step.Compile;
147const CrossTarget = std.zig.CrossTarget;
148const LazyPath = Build.LazyPath;
149const Run = Step.Run;
150const Step = Build.Step;
151const WriteFile = Step.WriteFile;