authorgravatar for motiejus@jakstys.ltMotiejus Jakštys <motiejus@jakstys.lt> 2023-04-25 16:57:43+03:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-05-16 20:38:39-07:00
logdf5085bde012773b974f58e8ee28ed90ff686468
tree3b3cc168b824c2196f52160dc021fc916a71eb2d
parentc6966486e3d33054f1dfc822704dc48c62466d54

stage2: implement --build-id styles

7 files changed, 191 insertions(+), 42 deletions(-)

build.zig+4-1
......@@ -165,8 +165,11 @@ pub fn build(b: *std.Build) !void {
165165 exe.strip = strip;
166166 exe.pie = pie;
167167 exe.sanitize_thread = sanitize_thread;
168 exe.build_id = b.option(bool, "build-id", "Include a build id note") orelse false;
169168 exe.entitlements = entitlements;
169
170 if (b.option([]const u8, "build-id", "Include a build id note")) |build_id|
171 exe.build_id = try std.Build.CompileStep.BuildId.parse(b.allocator, build_id);
172
170173 b.installArtifact(exe);
171174
172175 test_step.dependOn(&exe.step);
lib/std/Build/Step/Compile.zig+117-2
......@@ -116,7 +116,7 @@ each_lib_rpath: ?bool = null,
116116/// As an example, the bloaty project refuses to work unless its inputs have
117117/// build ids, in order to prevent accidental mismatches.
118118/// The default is to not include this section because it slows down linking.
119build_id: ?bool = null,
119build_id: ?BuildId = null,
120120
121121/// Create a .eh_frame_hdr section and a PT_GNU_EH_FRAME segment in the ELF
122122/// file.
......@@ -288,6 +288,68 @@ pub const Options = struct {
288288 use_lld: ?bool = null,
289289};
290290
291pub const BuildId = union(enum) {
292 none,
293 fast,
294 uuid,
295 sha1,
296 md5,
297 hexstring: []const u8,
298
299 pub fn hash(self: BuildId, hasher: anytype) void {
300 switch (self) {
301 .none, .fast, .uuid, .sha1, .md5 => {
302 hasher.update(@tagName(self));
303 },
304 .hexstring => |str| {
305 hasher.update("0x");
306 hasher.update(str);
307 },
308 }
309 }
310
311 // parses the incoming BuildId. If returns a hexstring, it is allocated
312 // by the provided allocator.
313 pub fn parse(allocator: std.mem.Allocator, text: []const u8) error{
314 InvalidHexInt,
315 InvalidBuildId,
316 OutOfMemory,
317 }!BuildId {
318 if (mem.eql(u8, text, "none")) {
319 return .none;
320 } else if (mem.eql(u8, text, "fast")) {
321 return .fast;
322 } else if (mem.eql(u8, text, "uuid")) {
323 return .uuid;
324 } else if (mem.eql(u8, text, "sha1") or mem.eql(u8, text, "tree")) {
325 return .sha1;
326 } else if (mem.eql(u8, text, "md5")) {
327 return .md5;
328 } else if (mem.startsWith(u8, text, "0x")) {
329 var clean_hex_string = try allocator.alloc(u8, text.len);
330 errdefer allocator.free(clean_hex_string);
331
332 var i: usize = 0;
333 for (text["0x".len..]) |c| {
334 if (std.ascii.isHex(c)) {
335 clean_hex_string[i] = c;
336 i += 1;
337 } else if (c == '-' or c == ':') {
338 continue;
339 } else {
340 return error.InvalidHexInt;
341 }
342 }
343 if (i < text.len)
344 _ = allocator.resize(clean_hex_string, i);
345
346 return BuildId{ .hexstring = clean_hex_string[0..i] };
347 }
348
349 return error.InvalidBuildId;
350 }
351};
352
291353pub const Kind = enum {
292354 exe,
293355 lib,
......@@ -1810,7 +1872,13 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
18101872
18111873 try addFlag(&zig_args, "valgrind", self.valgrind_support);
18121874 try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
1813 try addFlag(&zig_args, "build-id", self.build_id);
1875 if (self.build_id) |build_id| {
1876 const fmt_str = "--build-id={s}{s}";
1877 try zig_args.append(switch (build_id) {
1878 .hexstring => |str| try std.fmt.allocPrint(b.allocator, fmt_str, .{ "0x", str }),
1879 .none, .fast, .uuid, .sha1, .md5 => try std.fmt.allocPrint(b.allocator, fmt_str, .{ "", @tagName(build_id) }),
1880 });
1881 }
18141882
18151883 if (self.zig_lib_dir) |dir| {
18161884 try zig_args.append("--zig-lib-dir");
......@@ -2175,3 +2243,50 @@ fn checkCompileErrors(self: *Compile) !void {
21752243 \\=========================================
21762244 , .{ expected_generated.items, actual_stderr });
21772245}
2246
2247const testing = std.testing;
2248
2249test "BuildId.parse" {
2250 const tests = &[_]struct {
2251 []const u8,
2252 ?BuildId,
2253 ?anyerror,
2254 }{
2255 .{ "0x", BuildId{ .hexstring = "" }, null },
2256 .{ "0x12-34:", BuildId{ .hexstring = "1234" }, null },
2257 .{ "0x123456", BuildId{ .hexstring = "123456" }, null },
2258 .{ "md5", .md5, null },
2259 .{ "none", .none, null },
2260 .{ "fast", .fast, null },
2261 .{ "uuid", .uuid, null },
2262 .{ "sha1", .sha1, null },
2263 .{ "tree", .sha1, null },
2264 .{ "0xfoobbb", null, error.InvalidHexInt },
2265 .{ "yaddaxxx", null, error.InvalidBuildId },
2266 };
2267
2268 for (tests) |tt| {
2269 const input = tt[0];
2270 const expected = tt[1];
2271 const expected_err = tt[2];
2272
2273 _ = (if (expected_err) |err| {
2274 try testing.expectError(err, BuildId.parse(testing.allocator, input));
2275 } else blk: {
2276 const actual = BuildId.parse(testing.allocator, input) catch |e| break :blk e;
2277 switch (expected.?) {
2278 .hexstring => |expected_str| {
2279 try testing.expectEqualStrings(expected_str, actual.hexstring);
2280 testing.allocator.free(actual.hexstring);
2281 },
2282 else => try testing.expectEqual(expected.?, actual),
2283 }
2284 }) catch |e| {
2285 std.log.err(
2286 "BuildId.parse failed on {s}: expected {} got {!}",
2287 .{ input, expected.?, e },
2288 );
2289 return e;
2290 };
2291 }
2292}
src/Compilation.zig+7-5
......@@ -29,6 +29,7 @@ const wasi_libc = @import("wasi_libc.zig");
2929const fatal = @import("main.zig").fatal;
3030const clangMain = @import("main.zig").clangMain;
3131const Module = @import("Module.zig");
32const BuildId = std.Build.CompileStep.BuildId;
3233const Cache = std.Build.Cache;
3334const translate_c = @import("translate_c.zig");
3435const clang = @import("clang.zig");
......@@ -563,7 +564,7 @@ pub const InitOptions = struct {
563564 linker_print_map: bool = false,
564565 linker_opt_bisect_limit: i32 = -1,
565566 each_lib_rpath: ?bool = null,
566 build_id: ?bool = null,
567 build_id: ?BuildId = null,
567568 disable_c_depfile: bool = false,
568569 linker_z_nodelete: bool = false,
569570 linker_z_notext: bool = false,
......@@ -797,7 +798,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
797798 const unwind_tables = options.want_unwind_tables orelse
798799 (link_libunwind or target_util.needUnwindTables(options.target));
799800 const link_eh_frame_hdr = options.link_eh_frame_hdr or unwind_tables;
800 const build_id = options.build_id orelse false;
801801
802802 // Make a decision on whether to use LLD or our own linker.
803803 const use_lld = options.use_lld orelse blk: {
......@@ -828,7 +828,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
828828 options.output_mode == .Lib or
829829 options.linker_script != null or options.version_script != null or
830830 options.emit_implib != null or
831 build_id or
831 options.build_id != null or
832832 options.symbol_wrap_set.count() > 0)
833833 {
834834 break :blk true;
......@@ -1514,7 +1514,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
15141514 .skip_linker_dependencies = options.skip_linker_dependencies,
15151515 .parent_compilation_link_libc = options.parent_compilation_link_libc,
15161516 .each_lib_rpath = options.each_lib_rpath orelse options.is_native_os,
1517 .build_id = build_id,
1517 .build_id = options.build_id,
15181518 .cache_mode = cache_mode,
15191519 .disable_lld_caching = options.disable_lld_caching or cache_mode == .whole,
15201520 .subsystem = options.subsystem,
......@@ -2269,7 +2269,9 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
22692269 man.hash.addListOfBytes(comp.bin_file.options.rpath_list);
22702270 man.hash.addListOfBytes(comp.bin_file.options.symbol_wrap_set.keys());
22712271 man.hash.add(comp.bin_file.options.each_lib_rpath);
2272 man.hash.add(comp.bin_file.options.build_id);
2272 if (comp.bin_file.options.build_id) |build_id| {
2273 build_id.hash(&man.hash.hasher);
2274 }
22732275 man.hash.add(comp.bin_file.options.skip_linker_dependencies);
22742276 man.hash.add(comp.bin_file.options.z_nodelete);
22752277 man.hash.add(comp.bin_file.options.z_notext);
src/link.zig+2-1
......@@ -10,6 +10,7 @@ const wasi_libc = @import("wasi_libc.zig");
1010
1111const Air = @import("Air.zig");
1212const Allocator = std.mem.Allocator;
13const BuildId = std.Build.CompileStep.BuildId;
1314const Cache = std.Build.Cache;
1415const Compilation = @import("Compilation.zig");
1516const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
......@@ -157,7 +158,7 @@ pub const Options = struct {
157158 skip_linker_dependencies: bool,
158159 parent_compilation_link_libc: bool,
159160 each_lib_rpath: bool,
160 build_id: bool,
161 build_id: ?BuildId,
161162 disable_lld_caching: bool,
162163 is_test: bool,
163164 hash_style: HashStyle,
src/link/Elf.zig+8-3
......@@ -1399,7 +1399,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v
13991399 man.hash.add(self.base.options.each_lib_rpath);
14001400 if (self.base.options.output_mode == .Exe) {
14011401 man.hash.add(stack_size);
1402 man.hash.add(self.base.options.build_id);
1402 if (self.base.options.build_id) |build_id|
1403 build_id.hash(&man.hash.hasher);
14031404 }
14041405 man.hash.addListOfBytes(self.base.options.symbol_wrap_set.keys());
14051406 man.hash.add(self.base.options.skip_linker_dependencies);
......@@ -1542,8 +1543,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v
15421543 try argv.append("-z");
15431544 try argv.append(try std.fmt.allocPrint(arena, "stack-size={d}", .{stack_size}));
15441545
1545 if (self.base.options.build_id) {
1546 try argv.append("--build-id");
1546 if (self.base.options.build_id) |build_id| {
1547 const fmt_str = "--build-id={s}{s}";
1548 try argv.append(switch (build_id) {
1549 .hexstring => |str| try std.fmt.allocPrint(arena, fmt_str, .{ "0x", str }),
1550 .none, .fast, .uuid, .sha1, .md5 => try std.fmt.allocPrint(arena, fmt_str, .{ "", @tagName(build_id) }),
1551 });
15471552 }
15481553 }
15491554
src/link/Wasm.zig+31-18
......@@ -3163,7 +3163,8 @@ fn linkWithZld(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) l
31633163 try man.addOptionalFile(compiler_rt_path);
31643164 man.hash.addOptionalBytes(options.entry);
31653165 man.hash.addOptional(options.stack_size_override);
3166 man.hash.add(wasm.base.options.build_id);
3166 if (wasm.base.options.build_id) |build_id|
3167 build_id.hash(&man.hash.hasher);
31673168 man.hash.add(options.import_memory);
31683169 man.hash.add(options.import_table);
31693170 man.hash.add(options.export_table);
......@@ -3797,8 +3798,27 @@ fn writeToFile(
37973798 if (!wasm.base.options.strip) {
37983799 // The build id must be computed on the main sections only,
37993800 // so we have to do it now, before the debug sections.
3800 if (wasm.base.options.build_id) {
3801 try emitBuildIdSection(&binary_bytes);
3801 if (wasm.base.options.build_id) |build_id| {
3802 switch (build_id) {
3803 .none => {},
3804 .fast => {
3805 var id: [16]u8 = undefined;
3806 std.crypto.hash.sha3.TurboShake128(null).hash(binary_bytes.items, &id, .{});
3807 var uuid: [36]u8 = undefined;
3808 _ = try std.fmt.bufPrint(&uuid, "{s}-{s}-{s}-{s}-{s}", .{
3809 std.fmt.fmtSliceHexLower(id[0..4]),
3810 std.fmt.fmtSliceHexLower(id[4..6]),
3811 std.fmt.fmtSliceHexLower(id[6..8]),
3812 std.fmt.fmtSliceHexLower(id[8..10]),
3813 std.fmt.fmtSliceHexLower(id[10..]),
3814 });
3815 try emitBuildIdSection(&binary_bytes, &uuid);
3816 },
3817 .hexstring => |str| {
3818 try emitBuildIdSection(&binary_bytes, str);
3819 },
3820 else => |mode| log.err("build-id '{s}' is not supported for WASM", .{@tagName(mode)}),
3821 }
38023822 }
38033823
38043824 // if (wasm.dwarf) |*dwarf| {
......@@ -3942,25 +3962,17 @@ fn emitProducerSection(binary_bytes: *std.ArrayList(u8)) !void {
39423962 );
39433963}
39443964
3945fn emitBuildIdSection(binary_bytes: *std.ArrayList(u8)) !void {
3965fn emitBuildIdSection(binary_bytes: *std.ArrayList(u8), build_id: []const u8) !void {
39463966 const header_offset = try reserveCustomSectionHeader(binary_bytes);
39473967
39483968 const writer = binary_bytes.writer();
3949 const build_id = "build_id";
3950 try leb.writeULEB128(writer, @intCast(u32, build_id.len));
3951 try writer.writeAll(build_id);
3952
3953 var id: [16]u8 = undefined;
3954 std.crypto.hash.sha3.TurboShake128(null).hash(binary_bytes.items, &id, .{});
3955 var uuid: [36]u8 = undefined;
3956 _ = try std.fmt.bufPrint(&uuid, "{s}-{s}-{s}-{s}-{s}", .{
3957 std.fmt.fmtSliceHexLower(id[0..4]), std.fmt.fmtSliceHexLower(id[4..6]), std.fmt.fmtSliceHexLower(id[6..8]),
3958 std.fmt.fmtSliceHexLower(id[8..10]), std.fmt.fmtSliceHexLower(id[10..]),
3959 });
3969 const hdr_build_id = "build_id";
3970 try leb.writeULEB128(writer, @intCast(u32, hdr_build_id.len));
3971 try writer.writeAll(hdr_build_id);
39603972
39613973 try leb.writeULEB128(writer, @as(u32, 1));
3962 try leb.writeULEB128(writer, @as(u32, uuid.len));
3963 try writer.writeAll(&uuid);
3974 try leb.writeULEB128(writer, @intCast(u32, build_id.len));
3975 try writer.writeAll(build_id);
39643976
39653977 try writeCustomSectionHeader(
39663978 binary_bytes.items,
......@@ -4199,7 +4211,8 @@ fn linkWithLLD(wasm: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) !
41994211 try man.addOptionalFile(compiler_rt_path);
42004212 man.hash.addOptionalBytes(wasm.base.options.entry);
42014213 man.hash.addOptional(wasm.base.options.stack_size_override);
4202 man.hash.add(wasm.base.options.build_id);
4214 if (wasm.base.options.build_id) |build_id|
4215 build_id.hash(&man.hash.hasher);
42034216 man.hash.add(wasm.base.options.import_memory);
42044217 man.hash.add(wasm.base.options.import_table);
42054218 man.hash.add(wasm.base.options.export_table);
src/main.zig+22-12
......@@ -22,6 +22,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
2222const wasi_libc = @import("wasi_libc.zig");
2323const translate_c = @import("translate_c.zig");
2424const clang = @import("clang.zig");
25const BuildId = std.Build.CompileStep.BuildId;
2526const Cache = std.Build.Cache;
2627const target_util = @import("target.zig");
2728const crash_report = @import("crash_report.zig");
......@@ -493,8 +494,7 @@ const usage_build_generic =
493494 \\ -fno-each-lib-rpath Prevent adding rpath for each used dynamic library
494495 \\ -fallow-shlib-undefined Allows undefined symbols in shared libraries
495496 \\ -fno-allow-shlib-undefined Disallows undefined symbols in shared libraries
496 \\ -fbuild-id Helps coordinate stripped binaries with debug symbols
497 \\ -fno-build-id (default) Saves a bit of time linking
497 \\ --build-id[=style] Generate a build ID note
498498 \\ --eh-frame-hdr Enable C++ exception handling by passing --eh-frame-hdr to linker
499499 \\ --emit-relocs Enable output of relocation sections for post build tools
500500 \\ -z [arg] Set linker extension flags
......@@ -817,7 +817,7 @@ fn buildOutputType(
817817 var link_eh_frame_hdr = false;
818818 var link_emit_relocs = false;
819819 var each_lib_rpath: ?bool = null;
820 var build_id: ?bool = null;
820 var build_id: ?BuildId = null;
821821 var sysroot: ?[]const u8 = null;
822822 var libc_paths_file: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIBC");
823823 var machine_code_model: std.builtin.CodeModel = .default;
......@@ -1202,10 +1202,6 @@ fn buildOutputType(
12021202 each_lib_rpath = true;
12031203 } else if (mem.eql(u8, arg, "-fno-each-lib-rpath")) {
12041204 each_lib_rpath = false;
1205 } else if (mem.eql(u8, arg, "-fbuild-id")) {
1206 build_id = true;
1207 } else if (mem.eql(u8, arg, "-fno-build-id")) {
1208 build_id = false;
12091205 } else if (mem.eql(u8, arg, "--test-cmd-bin")) {
12101206 try test_exec_args.append(null);
12111207 } else if (mem.eql(u8, arg, "--test-evented-io")) {
......@@ -1446,6 +1442,15 @@ fn buildOutputType(
14461442 linker_gc_sections = true;
14471443 } else if (mem.eql(u8, arg, "--no-gc-sections")) {
14481444 linker_gc_sections = false;
1445 } else if (mem.eql(u8, arg, "--build-id")) {
1446 build_id = .fast;
1447 } else if (mem.startsWith(u8, arg, "--build-id=")) {
1448 const value = arg["--build-id=".len..];
1449 build_id = BuildId.parse(arena, value) catch |err| switch (err) {
1450 error.InvalidHexInt => fatal("failed to parse hex value {s}", .{value}),
1451 error.InvalidBuildId => fatal("invalid --build-id={s}", .{value}),
1452 error.OutOfMemory => fatal("OOM", .{}),
1453 };
14491454 } else if (mem.eql(u8, arg, "--debug-compile-errors")) {
14501455 if (!crash_report.is_enabled) {
14511456 std.log.warn("Zig was compiled in a release mode. --debug-compile-errors has no effect.", .{});
......@@ -1684,11 +1689,7 @@ fn buildOutputType(
16841689 if (mem.indexOfScalar(u8, linker_arg, '=')) |equals_pos| {
16851690 const key = linker_arg[0..equals_pos];
16861691 const value = linker_arg[equals_pos + 1 ..];
1687 if (mem.eql(u8, key, "build-id")) {
1688 build_id = true;
1689 warn("ignoring build-id style argument: '{s}'", .{value});
1690 continue;
1691 } else if (mem.eql(u8, key, "--sort-common")) {
1692 if (mem.eql(u8, key, "--sort-common")) {
16921693 // this ignores --sort=common=<anything>; ignoring plain --sort-common
16931694 // is done below.
16941695 continue;
......@@ -1730,6 +1731,15 @@ fn buildOutputType(
17301731 search_strategy = .paths_first;
17311732 } else if (mem.eql(u8, linker_arg, "-search_dylibs_first")) {
17321733 search_strategy = .dylibs_first;
1734 } else if (mem.eql(u8, linker_arg, "--build-id")) {
1735 build_id = .fast;
1736 } else if (mem.startsWith(u8, linker_arg, "--build-id=")) {
1737 const value = linker_arg["--build-id=".len..];
1738 build_id = BuildId.parse(arena, value) catch |err| switch (err) {
1739 error.InvalidHexInt => fatal("failed to parse hex value {s}", .{value}),
1740 error.InvalidBuildId => fatal("invalid --build-id={s}", .{value}),
1741 error.OutOfMemory => fatal("OOM", .{}),
1742 };
17331743 } else {
17341744 try linker_args.append(linker_arg);
17351745 }