authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-09-13 23:02:21+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2021-09-13 23:02:21+02:00
log760241ce50eaa9031339f6b591358b53f5797486
tree713b4ad62085eba2c29390d8003866f3290e16ce
parent46a10401f035b50122af9a91348edeb3f57e864e

macho: use the cache system to know if need to relink objects

This applies to stage2 where we make use of the cache system to work out if we need to relink objects when performing incremental updates. When the process is restarted however, while in principle the idea is to carry on where we left off by reparsing the prelinked binary from file, the required machinery is not there yet, and therefore we always fully relink upon restart.

2 files changed, 51 insertions(+), 66 deletions(-)

src/link/MachO.zig+40-55
...@@ -166,11 +166,13 @@ error_flags: File.ErrorFlags = File.ErrorFlags{},...@@ -166,11 +166,13 @@ error_flags: File.ErrorFlags = File.ErrorFlags{},
166166
167load_commands_dirty: bool = false,167load_commands_dirty: bool = false,
168sections_order_dirty: bool = false,168sections_order_dirty: bool = false,
169
170has_dices: bool = false,169has_dices: bool = false,
171has_stabs: bool = false,170has_stabs: bool = false,
172171/// A helper var to indicate if we are at the start of the incremental updates, or
173args_digest: [Cache.hex_digest_len]u8 = undefined,172/// already somewhere further along the update-and-run chain.
173/// TODO once we add opening a prelinked output binary from file, this will become
174/// obsolete as we will carry on where we left off.
175cold_start: bool = false,
174176
175section_ordinals: std.AutoArrayHashMapUnmanaged(MatchingSection, void) = .{},177section_ordinals: std.AutoArrayHashMapUnmanaged(MatchingSection, void) = .{},
176178
...@@ -336,6 +338,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio...@@ -336,6 +338,7 @@ pub fn openPath(allocator: *Allocator, sub_path: []const u8, options: link.Optio
336 return self;338 return self;
337 }339 }
338340
341 // TODO Migrate DebugSymbols to the merged linker codepaths
339 // if (!options.strip and options.module != null) {342 // if (!options.strip and options.module != null) {
340 // // Create dSYM bundle.343 // // Create dSYM bundle.
341 // const dir = options.module.?.zig_cache_artifact_directory;344 // const dir = options.module.?.zig_cache_artifact_directory;
...@@ -456,8 +459,11 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -456,8 +459,11 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
456 defer if (!self.base.options.disable_lld_caching) man.deinit();459 defer if (!self.base.options.disable_lld_caching) man.deinit();
457460
458 var digest: [Cache.hex_digest_len]u8 = undefined;461 var digest: [Cache.hex_digest_len]u8 = undefined;
462 var needs_full_relink = true;
463
464 cache: {
465 if (use_stage1 and self.base.options.disable_lld_caching) break :cache;
459466
460 if (!self.base.options.disable_lld_caching) {
461 man = comp.cache_parent.obtain();467 man = comp.cache_parent.obtain();
462468
463 // We are about to obtain this lock, so here we give other processes a chance first.469 // We are about to obtain this lock, so here we give other processes a chance first.
...@@ -491,17 +497,36 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -491,17 +497,36 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
491 id_symlink_basename,497 id_symlink_basename,
492 &prev_digest_buf,498 &prev_digest_buf,
493 ) catch |err| blk: {499 ) catch |err| blk: {
494 log.debug("MachO Zld new_digest={s} error: {s}", .{ std.fmt.fmtSliceHexLower(&digest), @errorName(err) });500 log.debug("MachO Zld new_digest={s} error: {s}", .{
501 std.fmt.fmtSliceHexLower(&digest),
502 @errorName(err),
503 });
495 // Handle this as a cache miss.504 // Handle this as a cache miss.
496 break :blk prev_digest_buf[0..0];505 break :blk prev_digest_buf[0..0];
497 };506 };
498 if (mem.eql(u8, prev_digest, &digest)) {507 if (mem.eql(u8, prev_digest, &digest)) {
499 log.debug("MachO Zld digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
500 // Hot diggity dog! The output binary is already there.508 // Hot diggity dog! The output binary is already there.
501 self.base.lock = man.toOwnedLock();509
502 return;510 if (use_stage1) {
511 log.debug("MachO Zld digest={s} match - skipping invocation", .{std.fmt.fmtSliceHexLower(&digest)});
512 self.base.lock = man.toOwnedLock();
513 return;
514 } else {
515 log.debug("MachO Zld digest={s} match", .{std.fmt.fmtSliceHexLower(&digest)});
516 if (!self.cold_start) {
517 log.debug(" no need to relink objects", .{});
518 needs_full_relink = false;
519 } else {
520 log.debug(" TODO parse prelinked binary and continue linking where we left off", .{});
521 // TODO until such time however, perform a full relink of objects.
522 needs_full_relink = true;
523 }
524 }
503 }525 }
504 log.debug("MachO Zld prev_digest={s} new_digest={s}", .{ std.fmt.fmtSliceHexLower(prev_digest), std.fmt.fmtSliceHexLower(&digest) });526 log.debug("MachO Zld prev_digest={s} new_digest={s}", .{
527 std.fmt.fmtSliceHexLower(prev_digest),
528 std.fmt.fmtSliceHexLower(&digest),
529 });
505530
506 // We are about to change the output file to be different, so we invalidate the build hash now.531 // We are about to change the output file to be different, so we invalidate the build hash now.
507 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {532 directory.handle.deleteFile(id_symlink_basename) catch |err| switch (err) {
...@@ -509,7 +534,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -509,7 +534,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
509 else => |e| return e,534 else => |e| return e,
510 };535 };
511 }536 }
512
513 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});537 const full_out_path = try directory.join(arena, &[_][]const u8{self.base.options.emit.?.sub_path});
514538
515 if (self.base.options.output_mode == .Obj) {539 if (self.base.options.output_mode == .Obj) {
...@@ -557,34 +581,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -557,34 +581,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
557 try self.strtab.append(self.base.allocator, 0);581 try self.strtab.append(self.base.allocator, 0);
558 }582 }
559583
560 const needs_full_relink = blk: {
561 if (use_stage1) break :blk true;
562
563 var hh: Cache.HashHelper = .{};
564 hh.addListOfBytes(self.base.options.objects);
565 for (comp.c_object_table.keys()) |key| {
566 hh.addBytes(key.status.success.object_path);
567 }
568 hh.addOptionalBytes(module_obj_path);
569 if (comp.compiler_rt_static_lib) |lib| {
570 hh.addBytes(lib.full_object_path);
571 }
572 if (self.base.options.link_libcpp) {
573 hh.addBytes(comp.libcxxabi_static_lib.?.full_object_path);
574 hh.addBytes(comp.libcxx_static_lib.?.full_object_path);
575 }
576 hh.addListOfBytes(self.base.options.lib_dirs);
577 hh.addListOfBytes(self.base.options.framework_dirs);
578 hh.addListOfBytes(self.base.options.frameworks);
579 hh.addListOfBytes(self.base.options.rpath_list);
580 hh.addStringSet(self.base.options.system_libs);
581 hh.addOptionalBytes(self.base.options.sysroot);
582 const new_digest = hh.final();
583 const needs_full_relink = !mem.eql(u8, &new_digest, &self.args_digest);
584 mem.copy(u8, &self.args_digest, &new_digest);
585 break :blk needs_full_relink;
586 };
587
588 if (needs_full_relink) {584 if (needs_full_relink) {
589 self.objects.clearRetainingCapacity();585 self.objects.clearRetainingCapacity();
590 self.archives.clearRetainingCapacity();586 self.archives.clearRetainingCapacity();
...@@ -848,22 +844,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -848,22 +844,6 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
848 try self.allocateGlobalSymbols();844 try self.allocateGlobalSymbols();
849 try self.writeAtoms();845 try self.writeAtoms();
850846
851 // log.warn("Locals:", .{});
852 // for (self.locals.items) |sym, i| {
853 // log.warn(" => {d}: {s}, {}", .{ i, self.getString(sym.n_strx), sym });
854 // }
855 // log.warn("Globals:", .{});
856 // for (self.globals.items) |sym, i| {
857 // log.warn(" => {d}: {s} {}", .{ i, self.getString(sym.n_strx), sym });
858 // }
859 // {
860 // log.warn("Resolver:", .{});
861 // var it = self.symbol_resolver.iterator();
862 // while (it.next()) |entry| {
863 // log.warn(" => {s}: {}", .{ self.getString(entry.key_ptr.*), entry.value_ptr.* });
864 // }
865 // }
866
867 if (self.bss_section_index) |idx| {847 if (self.bss_section_index) |idx| {
868 const seg = &self.load_commands.items[self.data_segment_cmd_index.?].Segment;848 const seg = &self.load_commands.items[self.data_segment_cmd_index.?].Segment;
869 const sect = &seg.sections.items[idx];849 const sect = &seg.sections.items[idx];
...@@ -880,7 +860,8 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -880,7 +860,8 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
880 try self.flushModule(comp);860 try self.flushModule(comp);
881 }861 }
882862
883 if (!self.base.options.disable_lld_caching) {863 cache: {
864 if (use_stage1 and self.base.options.disable_lld_caching) break :cache;
884 // Update the file with the digest. If it fails we can continue; it only865 // Update the file with the digest. If it fails we can continue; it only
885 // means that the next invocation will have an unnecessary cache miss.866 // means that the next invocation will have an unnecessary cache miss.
886 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {867 Cache.writeSmallFile(directory.handle, id_symlink_basename, &digest) catch |err| {
...@@ -894,6 +875,8 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {...@@ -894,6 +875,8 @@ pub fn flush(self: *MachO, comp: *Compilation) !void {
894 // other processes clobbering it.875 // other processes clobbering it.
895 self.base.lock = man.toOwnedLock();876 self.base.lock = man.toOwnedLock();
896 }877 }
878
879 self.cold_start = false;
897}880}
898881
899pub fn flushModule(self: *MachO, comp: *Compilation) !void {882pub fn flushModule(self: *MachO, comp: *Compilation) !void {
...@@ -3929,6 +3912,8 @@ pub fn populateMissingMetadata(self: *MachO) !void {...@@ -3929,6 +3912,8 @@ pub fn populateMissingMetadata(self: *MachO) !void {
3929 });3912 });
3930 self.load_commands_dirty = true;3913 self.load_commands_dirty = true;
3931 }3914 }
3915
3916 self.cold_start = true;
3932}3917}
39333918
3934const AllocateSectionOpts = struct {3919const AllocateSectionOpts = struct {
test/stage2/darwin.zig+11-11
...@@ -27,8 +27,8 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -27,8 +27,8 @@ pub fn addCases(ctx: *TestContext) !void {
2727
28 // Regular old hello world28 // Regular old hello world
29 case.addCompareOutput(29 case.addCompareOutput(
30 \\extern "c" fn write(usize, usize, usize) usize;30 \\extern fn write(usize, usize, usize) usize;
31 \\extern "c" fn exit(usize) noreturn;31 \\extern fn exit(usize) noreturn;
32 \\32 \\
33 \\pub export fn main() noreturn {33 \\pub export fn main() noreturn {
34 \\ print();34 \\ print();
...@@ -47,8 +47,8 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -47,8 +47,8 @@ pub fn addCases(ctx: *TestContext) !void {
4747
48 // Print it 4 times and force growth and realloc.48 // Print it 4 times and force growth and realloc.
49 case.addCompareOutput(49 case.addCompareOutput(
50 \\extern "c" fn write(usize, usize, usize) usize;50 \\extern fn write(usize, usize, usize) usize;
51 \\extern "c" fn exit(usize) noreturn;51 \\extern fn exit(usize) noreturn;
52 \\52 \\
53 \\pub export fn main() noreturn {53 \\pub export fn main() noreturn {
54 \\ print();54 \\ print();
...@@ -74,8 +74,8 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -74,8 +74,8 @@ pub fn addCases(ctx: *TestContext) !void {
7474
75 // Print it once, and change the message.75 // Print it once, and change the message.
76 case.addCompareOutput(76 case.addCompareOutput(
77 \\extern "c" fn write(usize, usize, usize) usize;77 \\extern fn write(usize, usize, usize) usize;
78 \\extern "c" fn exit(usize) noreturn;78 \\extern fn exit(usize) noreturn;
79 \\79 \\
80 \\pub export fn main() noreturn {80 \\pub export fn main() noreturn {
81 \\ print();81 \\ print();
...@@ -94,8 +94,8 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -94,8 +94,8 @@ pub fn addCases(ctx: *TestContext) !void {
9494
95 // Now we print it twice.95 // Now we print it twice.
96 case.addCompareOutput(96 case.addCompareOutput(
97 \\extern "c" fn write(usize, usize, usize) usize;97 \\extern fn write(usize, usize, usize) usize;
98 \\extern "c" fn exit(usize) noreturn;98 \\extern fn exit(usize) noreturn;
99 \\99 \\
100 \\pub export fn main() noreturn {100 \\pub export fn main() noreturn {
101 \\ print();101 \\ print();
...@@ -121,7 +121,7 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -121,7 +121,7 @@ pub fn addCases(ctx: *TestContext) !void {
121 // This test case also covers an infrequent scenarion where the string table *may* be relocated121 // This test case also covers an infrequent scenarion where the string table *may* be relocated
122 // into the position preceeding the symbol table which results in a dyld error.122 // into the position preceeding the symbol table which results in a dyld error.
123 case.addCompareOutput(123 case.addCompareOutput(
124 \\extern "c" fn exit(usize) noreturn;124 \\extern fn exit(usize) noreturn;
125 \\125 \\
126 \\pub export fn main() noreturn {126 \\pub export fn main() noreturn {
127 \\ exit(0);127 \\ exit(0);
...@@ -131,8 +131,8 @@ pub fn addCases(ctx: *TestContext) !void {...@@ -131,8 +131,8 @@ pub fn addCases(ctx: *TestContext) !void {
131 );131 );
132132
133 case.addCompareOutput(133 case.addCompareOutput(
134 \\extern "c" fn exit(usize) noreturn;134 \\extern fn exit(usize) noreturn;
135 \\extern "c" fn write(usize, usize, usize) usize;135 \\extern fn write(usize, usize, usize) usize;
136 \\136 \\
137 \\pub export fn main() noreturn {137 \\pub export fn main() noreturn {
138 \\ _ = write(1, @ptrToInt("Hey!\n"), 5);138 \\ _ = write(1, @ptrToInt("Hey!\n"), 5);