authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-06-19 11:26:56+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-06-19 11:26:56+02:00
log10aaf2983d5db65082c4b348269150eddc12e67e
tree48fb1227fb3660db6b5b953e37fd52450f2b1717
parentb3a2ab3fedfc2e3e15f9024c7334a1f53d9aa7c5

macho: hash the entire file contents for UUID but calc in parallel

3 files changed, 67 insertions(+), 166 deletions(-)

src/link/MachO.zig+12-7
...@@ -13,6 +13,7 @@ const mem = std.mem;...@@ -13,6 +13,7 @@ const mem = std.mem;
13const meta = std.meta;13const meta = std.meta;
1414
15const aarch64 = @import("../arch/aarch64/bits.zig");15const aarch64 = @import("../arch/aarch64/bits.zig");
16const calcUuid = @import("MachO/uuid.zig").calcUuid;
16const codegen = @import("../codegen.zig");17const codegen = @import("../codegen.zig");
17const dead_strip = @import("MachO/dead_strip.zig");18const dead_strip = @import("MachO/dead_strip.zig");
18const fat = @import("MachO/fat.zig");19const fat = @import("MachO/fat.zig");
...@@ -756,11 +757,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -756,11 +757,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
756 });757 });
757 try load_commands.writeBuildVersionLC(&self.base.options, lc_writer);758 try load_commands.writeBuildVersionLC(&self.base.options, lc_writer);
758759
759 if (self.cold_start) {760 const uuid_cmd_offset = @sizeOf(macho.mach_header_64) + @intCast(u32, lc_buffer.items.len);
760 std.crypto.random.bytes(&self.uuid_cmd.uuid);
761 Md5.hash(&self.uuid_cmd.uuid, &self.uuid_cmd.uuid, .{});
762 conformUuid(&self.uuid_cmd.uuid);
763 }
764 try lc_writer.writeStruct(self.uuid_cmd);761 try lc_writer.writeStruct(self.uuid_cmd);
765762
766 try load_commands.writeLoadDylibLCs(self.dylibs.items, self.referenced_dylibs.keys(), lc_writer);763 try load_commands.writeLoadDylibLCs(self.dylibs.items, self.referenced_dylibs.keys(), lc_writer);
...@@ -769,10 +766,10 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -769,10 +766,10 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
769 try lc_writer.writeStruct(self.codesig_cmd);766 try lc_writer.writeStruct(self.codesig_cmd);
770 }767 }
771768
772 try self.base.file.?.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64));
773
774 const ncmds = load_commands.calcNumOfLCs(lc_buffer.items);769 const ncmds = load_commands.calcNumOfLCs(lc_buffer.items);
770 try self.base.file.?.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64));
775 try self.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len));771 try self.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len));
772 try self.writeUuid(comp, uuid_cmd_offset);
776773
777 if (codesig) |*csig| {774 if (codesig) |*csig| {
778 try self.writeCodeSignature(comp, csig); // code signing always comes last775 try self.writeCodeSignature(comp, csig); // code signing always comes last
...@@ -3510,6 +3507,14 @@ fn writeDysymtab(self: *MachO, ctx: SymtabCtx) !void {...@@ -3510,6 +3507,14 @@ fn writeDysymtab(self: *MachO, ctx: SymtabCtx) !void {
3510 self.dysymtab_cmd.nindirectsyms = nindirectsyms;3507 self.dysymtab_cmd.nindirectsyms = nindirectsyms;
3511}3508}
35123509
3510fn writeUuid(self: *MachO, comp: *const Compilation, uuid_cmd_offset: u32) !void {
3511 const seg = self.getLinkeditSegmentPtr();
3512 const file_size = seg.fileoff + seg.filesize;
3513 try calcUuid(comp, self.base.file.?, file_size, &self.uuid_cmd.uuid);
3514 const offset = uuid_cmd_offset + @sizeOf(macho.load_command);
3515 try self.base.file.?.pwriteAll(&self.uuid_cmd.uuid, offset);
3516}
3517
3513fn writeCodeSignaturePadding(self: *MachO, code_sig: *CodeSignature) !void {3518fn writeCodeSignaturePadding(self: *MachO, code_sig: *CodeSignature) !void {
3514 const seg = self.getLinkeditSegmentPtr();3519 const seg = self.getLinkeditSegmentPtr();
3515 // Code signature data has to be 16-bytes aligned for Apple tools to recognize the file3520 // Code signature data has to be 16-bytes aligned for Apple tools to recognize the file
src/link/MachO/uuid.zig created+46
...@@ -0,0 +1,46 @@
1const std = @import("std");
2const fs = std.fs;
3const mem = std.mem;
4
5const Allocator = mem.Allocator;
6const Compilation = @import("../../Compilation.zig");
7const Md5 = std.crypto.hash.Md5;
8const Hasher = @import("hasher.zig").ParallelHasher;
9
10/// Somewhat random chunk size for MD5 hash calculation.
11pub const chunk_size = 0x4000;
12
13/// Calculates Md5 hash of each chunk in parallel and then hashes all Md5 hashes to produce
14/// the final digest.
15/// While this is NOT a correct MD5 hash of the contents, this methodology is used by LLVM/LLD
16/// and we will use it too as it seems accepted by Apple OSes.
17/// TODO LLD also hashes the output filename to disambiguate between same builds with different
18/// output files. Should we also do that?
19pub fn calcUuid(comp: *const Compilation, file: fs.File, file_size: u64, out: *[Md5.digest_length]u8) !void {
20 const total_hashes = mem.alignForward(u64, file_size, chunk_size) / chunk_size;
21
22 const hashes = try comp.gpa.alloc([Md5.digest_length]u8, total_hashes);
23 defer comp.gpa.free(hashes);
24
25 var hasher = Hasher(Md5){};
26 try hasher.hash(comp.gpa, comp.thread_pool, file, hashes, .{
27 .chunk_size = chunk_size,
28 .max_file_size = file_size,
29 });
30
31 const final_buffer = try comp.gpa.alloc(u8, total_hashes * Md5.digest_length);
32 defer comp.gpa.free(final_buffer);
33
34 for (hashes, 0..) |hash, i| {
35 mem.copy(u8, final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);
36 }
37
38 Md5.hash(final_buffer, out, .{});
39 conform(out);
40}
41
42inline fn conform(out: *[Md5.digest_length]u8) void {
43 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats
44 out[6] = (out[6] & 0x0F) | (3 << 4);
45 out[8] = (out[8] & 0x3F) | 0x80;
46}
src/link/MachO/zld.zig+9-159
...@@ -9,14 +9,15 @@ const math = std.math;...@@ -9,14 +9,15 @@ const math = std.math;
9const mem = std.mem;9const mem = std.mem;
1010
11const aarch64 = @import("../../arch/aarch64/bits.zig");11const aarch64 = @import("../../arch/aarch64/bits.zig");
12const calcUuid = @import("uuid.zig").calcUuid;
12const dead_strip = @import("dead_strip.zig");13const dead_strip = @import("dead_strip.zig");
13const eh_frame = @import("eh_frame.zig");14const eh_frame = @import("eh_frame.zig");
14const fat = @import("fat.zig");15const fat = @import("fat.zig");
15const link = @import("../../link.zig");16const link = @import("../../link.zig");
16const load_commands = @import("load_commands.zig");17const load_commands = @import("load_commands.zig");
18const stub_helpers = @import("stubs.zig");
17const thunks = @import("thunks.zig");19const thunks = @import("thunks.zig");
18const trace = @import("../../tracy.zig").trace;20const trace = @import("../../tracy.zig").trace;
19const stub_helpers = @import("stubs.zig");
2021
21const Allocator = mem.Allocator;22const Allocator = mem.Allocator;
22const Archive = @import("Archive.zig");23const Archive = @import("Archive.zig");
...@@ -2575,150 +2576,12 @@ pub const Zld = struct {...@@ -2575,150 +2576,12 @@ pub const Zld = struct {
2575 self.dysymtab_cmd.nindirectsyms = nindirectsyms;2576 self.dysymtab_cmd.nindirectsyms = nindirectsyms;
2576 }2577 }
25772578
2578 fn writeUuid(self: *Zld, comp: *const Compilation, args: struct {2579 fn writeUuid(self: *Zld, comp: *const Compilation, uuid_cmd_offset: u32) !void {
2579 linkedit_cmd_offset: u32,2580 const seg = self.getLinkeditSegmentPtr();
2580 symtab_cmd_offset: u32,2581 const file_size = seg.fileoff + seg.filesize;
2581 uuid_cmd_offset: u32,2582 try calcUuid(comp, self.file, file_size, &self.uuid_cmd.uuid);
2582 codesig_cmd_offset: ?u32,2583 const offset = uuid_cmd_offset + @sizeOf(macho.load_command);
2583 }) !void {2584 try self.file.pwriteAll(&self.uuid_cmd.uuid, offset);
2584 _ = comp;
2585 switch (self.options.optimize_mode) {
2586 .Debug => {
2587 // In Debug we don't really care about reproducibility, so put in a random value
2588 // and be done with it.
2589 std.crypto.random.bytes(&self.uuid_cmd.uuid);
2590 Md5.hash(&self.uuid_cmd.uuid, &self.uuid_cmd.uuid, .{});
2591 conformUuid(&self.uuid_cmd.uuid);
2592 },
2593 else => {
2594 // We set the max file size to the actual strtab buffer length to exclude any strtab padding.
2595 const max_file_end = @intCast(u32, self.symtab_cmd.stroff + self.strtab.buffer.items.len);
2596
2597 const FileSubsection = struct {
2598 start: u32,
2599 end: u32,
2600 };
2601
2602 var subsections: [5]FileSubsection = undefined;
2603 var count: usize = 0;
2604
2605 // Exclude LINKEDIT segment command as it contains file size that includes stabs contribution
2606 // and code signature.
2607 subsections[count] = .{
2608 .start = 0,
2609 .end = args.linkedit_cmd_offset,
2610 };
2611 count += 1;
2612
2613 // Exclude SYMTAB and DYSYMTAB commands for the same reason.
2614 subsections[count] = .{
2615 .start = subsections[count - 1].end + @sizeOf(macho.segment_command_64),
2616 .end = args.symtab_cmd_offset,
2617 };
2618 count += 1;
2619
2620 // Exclude CODE_SIGNATURE command (if present).
2621 if (args.codesig_cmd_offset) |offset| {
2622 subsections[count] = .{
2623 .start = subsections[count - 1].end + @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command),
2624 .end = offset,
2625 };
2626 count += 1;
2627 }
2628
2629 if (!self.options.strip) {
2630 // Exclude region comprising all symbol stabs.
2631 const nlocals = self.dysymtab_cmd.nlocalsym;
2632
2633 const locals = try self.gpa.alloc(macho.nlist_64, nlocals);
2634 defer self.gpa.free(locals);
2635
2636 const locals_buf = @ptrCast([*]u8, locals.ptr)[0 .. @sizeOf(macho.nlist_64) * nlocals];
2637 const amt = try self.file.preadAll(locals_buf, self.symtab_cmd.symoff);
2638 if (amt != locals_buf.len) return error.InputOutput;
2639
2640 const istab: usize = for (locals, 0..) |local, i| {
2641 if (local.stab()) break i;
2642 } else locals.len;
2643 const nstabs = locals.len - istab;
2644
2645 if (nstabs == 0) {
2646 subsections[count] = .{
2647 .start = subsections[count - 1].end + if (args.codesig_cmd_offset == null)
2648 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2649 else
2650 @sizeOf(macho.linkedit_data_command),
2651 .end = max_file_end,
2652 };
2653 count += 1;
2654 } else {
2655 // Exclude a subsection of the strtab with names of the stabs.
2656 // We do not care about anything succeeding strtab as it is the code signature data which is
2657 // not part of the UUID calculation anyway.
2658 const stab_stroff = locals[istab].n_strx;
2659
2660 subsections[count] = .{
2661 .start = subsections[count - 1].end + if (args.codesig_cmd_offset == null)
2662 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2663 else
2664 @sizeOf(macho.linkedit_data_command),
2665 .end = @intCast(u32, self.symtab_cmd.symoff + istab * @sizeOf(macho.nlist_64)),
2666 };
2667 count += 1;
2668
2669 subsections[count] = .{
2670 .start = subsections[count - 1].end + @intCast(u32, nstabs * @sizeOf(macho.nlist_64)),
2671 .end = self.symtab_cmd.stroff + stab_stroff,
2672 };
2673 count += 1;
2674 }
2675 } else {
2676 subsections[count] = .{
2677 .start = subsections[count - 1].end + if (args.codesig_cmd_offset == null)
2678 @as(u32, @sizeOf(macho.symtab_command) + @sizeOf(macho.dysymtab_command))
2679 else
2680 @sizeOf(macho.linkedit_data_command),
2681 .end = max_file_end,
2682 };
2683 count += 1;
2684 }
2685
2686 const chunk_size = 0x4000;
2687
2688 var hasher = Md5.init(.{});
2689 var buffer: [chunk_size]u8 = undefined;
2690
2691 for (subsections[0..count]) |cut| {
2692 const size = cut.end - cut.start;
2693 const num_chunks = mem.alignForward(usize, size, chunk_size) / chunk_size;
2694
2695 var i: usize = 0;
2696 while (i < num_chunks) : (i += 1) {
2697 const fstart = cut.start + i * chunk_size;
2698 const fsize = if (fstart + chunk_size > cut.end)
2699 cut.end - fstart
2700 else
2701 chunk_size;
2702 const amt = try self.file.preadAll(buffer[0..fsize], fstart);
2703 if (amt != fsize) return error.InputOutput;
2704
2705 hasher.update(buffer[0..fsize]);
2706 }
2707 }
2708
2709 hasher.final(&self.uuid_cmd.uuid);
2710 conformUuid(&self.uuid_cmd.uuid);
2711 },
2712 }
2713
2714 const in_file = args.uuid_cmd_offset + @sizeOf(macho.load_command);
2715 try self.file.pwriteAll(&self.uuid_cmd.uuid, in_file);
2716 }
2717
2718 inline fn conformUuid(out: *[Md5.digest_length]u8) void {
2719 // LC_UUID uuids should conform to RFC 4122 UUID version 4 & UUID version 5 formats
2720 out[6] = (out[6] & 0x0F) | (3 << 4);
2721 out[8] = (out[8] & 0x3F) | 0x80;
2722 }2585 }
27232586
2724 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {2587 fn writeCodeSignaturePadding(self: *Zld, code_sig: *CodeSignature) !void {
...@@ -4041,16 +3904,11 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr...@@ -4041,16 +3904,11 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
4041 const lc_writer = lc_buffer.writer();3904 const lc_writer = lc_buffer.writer();
40423905
4043 try zld.writeSegmentHeaders(lc_writer);3906 try zld.writeSegmentHeaders(lc_writer);
4044 const linkedit_cmd_offset = @sizeOf(macho.mach_header_64) + @intCast(u32, lc_buffer.items.len - @sizeOf(macho.segment_command_64));
4045
4046 try lc_writer.writeStruct(zld.dyld_info_cmd);3907 try lc_writer.writeStruct(zld.dyld_info_cmd);
4047 try lc_writer.writeStruct(zld.function_starts_cmd);3908 try lc_writer.writeStruct(zld.function_starts_cmd);
4048 try lc_writer.writeStruct(zld.data_in_code_cmd);3909 try lc_writer.writeStruct(zld.data_in_code_cmd);
4049
4050 const symtab_cmd_offset = @sizeOf(macho.mach_header_64) + @intCast(u32, lc_buffer.items.len);
4051 try lc_writer.writeStruct(zld.symtab_cmd);3910 try lc_writer.writeStruct(zld.symtab_cmd);
4052 try lc_writer.writeStruct(zld.dysymtab_cmd);3911 try lc_writer.writeStruct(zld.dysymtab_cmd);
4053
4054 try load_commands.writeDylinkerLC(lc_writer);3912 try load_commands.writeDylinkerLC(lc_writer);
40553913
4056 if (zld.options.output_mode == .Exe) {3914 if (zld.options.output_mode == .Exe) {
...@@ -4088,22 +3946,14 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr...@@ -4088,22 +3946,14 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
40883946
4089 try load_commands.writeLoadDylibLCs(zld.dylibs.items, zld.referenced_dylibs.keys(), lc_writer);3947 try load_commands.writeLoadDylibLCs(zld.dylibs.items, zld.referenced_dylibs.keys(), lc_writer);
40903948
4091 var codesig_cmd_offset: ?u32 = null;
4092 if (requires_codesig) {3949 if (requires_codesig) {
4093 codesig_cmd_offset = @sizeOf(macho.mach_header_64) + @intCast(u32, lc_buffer.items.len);
4094 try lc_writer.writeStruct(zld.codesig_cmd);3950 try lc_writer.writeStruct(zld.codesig_cmd);
4095 }3951 }
40963952
4097 const ncmds = load_commands.calcNumOfLCs(lc_buffer.items);3953 const ncmds = load_commands.calcNumOfLCs(lc_buffer.items);
4098 try zld.file.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64));3954 try zld.file.pwriteAll(lc_buffer.items, @sizeOf(macho.mach_header_64));
4099 try zld.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len));3955 try zld.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len));
41003956 try zld.writeUuid(comp, uuid_cmd_offset);
4101 try zld.writeUuid(comp, .{
4102 .linkedit_cmd_offset = linkedit_cmd_offset,
4103 .symtab_cmd_offset = symtab_cmd_offset,
4104 .uuid_cmd_offset = uuid_cmd_offset,
4105 .codesig_cmd_offset = codesig_cmd_offset,
4106 });
41073957
4108 if (codesig) |*csig| {3958 if (codesig) |*csig| {
4109 try zld.writeCodeSignature(comp, csig); // code signing always comes last3959 try zld.writeCodeSignature(comp, csig); // code signing always comes last