authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-10-23 11:35:03-07:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-10-23 11:35:03-07:00
loga3033c7bd9ec26abd2f831cf75eb85f06590deed
tree4b293310f5f669efbc00e14fa6e92656a509a15a
parent0ae60f7236e4da0c234f59f3671f742695d54905
parent2f37278540e4783a236a2c095fc905e613fccf1d
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #13267 from ziglang/macho-threaded-sha256

link.MachO: multi-thread first round of sha256 hashing

3 files changed, 64 insertions(+), 30 deletions(-)

src/link/MachO.zig+3-3
...@@ -625,7 +625,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No...@@ -625,7 +625,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No
625 try self.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len + headers_buf.items.len));625 try self.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len + headers_buf.items.len));
626626
627 if (codesig) |*csig| {627 if (codesig) |*csig| {
628 try self.writeCodeSignature(csig, codesig_offset.?); // code signing always comes last628 try self.writeCodeSignature(comp, csig, codesig_offset.?); // code signing always comes last
629 }629 }
630630
631 if (self.d_sym) |*d_sym| {631 if (self.d_sym) |*d_sym| {
...@@ -4037,13 +4037,13 @@ fn writeCodeSignaturePadding(...@@ -4037,13 +4037,13 @@ fn writeCodeSignaturePadding(
4037 return @intCast(u32, offset);4037 return @intCast(u32, offset);
4038}4038}
40394039
4040fn writeCodeSignature(self: *MachO, code_sig: *CodeSignature, offset: u32) !void {4040fn writeCodeSignature(self: *MachO, comp: *const Compilation, code_sig: *CodeSignature, offset: u32) !void {
4041 const seg = self.getSegment(self.text_section_index.?);4041 const seg = self.getSegment(self.text_section_index.?);
40424042
4043 var buffer = std.ArrayList(u8).init(self.base.allocator);4043 var buffer = std.ArrayList(u8).init(self.base.allocator);
4044 defer buffer.deinit();4044 defer buffer.deinit();
4045 try buffer.ensureTotalCapacityPrecise(code_sig.size());4045 try buffer.ensureTotalCapacityPrecise(code_sig.size());
4046 try code_sig.writeAdhocSignature(self.base.allocator, .{4046 try code_sig.writeAdhocSignature(comp, .{
4047 .file = self.base.file.?,4047 .file = self.base.file.?,
4048 .exec_seg_base = seg.fileoff,4048 .exec_seg_base = seg.fileoff,
4049 .exec_seg_limit = seg.filesize,4049 .exec_seg_limit = seg.filesize,
src/link/MachO/CodeSignature.zig+53-24
...@@ -1,4 +1,6 @@...@@ -1,4 +1,6 @@
1const CodeSignature = @This();1const CodeSignature = @This();
2const Compilation = @import("../../Compilation.zig");
3const WaitGroup = @import("../../WaitGroup.zig");
24
3const std = @import("std");5const std = @import("std");
4const assert = std.debug.assert;6const assert = std.debug.assert;
...@@ -258,17 +260,19 @@ pub const WriteOpts = struct {...@@ -258,17 +260,19 @@ pub const WriteOpts = struct {
258260
259pub fn writeAdhocSignature(261pub fn writeAdhocSignature(
260 self: *CodeSignature,262 self: *CodeSignature,
261 allocator: Allocator,263 comp: *const Compilation,
262 opts: WriteOpts,264 opts: WriteOpts,
263 writer: anytype,265 writer: anytype,
264) !void {266) !void {
267 const gpa = comp.gpa;
268
265 var header: macho.SuperBlob = .{269 var header: macho.SuperBlob = .{
266 .magic = macho.CSMAGIC_EMBEDDED_SIGNATURE,270 .magic = macho.CSMAGIC_EMBEDDED_SIGNATURE,
267 .length = @sizeOf(macho.SuperBlob),271 .length = @sizeOf(macho.SuperBlob),
268 .count = 0,272 .count = 0,
269 };273 };
270274
271 var blobs = std.ArrayList(Blob).init(allocator);275 var blobs = std.ArrayList(Blob).init(gpa);
272 defer blobs.deinit();276 defer blobs.deinit();
273277
274 self.code_directory.inner.execSegBase = opts.exec_seg_base;278 self.code_directory.inner.execSegBase = opts.exec_seg_base;
...@@ -276,37 +280,49 @@ pub fn writeAdhocSignature(...@@ -276,37 +280,49 @@ pub fn writeAdhocSignature(
276 self.code_directory.inner.execSegFlags = if (opts.output_mode == .Exe) macho.CS_EXECSEG_MAIN_BINARY else 0;280 self.code_directory.inner.execSegFlags = if (opts.output_mode == .Exe) macho.CS_EXECSEG_MAIN_BINARY else 0;
277 self.code_directory.inner.codeLimit = opts.file_size;281 self.code_directory.inner.codeLimit = opts.file_size;
278282
279 const total_pages = mem.alignForward(opts.file_size, self.page_size) / self.page_size;283 const total_pages = @intCast(u32, mem.alignForward(opts.file_size, self.page_size) / self.page_size);
280
281 var buffer = try allocator.alloc(u8, self.page_size);
282 defer allocator.free(buffer);
283284
284 try self.code_directory.code_slots.ensureTotalCapacityPrecise(allocator, total_pages);285 try self.code_directory.code_slots.ensureTotalCapacityPrecise(gpa, total_pages);
286 self.code_directory.code_slots.items.len = total_pages;
287 self.code_directory.inner.nCodeSlots = total_pages;
285288
286 // Calculate hash for each page (in file) and write it to the buffer289 // Calculate hash for each page (in file) and write it to the buffer
287 var hash: [hash_size]u8 = undefined;290 var wg: WaitGroup = .{};
288 var i: usize = 0;291 {
289 while (i < total_pages) : (i += 1) {292 const buffer = try gpa.alloc(u8, self.page_size * total_pages);
290 const fstart = i * self.page_size;293 defer gpa.free(buffer);
291 const fsize = if (fstart + self.page_size > opts.file_size)294
292 opts.file_size - fstart295 const results = try gpa.alloc(fs.File.PReadError!usize, total_pages);
293 else296 defer gpa.free(results);
294 self.page_size;297 {
295 const len = try opts.file.preadAll(buffer, fstart);298 wg.reset();
296 assert(fsize <= len);299 defer wg.wait();
297300
298 Sha256.hash(buffer[0..fsize], &hash, .{});301 var i: usize = 0;
299302 while (i < total_pages) : (i += 1) {
300 self.code_directory.code_slots.appendAssumeCapacity(hash);303 const fstart = i * self.page_size;
301 self.code_directory.inner.nCodeSlots += 1;304 const fsize = if (fstart + self.page_size > opts.file_size)
305 opts.file_size - fstart
306 else
307 self.page_size;
308 const out_hash = &self.code_directory.code_slots.items[i];
309 wg.start();
310 try comp.thread_pool.spawn(workerSha256Hash, .{
311 opts.file, fstart, buffer[fstart..][0..fsize], out_hash, &results[i], &wg,
312 });
313 }
314 }
315 for (results) |result| _ = try result;
302 }316 }
303317
304 try blobs.append(.{ .code_directory = &self.code_directory });318 try blobs.append(.{ .code_directory = &self.code_directory });
305 header.length += @sizeOf(macho.BlobIndex);319 header.length += @sizeOf(macho.BlobIndex);
306 header.count += 1;320 header.count += 1;
307321
322 var hash: [hash_size]u8 = undefined;
323
308 if (self.requirements) |*req| {324 if (self.requirements) |*req| {
309 var buf = std.ArrayList(u8).init(allocator);325 var buf = std.ArrayList(u8).init(gpa);
310 defer buf.deinit();326 defer buf.deinit();
311 try req.write(buf.writer());327 try req.write(buf.writer());
312 Sha256.hash(buf.items, &hash, .{});328 Sha256.hash(buf.items, &hash, .{});
...@@ -318,7 +334,7 @@ pub fn writeAdhocSignature(...@@ -318,7 +334,7 @@ pub fn writeAdhocSignature(
318 }334 }
319335
320 if (self.entitlements) |*ents| {336 if (self.entitlements) |*ents| {
321 var buf = std.ArrayList(u8).init(allocator);337 var buf = std.ArrayList(u8).init(gpa);
322 defer buf.deinit();338 defer buf.deinit();
323 try ents.write(buf.writer());339 try ents.write(buf.writer());
324 Sha256.hash(buf.items, &hash, .{});340 Sha256.hash(buf.items, &hash, .{});
...@@ -356,6 +372,19 @@ pub fn writeAdhocSignature(...@@ -356,6 +372,19 @@ pub fn writeAdhocSignature(
356 }372 }
357}373}
358374
375fn workerSha256Hash(
376 file: fs.File,
377 fstart: usize,
378 buffer: []u8,
379 hash: *[hash_size]u8,
380 err: *fs.File.PReadError!usize,
381 wg: *WaitGroup,
382) void {
383 defer wg.finish();
384 err.* = file.preadAll(buffer, fstart);
385 Sha256.hash(buffer, hash, .{});
386}
387
359pub fn size(self: CodeSignature) u32 {388pub fn size(self: CodeSignature) u32 {
360 var ssize: u32 = @sizeOf(macho.SuperBlob) + @sizeOf(macho.BlobIndex) + self.code_directory.size();389 var ssize: u32 = @sizeOf(macho.SuperBlob) + @sizeOf(macho.BlobIndex) + self.code_directory.size();
361 if (self.requirements) |req| {390 if (self.requirements) |req| {
src/link/MachO/zld.zig+8-3
...@@ -3035,14 +3035,19 @@ pub const Zld = struct {...@@ -3035,14 +3035,19 @@ pub const Zld = struct {
3035 return @intCast(u32, offset);3035 return @intCast(u32, offset);
3036 }3036 }
30373037
3038 fn writeCodeSignature(self: *Zld, code_sig: *CodeSignature, offset: u32) !void {3038 fn writeCodeSignature(
3039 self: *Zld,
3040 comp: *const Compilation,
3041 code_sig: *CodeSignature,
3042 offset: u32,
3043 ) !void {
3039 const seg_id = self.getSegmentByName("__TEXT").?;3044 const seg_id = self.getSegmentByName("__TEXT").?;
3040 const seg = self.segments.items[seg_id];3045 const seg = self.segments.items[seg_id];
30413046
3042 var buffer = std.ArrayList(u8).init(self.gpa);3047 var buffer = std.ArrayList(u8).init(self.gpa);
3043 defer buffer.deinit();3048 defer buffer.deinit();
3044 try buffer.ensureTotalCapacityPrecise(code_sig.size());3049 try buffer.ensureTotalCapacityPrecise(code_sig.size());
3045 try code_sig.writeAdhocSignature(self.gpa, .{3050 try code_sig.writeAdhocSignature(comp, .{
3046 .file = self.file,3051 .file = self.file,
3047 .exec_seg_base = seg.fileoff,3052 .exec_seg_base = seg.fileoff,
3048 .exec_seg_limit = seg.filesize,3053 .exec_seg_limit = seg.filesize,
...@@ -4379,7 +4384,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr...@@ -4379,7 +4384,7 @@ pub fn linkWithZld(macho_file: *MachO, comp: *Compilation, prog_node: *std.Progr
4379 try zld.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len + headers_buf.items.len));4384 try zld.writeHeader(ncmds, @intCast(u32, lc_buffer.items.len + headers_buf.items.len));
43804385
4381 if (codesig) |*csig| {4386 if (codesig) |*csig| {
4382 try zld.writeCodeSignature(csig, codesig_offset.?); // code signing always comes last4387 try zld.writeCodeSignature(comp, csig, codesig_offset.?); // code signing always comes last
4383 }4388 }
4384 }4389 }
43854390