authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-06-19 10:29:14+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-06-19 10:29:39+02:00
logb3a2ab3fedfc2e3e15f9024c7334a1f53d9aa7c5
treec462d0b3d43f3c59a7a8d31164c56305e598f2de
parent423d7b848b1953173df99fde1f83166dc68c2a2c

macho: extract parallel hasher into a generic helper struct

3 files changed, 67 insertions(+), 59 deletions(-)

CMakeLists.txt+1
...@@ -594,6 +594,7 @@ set(ZIG_STAGE2_SOURCES...@@ -594,6 +594,7 @@ set(ZIG_STAGE2_SOURCES
594 "${CMAKE_SOURCE_DIR}/src/link/MachO/dead_strip.zig"594 "${CMAKE_SOURCE_DIR}/src/link/MachO/dead_strip.zig"
595 "${CMAKE_SOURCE_DIR}/src/link/MachO/eh_frame.zig"595 "${CMAKE_SOURCE_DIR}/src/link/MachO/eh_frame.zig"
596 "${CMAKE_SOURCE_DIR}/src/link/MachO/fat.zig"596 "${CMAKE_SOURCE_DIR}/src/link/MachO/fat.zig"
597 "${CMAKE_SOURCE_DIR}/src/link/MachO/hasher.zig"
597 "${CMAKE_SOURCE_DIR}/src/link/MachO/load_commands.zig"598 "${CMAKE_SOURCE_DIR}/src/link/MachO/load_commands.zig"
598 "${CMAKE_SOURCE_DIR}/src/link/MachO/thunks.zig"599 "${CMAKE_SOURCE_DIR}/src/link/MachO/thunks.zig"
599 "${CMAKE_SOURCE_DIR}/src/link/MachO/zld.zig"600 "${CMAKE_SOURCE_DIR}/src/link/MachO/zld.zig"
src/link/MachO/CodeSignature.zig+6-59
...@@ -7,11 +7,10 @@ const log = std.log.scoped(.link);...@@ -7,11 +7,10 @@ const log = std.log.scoped(.link);
7const macho = std.macho;7const macho = std.macho;
8const mem = std.mem;8const mem = std.mem;
9const testing = std.testing;9const testing = std.testing;
10const ThreadPool = std.Thread.Pool;
11const WaitGroup = std.Thread.WaitGroup;
1210
13const Allocator = mem.Allocator;11const Allocator = mem.Allocator;
14const Compilation = @import("../../Compilation.zig");12const Compilation = @import("../../Compilation.zig");
13const Hasher = @import("hasher.zig").ParallelHasher;
15const Sha256 = std.crypto.hash.sha2.Sha256;14const Sha256 = std.crypto.hash.sha2.Sha256;
1615
17const hash_size = Sha256.digest_length;16const hash_size = Sha256.digest_length;
...@@ -289,7 +288,11 @@ pub fn writeAdhocSignature(...@@ -289,7 +288,11 @@ pub fn writeAdhocSignature(
289 self.code_directory.inner.nCodeSlots = total_pages;288 self.code_directory.inner.nCodeSlots = total_pages;
290289
291 // Calculate hash for each page (in file) and write it to the buffer290 // Calculate hash for each page (in file) and write it to the buffer
292 try self.parallelHash(gpa, comp.thread_pool, opts.file, opts.file_size);291 var hasher = Hasher(Sha256){};
292 try hasher.hash(gpa, comp.thread_pool, opts.file, self.code_directory.code_slots.items, .{
293 .chunk_size = self.page_size,
294 .max_file_size = opts.file_size,
295 });
293296
294 try blobs.append(.{ .code_directory = &self.code_directory });297 try blobs.append(.{ .code_directory = &self.code_directory });
295 header.length += @sizeOf(macho.BlobIndex);298 header.length += @sizeOf(macho.BlobIndex);
...@@ -348,62 +351,6 @@ pub fn writeAdhocSignature(...@@ -348,62 +351,6 @@ pub fn writeAdhocSignature(
348 }351 }
349}352}
350353
351fn parallelHash(
352 self: *CodeSignature,
353 gpa: Allocator,
354 pool: *ThreadPool,
355 file: fs.File,
356 file_size: u32,
357) !void {
358 var wg: WaitGroup = .{};
359
360 const total_num_chunks = mem.alignForward(usize, file_size, self.page_size) / self.page_size;
361 assert(self.code_directory.code_slots.items.len >= total_num_chunks);
362
363 const buffer = try gpa.alloc(u8, self.page_size * total_num_chunks);
364 defer gpa.free(buffer);
365
366 const results = try gpa.alloc(fs.File.PReadError!usize, total_num_chunks);
367 defer gpa.free(results);
368
369 {
370 wg.reset();
371 defer wg.wait();
372
373 var i: usize = 0;
374 while (i < total_num_chunks) : (i += 1) {
375 const fstart = i * self.page_size;
376 const fsize = if (fstart + self.page_size > file_size)
377 file_size - fstart
378 else
379 self.page_size;
380 wg.start();
381 try pool.spawn(worker, .{
382 file,
383 fstart,
384 buffer[fstart..][0..fsize],
385 &self.code_directory.code_slots.items[i],
386 &results[i],
387 &wg,
388 });
389 }
390 }
391 for (results) |result| _ = try result;
392}
393
394fn worker(
395 file: fs.File,
396 fstart: usize,
397 buffer: []u8,
398 out: *[hash_size]u8,
399 err: *fs.File.PReadError!usize,
400 wg: *WaitGroup,
401) void {
402 defer wg.finish();
403 err.* = file.preadAll(buffer, fstart);
404 Sha256.hash(buffer, out, .{});
405}
406
407pub fn size(self: CodeSignature) u32 {354pub fn size(self: CodeSignature) u32 {
408 var ssize: u32 = @sizeOf(macho.SuperBlob) + @sizeOf(macho.BlobIndex) + self.code_directory.size();355 var ssize: u32 = @sizeOf(macho.SuperBlob) + @sizeOf(macho.BlobIndex) + self.code_directory.size();
409 if (self.requirements) |req| {356 if (self.requirements) |req| {
src/link/MachO/hasher.zig created+60
...@@ -0,0 +1,60 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const fs = std.fs;
4const mem = std.mem;
5
6const Allocator = mem.Allocator;
7const ThreadPool = std.Thread.Pool;
8const WaitGroup = std.Thread.WaitGroup;
9
10pub fn ParallelHasher(comptime Hasher: type) type {
11 const hash_size = Hasher.digest_length;
12
13 return struct {
14 pub fn hash(self: @This(), gpa: Allocator, pool: *ThreadPool, file: fs.File, out: [][hash_size]u8, opts: struct {
15 chunk_size: u16 = 0x4000,
16 max_file_size: ?u64 = null,
17 }) !void {
18 _ = self;
19
20 var wg: WaitGroup = .{};
21
22 const file_size = opts.max_file_size orelse try file.getEndPos();
23 const total_num_chunks = mem.alignForward(u64, file_size, opts.chunk_size) / opts.chunk_size;
24 assert(out.len >= total_num_chunks);
25
26 const buffer = try gpa.alloc(u8, opts.chunk_size * total_num_chunks);
27 defer gpa.free(buffer);
28
29 const results = try gpa.alloc(fs.File.PReadError!usize, total_num_chunks);
30 defer gpa.free(results);
31
32 {
33 wg.reset();
34 defer wg.wait();
35
36 var i: usize = 0;
37 while (i < total_num_chunks) : (i += 1) {
38 const fstart = i * opts.chunk_size;
39 const fsize = if (fstart + opts.chunk_size > file_size) file_size - fstart else opts.chunk_size;
40 wg.start();
41 try pool.spawn(worker, .{ file, fstart, buffer[fstart..][0..fsize], &out[i], &results[i], &wg });
42 }
43 }
44 for (results) |result| _ = try result;
45 }
46
47 fn worker(
48 file: fs.File,
49 fstart: usize,
50 buffer: []u8,
51 out: *[hash_size]u8,
52 err: *fs.File.PReadError!usize,
53 wg: *WaitGroup,
54 ) void {
55 defer wg.finish();
56 err.* = file.preadAll(buffer, fstart);
57 Hasher.hash(buffer, out, .{});
58 }
59 };
60}