authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-08 15:23:18-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-12-23 22:15:08-08:00
log6f46570958af8ae27308eb4a9470e05f33aaa522
tree19aec2fa52364c78ffa9a9d8dc14d335f664fe06
parent181ac08459f8d4001c504330ee66037135e56908

link.MachO: update parallel hasher to std.Io

8 files changed, 100 insertions(+), 81 deletions(-)

lib/std/Build/Cache.zig+29-37
...@@ -800,7 +800,7 @@ pub const Manifest = struct {...@@ -800,7 +800,7 @@ pub const Manifest = struct {
800 }800 }
801801
802 var actual_digest: BinDigest = undefined;802 var actual_digest: BinDigest = undefined;
803 hashFile(this_file, &actual_digest) catch |err| {803 hashFile(io, this_file, &actual_digest) catch |err| {
804 self.diagnostic = .{ .file_read = .{804 self.diagnostic = .{ .file_read = .{
805 .file_index = idx,805 .file_index = idx,
806 .err = err,806 .err = err,
...@@ -908,9 +908,11 @@ pub const Manifest = struct {...@@ -908,9 +908,11 @@ pub const Manifest = struct {
908 }908 }
909 }909 }
910910
911 fn populateFileHashHandle(self: *Manifest, ch_file: *File, handle: Io.File) !void {911 fn populateFileHashHandle(self: *Manifest, ch_file: *File, io_file: Io.File) !void {
912 const io = self.cache.io;912 const io = self.cache.io;
913 const actual_stat = try handle.stat(io);913 const gpa = self.cache.gpa;
914
915 const actual_stat = try io_file.stat(io);
914 ch_file.stat = .{916 ch_file.stat = .{
915 .size = actual_stat.size,917 .size = actual_stat.size,
916 .mtime = actual_stat.mtime,918 .mtime = actual_stat.mtime,
...@@ -924,19 +926,17 @@ pub const Manifest = struct {...@@ -924,19 +926,17 @@ pub const Manifest = struct {
924 }926 }
925927
926 if (ch_file.max_file_size) |max_file_size| {928 if (ch_file.max_file_size) |max_file_size| {
927 if (ch_file.stat.size > max_file_size) {929 if (ch_file.stat.size > max_file_size) return error.FileTooBig;
928 return error.FileTooBig;
929 }
930930
931 const contents = try self.cache.gpa.alloc(u8, @as(usize, @intCast(ch_file.stat.size)));931 // Hash while reading from disk, to keep the contents in the cpu
932 errdefer self.cache.gpa.free(contents);932 // cache while doing hashing.
933 const contents = try gpa.alloc(u8, @intCast(ch_file.stat.size));
934 errdefer gpa.free(contents);
933935
934 // Hash while reading from disk, to keep the contents in the cpu cache while
935 // doing hashing.
936 var hasher = hasher_init;936 var hasher = hasher_init;
937 var off: usize = 0;937 var off: usize = 0;
938 while (true) {938 while (true) {
939 const bytes_read = try handle.pread(contents[off..], off);939 const bytes_read = try io_file.readPositional(io, &.{contents[off..]}, off);
940 if (bytes_read == 0) break;940 if (bytes_read == 0) break;
941 hasher.update(contents[off..][0..bytes_read]);941 hasher.update(contents[off..][0..bytes_read]);
942 off += bytes_read;942 off += bytes_read;
...@@ -945,7 +945,7 @@ pub const Manifest = struct {...@@ -945,7 +945,7 @@ pub const Manifest = struct {
945945
946 ch_file.contents = contents;946 ch_file.contents = contents;
947 } else {947 } else {
948 try hashFile(handle, &ch_file.bin_digest);948 try hashFile(io, io_file, &ch_file.bin_digest);
949 }949 }
950950
951 self.hash.hasher.update(&ch_file.bin_digest);951 self.hash.hasher.update(&ch_file.bin_digest);
...@@ -1169,13 +1169,11 @@ pub const Manifest = struct {...@@ -1169,13 +1169,11 @@ pub const Manifest = struct {
11691169
1170 fn downgradeToSharedLock(self: *Manifest) !void {1170 fn downgradeToSharedLock(self: *Manifest) !void {
1171 if (!self.have_exclusive_lock) return;1171 if (!self.have_exclusive_lock) return;
1172 const io = self.cache.io;
11721173
1173 // WASI does not currently support flock, so we bypass it here.1174 if (std.process.can_spawn or !builtin.single_threaded) {
1174 // TODO: If/when flock is supported on WASI, this check should be removed.
1175 // See https://github.com/WebAssembly/wasi-filesystem/issues/2
1176 if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
1177 const manifest_file = self.manifest_file.?;1175 const manifest_file = self.manifest_file.?;
1178 try manifest_file.downgradeLock();1176 try manifest_file.downgradeLock(io);
1179 }1177 }
11801178
1181 self.have_exclusive_lock = false;1179 self.have_exclusive_lock = false;
...@@ -1184,16 +1182,14 @@ pub const Manifest = struct {...@@ -1184,16 +1182,14 @@ pub const Manifest = struct {
1184 fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool {1182 fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool {
1185 if (self.have_exclusive_lock) return false;1183 if (self.have_exclusive_lock) return false;
1186 assert(self.manifest_file != null);1184 assert(self.manifest_file != null);
1185 const io = self.cache.io;
11871186
1188 // WASI does not currently support flock, so we bypass it here.1187 if (std.process.can_spawn or !builtin.single_threaded) {
1189 // TODO: If/when flock is supported on WASI, this check should be removed.
1190 // See https://github.com/WebAssembly/wasi-filesystem/issues/2
1191 if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
1192 const manifest_file = self.manifest_file.?;1188 const manifest_file = self.manifest_file.?;
1193 // Here we intentionally have a period where the lock is released, in case there are1189 // Here we intentionally have a period where the lock is released, in case there are
1194 // other processes holding a shared lock.1190 // other processes holding a shared lock.
1195 manifest_file.unlock();1191 manifest_file.unlock(io);
1196 manifest_file.lock(.exclusive) catch |err| {1192 manifest_file.lock(io, .exclusive) catch |err| {
1197 self.diagnostic = .{ .manifest_lock = err };1193 self.diagnostic = .{ .manifest_lock = err };
1198 return error.CacheCheckFailed;1194 return error.CacheCheckFailed;
1199 };1195 };
...@@ -1206,12 +1202,8 @@ pub const Manifest = struct {...@@ -1206,12 +1202,8 @@ pub const Manifest = struct {
1206 /// The `Manifest` remains safe to deinit.1202 /// The `Manifest` remains safe to deinit.
1207 /// Don't forget to call `writeManifest` before this!1203 /// Don't forget to call `writeManifest` before this!
1208 pub fn toOwnedLock(self: *Manifest) Lock {1204 pub fn toOwnedLock(self: *Manifest) Lock {
1209 const lock: Lock = .{1205 defer self.manifest_file = null;
1210 .manifest_file = self.manifest_file.?,1206 return .{ .manifest_file = self.manifest_file.? };
1211 };
1212
1213 self.manifest_file = null;
1214 return lock;
1215 }1207 }
12161208
1217 /// Releases the manifest file and frees any memory the Manifest was using.1209 /// Releases the manifest file and frees any memory the Manifest was using.
...@@ -1223,7 +1215,7 @@ pub const Manifest = struct {...@@ -1223,7 +1215,7 @@ pub const Manifest = struct {
1223 if (self.manifest_file) |file| {1215 if (self.manifest_file) |file| {
1224 if (builtin.os.tag == .windows) {1216 if (builtin.os.tag == .windows) {
1225 // See Lock.release for why this is required on Windows1217 // See Lock.release for why this is required on Windows
1226 file.unlock();1218 file.unlock(io);
1227 }1219 }
12281220
1229 file.close(io);1221 file.close(io);
...@@ -1308,15 +1300,15 @@ pub fn writeSmallFile(dir: Io.Dir, sub_path: []const u8, data: []const u8) !void...@@ -1308,15 +1300,15 @@ pub fn writeSmallFile(dir: Io.Dir, sub_path: []const u8, data: []const u8) !void
1308 }1300 }
1309}1301}
13101302
1311fn hashFile(file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.PReadError!void {1303fn hashFile(io: Io, file: Io.File, bin_digest: *[Hasher.mac_length]u8) Io.File.ReadPositionalError!void {
1312 var buf: [1024]u8 = undefined;1304 var buffer: [2048]u8 = undefined;
1313 var hasher = hasher_init;1305 var hasher = hasher_init;
1314 var off: u64 = 0;1306 var offset: u64 = 0;
1315 while (true) {1307 while (true) {
1316 const bytes_read = try file.pread(&buf, off);1308 const n = try file.readPositional(io, &.{&buffer}, offset);
1317 if (bytes_read == 0) break;1309 if (n == 0) break;
1318 hasher.update(buf[0..bytes_read]);1310 hasher.update(buffer[0..n]);
1319 off += bytes_read;1311 offset += n;
1320 }1312 }
1321 hasher.final(bin_digest);1313 hasher.final(bin_digest);
1322}1314}
lib/std/Build/WebServer.zig+3-3
...@@ -218,9 +218,9 @@ pub fn finishBuild(ws: *WebServer, opts: struct {...@@ -218,9 +218,9 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
218 else => {},218 else => {},
219 }219 }
220 if (@bitSizeOf(usize) != 64) {220 if (@bitSizeOf(usize) != 64) {
221 // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,221 // Current implementation depends on posix.mmap()'s second
222 // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case222 // parameter, `length: usize`, being compatible with file system's
223 // on 32-bit platforms.223 // u64 return value. This is not the case on 32-bit platforms.
224 // Affects or affected by issues #5185, #22523, and #22464.224 // Affects or affected by issues #5185, #22523, and #22464.
225 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});225 std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
226 }226 }
lib/std/Io.zig+2-2
...@@ -692,9 +692,9 @@ pub const VTable = struct {...@@ -692,9 +692,9 @@ pub const VTable = struct {
692 fileWriteFileStreaming: *const fn (?*anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit) File.Writer.WriteFileError!usize,692 fileWriteFileStreaming: *const fn (?*anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit) File.Writer.WriteFileError!usize,
693 fileWriteFilePositional: *const fn (?*anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit, offset: u64) File.WriteFilePositionalError!usize,693 fileWriteFilePositional: *const fn (?*anyopaque, File, header: []const u8, *Io.File.Reader, Io.Limit, offset: u64) File.WriteFilePositionalError!usize,
694 /// Returns 0 on end of stream.694 /// Returns 0 on end of stream.
695 fileReadStreaming: *const fn (?*anyopaque, File, data: [][]u8) File.Reader.Error!usize,695 fileReadStreaming: *const fn (?*anyopaque, File, data: []const []u8) File.Reader.Error!usize,
696 /// Returns 0 on end of stream.696 /// Returns 0 on end of stream.
697 fileReadPositional: *const fn (?*anyopaque, File, data: [][]u8, offset: u64) File.ReadPositionalError!usize,697 fileReadPositional: *const fn (?*anyopaque, File, data: []const []u8, offset: u64) File.ReadPositionalError!usize,
698 fileSeekBy: *const fn (?*anyopaque, File, relative_offset: i64) File.SeekError!void,698 fileSeekBy: *const fn (?*anyopaque, File, relative_offset: i64) File.SeekError!void,
699 fileSeekTo: *const fn (?*anyopaque, File, absolute_offset: u64) File.SeekError!void,699 fileSeekTo: *const fn (?*anyopaque, File, absolute_offset: u64) File.SeekError!void,
700 fileSync: *const fn (?*anyopaque, File) File.SyncError!void,700 fileSync: *const fn (?*anyopaque, File) File.SyncError!void,
lib/std/Io/File.zig+32-2
...@@ -466,13 +466,21 @@ pub fn setTimestampsNow(file: File, io: Io) SetTimestampsError!void {...@@ -466,13 +466,21 @@ pub fn setTimestampsNow(file: File, io: Io) SetTimestampsError!void {
466466
467pub const ReadPositionalError = Reader.Error || error{Unseekable};467pub const ReadPositionalError = Reader.Error || error{Unseekable};
468468
469pub fn readPositional(file: File, io: Io, buffer: [][]u8, offset: u64) ReadPositionalError!usize {469/// Returns 0 on end of stream.
470///
471/// See also:
472/// * `reader`
473pub fn readPositional(file: File, io: Io, buffer: []const []u8, offset: u64) ReadPositionalError!usize {
470 return io.vtable.fileReadPositional(io.userdata, file, buffer, offset);474 return io.vtable.fileReadPositional(io.userdata, file, buffer, offset);
471}475}
472476
473pub const WritePositionalError = Writer.Error || error{Unseekable};477pub const WritePositionalError = Writer.Error || error{Unseekable};
474478
475pub fn writePositional(file: File, io: Io, buffer: [][]const u8, offset: u64) WritePositionalError!usize {479/// Returns 0 on end of stream.
480///
481/// See also:
482/// * `writer`
483pub fn writePositional(file: File, io: Io, buffer: []const []const u8, offset: u64) WritePositionalError!usize {
476 return io.vtable.fileWritePositional(io.userdata, file, buffer, offset);484 return io.vtable.fileWritePositional(io.userdata, file, buffer, offset);
477}485}
478486
...@@ -501,13 +509,35 @@ pub const WriteFilePositionalError = Writer.WriteFileError || error{Unseekable};...@@ -501,13 +509,35 @@ pub const WriteFilePositionalError = Writer.WriteFileError || error{Unseekable};
501///509///
502/// Positional is more threadsafe, since the global seek position is not510/// Positional is more threadsafe, since the global seek position is not
503/// affected.511/// affected.
512///
513/// See also:
514/// * `readerStreaming`
504pub fn reader(file: File, io: Io, buffer: []u8) Reader {515pub fn reader(file: File, io: Io, buffer: []u8) Reader {
505 return .init(file, io, buffer);516 return .init(file, io, buffer);
506}517}
507518
519/// Equivalent to creating a positional reader and reading multiple times to fill `buffer`.
520///
521/// Returns number of bytes read into `buffer`. If less than `buffer.len`, end of file occurred.
522///
523/// See also:
524/// * `reader`
525pub fn readPositionalAll(file: File, io: Io, buffer: []u8, offset: u64) ReadPositionalError!usize {
526 var index: usize = 0;
527 while (index != buffer.len) {
528 const amt = try file.readPositional(io, &.{buffer[index..]}, offset + index);
529 if (amt == 0) break;
530 index += amt;
531 }
532 return index;
533}
534
508/// Positional is more threadsafe, since the global seek position is not535/// Positional is more threadsafe, since the global seek position is not
509/// affected, but when such syscalls are not available, preemptively536/// affected, but when such syscalls are not available, preemptively
510/// initializing in streaming mode skips a failed syscall.537/// initializing in streaming mode skips a failed syscall.
538///
539/// See also:
540/// * `reader`
511pub fn readerStreaming(file: File, io: Io, buffer: []u8) Reader {541pub fn readerStreaming(file: File, io: Io, buffer: []u8) Reader {
512 return .initStreaming(file, io, buffer);542 return .initStreaming(file, io, buffer);
513}543}
lib/std/fs/test.zig+7-7
...@@ -1455,7 +1455,7 @@ test "writev, readv" {...@@ -1455,7 +1455,7 @@ test "writev, readv" {
14551455
1456 try writer.interface.writeVecAll(&write_vecs);1456 try writer.interface.writeVecAll(&write_vecs);
1457 try writer.interface.flush();1457 try writer.interface.flush();
1458 try testing.expectEqual(@as(u64, line1.len + line2.len), try src_file.getEndPos());1458 try testing.expectEqual(@as(u64, line1.len + line2.len), try src_file.length(io));
14591459
1460 var reader = writer.moveToReader(io);1460 var reader = writer.moveToReader(io);
1461 try reader.seekTo(0);1461 try reader.seekTo(0);
...@@ -1486,7 +1486,7 @@ test "pwritev, preadv" {...@@ -1486,7 +1486,7 @@ test "pwritev, preadv" {
1486 try writer.seekTo(16);1486 try writer.seekTo(16);
1487 try writer.interface.writeVecAll(&lines);1487 try writer.interface.writeVecAll(&lines);
1488 try writer.interface.flush();1488 try writer.interface.flush();
1489 try testing.expectEqual(@as(u64, 16 + line1.len + line2.len), try src_file.getEndPos());1489 try testing.expectEqual(@as(u64, 16 + line1.len + line2.len), try src_file.length(io));
14901490
1491 var reader = writer.moveToReader(io);1491 var reader = writer.moveToReader(io);
1492 try reader.seekTo(16);1492 try reader.seekTo(16);
...@@ -1511,13 +1511,13 @@ test "setEndPos" {...@@ -1511,13 +1511,13 @@ test "setEndPos" {
1511 const f = try tmp.dir.openFile(io, file_name, .{ .mode = .read_write });1511 const f = try tmp.dir.openFile(io, file_name, .{ .mode = .read_write });
1512 defer f.close(io);1512 defer f.close(io);
15131513
1514 const initial_size = try f.getEndPos();1514 const initial_size = try f.length(io);
1515 var buffer: [32]u8 = undefined;1515 var buffer: [32]u8 = undefined;
1516 var reader = f.reader(io, &.{});1516 var reader = f.reader(io, &.{});
15171517
1518 {1518 {
1519 try f.setEndPos(initial_size);1519 try f.setEndPos(initial_size);
1520 try testing.expectEqual(initial_size, try f.getEndPos());1520 try testing.expectEqual(initial_size, try f.length(io));
1521 try reader.seekTo(0);1521 try reader.seekTo(0);
1522 try testing.expectEqual(initial_size, try reader.interface.readSliceShort(&buffer));1522 try testing.expectEqual(initial_size, try reader.interface.readSliceShort(&buffer));
1523 try testing.expectEqualStrings("ninebytes", buffer[0..@intCast(initial_size)]);1523 try testing.expectEqualStrings("ninebytes", buffer[0..@intCast(initial_size)]);
...@@ -1526,7 +1526,7 @@ test "setEndPos" {...@@ -1526,7 +1526,7 @@ test "setEndPos" {
1526 {1526 {
1527 const larger = initial_size + 4;1527 const larger = initial_size + 4;
1528 try f.setEndPos(larger);1528 try f.setEndPos(larger);
1529 try testing.expectEqual(larger, try f.getEndPos());1529 try testing.expectEqual(larger, try f.length(io));
1530 try reader.seekTo(0);1530 try reader.seekTo(0);
1531 try testing.expectEqual(larger, try reader.interface.readSliceShort(&buffer));1531 try testing.expectEqual(larger, try reader.interface.readSliceShort(&buffer));
1532 try testing.expectEqualStrings("ninebytes\x00\x00\x00\x00", buffer[0..@intCast(larger)]);1532 try testing.expectEqualStrings("ninebytes\x00\x00\x00\x00", buffer[0..@intCast(larger)]);
...@@ -1535,14 +1535,14 @@ test "setEndPos" {...@@ -1535,14 +1535,14 @@ test "setEndPos" {
1535 {1535 {
1536 const smaller = initial_size - 5;1536 const smaller = initial_size - 5;
1537 try f.setEndPos(smaller);1537 try f.setEndPos(smaller);
1538 try testing.expectEqual(smaller, try f.getEndPos());1538 try testing.expectEqual(smaller, try f.length(io));
1539 try reader.seekTo(0);1539 try reader.seekTo(0);
1540 try testing.expectEqual(smaller, try reader.interface.readSliceShort(&buffer));1540 try testing.expectEqual(smaller, try reader.interface.readSliceShort(&buffer));
1541 try testing.expectEqualStrings("nine", buffer[0..@intCast(smaller)]);1541 try testing.expectEqualStrings("nine", buffer[0..@intCast(smaller)]);
1542 }1542 }
15431543
1544 try f.setEndPos(0);1544 try f.setEndPos(0);
1545 try testing.expectEqual(0, try f.getEndPos());1545 try testing.expectEqual(0, try f.length(io));
1546 try reader.seekTo(0);1546 try reader.seekTo(0);
1547 try testing.expectEqual(0, try reader.interface.readSliceShort(&buffer));1547 try testing.expectEqual(0, try reader.interface.readSliceShort(&buffer));
1548}1548}
src/link/MachO/CodeSignature.zig+9-8
...@@ -12,7 +12,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256;...@@ -12,7 +12,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256;
12const Allocator = std.mem.Allocator;12const Allocator = std.mem.Allocator;
1313
14const trace = @import("../../tracy.zig").trace;14const trace = @import("../../tracy.zig").trace;
15const Hasher = @import("hasher.zig").ParallelHasher;15const ParallelHasher = @import("hasher.zig").ParallelHasher;
16const MachO = @import("../MachO.zig");16const MachO = @import("../MachO.zig");
1717
18const hash_size = Sha256.digest_length;18const hash_size = Sha256.digest_length;
...@@ -268,7 +268,9 @@ pub fn writeAdhocSignature(...@@ -268,7 +268,9 @@ pub fn writeAdhocSignature(
268 const tracy = trace(@src());268 const tracy = trace(@src());
269 defer tracy.end();269 defer tracy.end();
270270
271 const allocator = macho_file.base.comp.gpa;271 const comp = macho_file.base.comp;
272 const gpa = comp.gpa;
273 const io = comp.io;
272274
273 var header: macho.SuperBlob = .{275 var header: macho.SuperBlob = .{
274 .magic = macho.CSMAGIC_EMBEDDED_SIGNATURE,276 .magic = macho.CSMAGIC_EMBEDDED_SIGNATURE,
...@@ -276,7 +278,7 @@ pub fn writeAdhocSignature(...@@ -276,7 +278,7 @@ pub fn writeAdhocSignature(
276 .count = 0,278 .count = 0,
277 };279 };
278280
279 var blobs = std.array_list.Managed(Blob).init(allocator);281 var blobs = std.array_list.Managed(Blob).init(gpa);
280 defer blobs.deinit();282 defer blobs.deinit();
281283
282 self.code_directory.inner.execSegBase = opts.exec_seg_base;284 self.code_directory.inner.execSegBase = opts.exec_seg_base;
...@@ -286,13 +288,12 @@ pub fn writeAdhocSignature(...@@ -286,13 +288,12 @@ pub fn writeAdhocSignature(
286288
287 const total_pages = @as(u32, @intCast(mem.alignForward(usize, opts.file_size, self.page_size) / self.page_size));289 const total_pages = @as(u32, @intCast(mem.alignForward(usize, opts.file_size, self.page_size) / self.page_size));
288290
289 try self.code_directory.code_slots.ensureTotalCapacityPrecise(allocator, total_pages);291 try self.code_directory.code_slots.ensureTotalCapacityPrecise(gpa, total_pages);
290 self.code_directory.code_slots.items.len = total_pages;292 self.code_directory.code_slots.items.len = total_pages;
291 self.code_directory.inner.nCodeSlots = total_pages;293 self.code_directory.inner.nCodeSlots = total_pages;
292294
293 // Calculate hash for each page (in file) and write it to the buffer295 // Calculate hash for each page (in file) and write it to the buffer
294 var hasher = Hasher(Sha256){ .allocator = allocator, .io = macho_file.base.comp.io };296 try ParallelHasher(Sha256).hash(gpa, io, opts.file, self.code_directory.code_slots.items, .{
295 try hasher.hash(opts.file, self.code_directory.code_slots.items, .{
296 .chunk_size = self.page_size,297 .chunk_size = self.page_size,
297 .max_file_size = opts.file_size,298 .max_file_size = opts.file_size,
298 });299 });
...@@ -304,7 +305,7 @@ pub fn writeAdhocSignature(...@@ -304,7 +305,7 @@ pub fn writeAdhocSignature(
304 var hash: [hash_size]u8 = undefined;305 var hash: [hash_size]u8 = undefined;
305306
306 if (self.requirements) |*req| {307 if (self.requirements) |*req| {
307 var a: std.Io.Writer.Allocating = .init(allocator);308 var a: std.Io.Writer.Allocating = .init(gpa);
308 defer a.deinit();309 defer a.deinit();
309 try req.write(&a.writer);310 try req.write(&a.writer);
310 Sha256.hash(a.written(), &hash, .{});311 Sha256.hash(a.written(), &hash, .{});
...@@ -316,7 +317,7 @@ pub fn writeAdhocSignature(...@@ -316,7 +317,7 @@ pub fn writeAdhocSignature(
316 }317 }
317318
318 if (self.entitlements) |*ents| {319 if (self.entitlements) |*ents| {
319 var a: std.Io.Writer.Allocating = .init(allocator);320 var a: std.Io.Writer.Allocating = .init(gpa);
320 defer a.deinit();321 defer a.deinit();
321 try ents.write(&a.writer);322 try ents.write(&a.writer);
322 Sha256.hash(a.written(), &hash, .{});323 Sha256.hash(a.written(), &hash, .{});
src/link/MachO/hasher.zig+9-15
...@@ -1,5 +1,6 @@...@@ -1,5 +1,6 @@
1const std = @import("std");1const std = @import("std");
2const Io = std.Io;2const Io = std.Io;
3const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;4const Allocator = std.mem.Allocator;
45
5const trace = @import("../../tracy.zig").trace;6const trace = @import("../../tracy.zig").trace;
...@@ -8,20 +9,15 @@ pub fn ParallelHasher(comptime Hasher: type) type {...@@ -8,20 +9,15 @@ pub fn ParallelHasher(comptime Hasher: type) type {
8 const hash_size = Hasher.digest_length;9 const hash_size = Hasher.digest_length;
910
10 return struct {11 return struct {
11 allocator: Allocator,12 pub fn hash(self: Self, io: Io, file: Io.File, out: [][hash_size]u8, opts: struct {
12 io: std.Io,
13
14 pub fn hash(self: Self, file: Io.File, out: [][hash_size]u8, opts: struct {
15 chunk_size: u64 = 0x4000,13 chunk_size: u64 = 0x4000,
16 max_file_size: ?u64 = null,14 max_file_size: ?u64 = null,
17 }) !void {15 }) !void {
18 const tracy = trace(@src());16 const tracy = trace(@src());
19 defer tracy.end();17 defer tracy.end();
2018
21 const io = self.io;
22
23 const file_size = blk: {19 const file_size = blk: {
24 const file_size = opts.max_file_size orelse try file.getEndPos();20 const file_size = opts.max_file_size orelse try file.length(io);
25 break :blk std.math.cast(usize, file_size) orelse return error.Overflow;21 break :blk std.math.cast(usize, file_size) orelse return error.Overflow;
26 };22 };
27 const chunk_size = std.math.cast(usize, opts.chunk_size) orelse return error.Overflow;23 const chunk_size = std.math.cast(usize, opts.chunk_size) orelse return error.Overflow;
...@@ -29,12 +25,12 @@ pub fn ParallelHasher(comptime Hasher: type) type {...@@ -29,12 +25,12 @@ pub fn ParallelHasher(comptime Hasher: type) type {
29 const buffer = try self.allocator.alloc(u8, chunk_size * out.len);25 const buffer = try self.allocator.alloc(u8, chunk_size * out.len);
30 defer self.allocator.free(buffer);26 defer self.allocator.free(buffer);
3127
32 const results = try self.allocator.alloc(Io.File.PReadError!usize, out.len);28 const results = try self.allocator.alloc(Io.File.ReadPositionalError!usize, out.len);
33 defer self.allocator.free(results);29 defer self.allocator.free(results);
3430
35 {31 {
36 var group: std.Io.Group = .init;32 var group: Io.Group = .init;
37 errdefer group.cancel(io);33 defer group.cancel(io);
3834
39 for (out, results, 0..) |*out_buf, *result, i| {35 for (out, results, 0..) |*out_buf, *result, i| {
40 const fstart = i * chunk_size;36 const fstart = i * chunk_size;
...@@ -42,7 +38,7 @@ pub fn ParallelHasher(comptime Hasher: type) type {...@@ -42,7 +38,7 @@ pub fn ParallelHasher(comptime Hasher: type) type {
42 file_size - fstart38 file_size - fstart
43 else39 else
44 chunk_size;40 chunk_size;
45 group.async(io, worker, .{41 group.async(worker, .{
46 file,42 file,
47 fstart,43 fstart,
48 buffer[fstart..][0..fsize],44 buffer[fstart..][0..fsize],
...@@ -61,11 +57,9 @@ pub fn ParallelHasher(comptime Hasher: type) type {...@@ -61,11 +57,9 @@ pub fn ParallelHasher(comptime Hasher: type) type {
61 fstart: usize,57 fstart: usize,
62 buffer: []u8,58 buffer: []u8,
63 out: *[hash_size]u8,59 out: *[hash_size]u8,
64 err: *Io.File.PReadError!usize,60 err: *Io.File.ReadPositionalError!usize,
65 ) void {61 ) void {
66 const tracy = trace(@src());62 err.* = file.readPositionalAll(buffer, fstart);
67 defer tracy.end();
68 err.* = file.preadAll(buffer, fstart);
69 Hasher.hash(buffer, out, .{});63 Hasher.hash(buffer, out, .{});
70 }64 }
7165
src/link/MachO/uuid.zig+9-7
...@@ -4,7 +4,7 @@ const Md5 = std.crypto.hash.Md5;...@@ -4,7 +4,7 @@ const Md5 = std.crypto.hash.Md5;
44
5const trace = @import("../../tracy.zig").trace;5const trace = @import("../../tracy.zig").trace;
6const Compilation = @import("../../Compilation.zig");6const Compilation = @import("../../Compilation.zig");
7const Hasher = @import("hasher.zig").ParallelHasher;7const ParallelHasher = @import("hasher.zig").ParallelHasher;
88
9/// Calculates Md5 hash of each chunk in parallel and then hashes all Md5 hashes to produce9/// Calculates Md5 hash of each chunk in parallel and then hashes all Md5 hashes to produce
10/// the final digest.10/// the final digest.
...@@ -16,21 +16,23 @@ pub fn calcUuid(comp: *const Compilation, file: Io.File, file_size: u64, out: *[...@@ -16,21 +16,23 @@ pub fn calcUuid(comp: *const Compilation, file: Io.File, file_size: u64, out: *[
16 const tracy = trace(@src());16 const tracy = trace(@src());
17 defer tracy.end();17 defer tracy.end();
1818
19 const gpa = comp.gpa;
20 const io = comp.io;
21
19 const chunk_size: usize = 1024 * 1024;22 const chunk_size: usize = 1024 * 1024;
20 const num_chunks: usize = std.math.cast(usize, @divTrunc(file_size, chunk_size)) orelse return error.Overflow;23 const num_chunks: usize = std.math.cast(usize, @divTrunc(file_size, chunk_size)) orelse return error.Overflow;
21 const actual_num_chunks = if (@rem(file_size, chunk_size) > 0) num_chunks + 1 else num_chunks;24 const actual_num_chunks = if (@rem(file_size, chunk_size) > 0) num_chunks + 1 else num_chunks;
2225
23 const hashes = try comp.gpa.alloc([Md5.digest_length]u8, actual_num_chunks);26 const hashes = try gpa.alloc([Md5.digest_length]u8, actual_num_chunks);
24 defer comp.gpa.free(hashes);27 defer gpa.free(hashes);
2528
26 var hasher = Hasher(Md5){ .allocator = comp.gpa, .io = comp.io };29 try ParallelHasher(Md5).hash(gpa, io, file, hashes, .{
27 try hasher.hash(file, hashes, .{
28 .chunk_size = chunk_size,30 .chunk_size = chunk_size,
29 .max_file_size = file_size,31 .max_file_size = file_size,
30 });32 });
3133
32 const final_buffer = try comp.gpa.alloc(u8, actual_num_chunks * Md5.digest_length);34 const final_buffer = try gpa.alloc(u8, actual_num_chunks * Md5.digest_length);
33 defer comp.gpa.free(final_buffer);35 defer gpa.free(final_buffer);
3436
35 for (hashes, 0..) |hash, i| {37 for (hashes, 0..) |hash, i| {
36 @memcpy(final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);38 @memcpy(final_buffer[i * Md5.digest_length ..][0..Md5.digest_length], &hash);