authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-23 15:58:51-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-02-26 11:42:03-08:00
log12355cfb4cb14ba78423fb38838f4485bb563c9b
treebd3f0fbfc489ccbeb87d5913a26639a8e2033d63
parente0129b387ff962c0f89d62c8ab0409145a19f453

Package: new hash format

legacy format is also supported. closes #20178

4 files changed, 269 insertions(+), 145 deletions(-)

src/Package.zig+156
...@@ -1,8 +1,164 @@...@@ -1,8 +1,164 @@
1const std = @import("std");
2const assert = std.debug.assert;
3
1pub const Module = @import("Package/Module.zig");4pub const Module = @import("Package/Module.zig");
2pub const Fetch = @import("Package/Fetch.zig");5pub const Fetch = @import("Package/Fetch.zig");
3pub const build_zig_basename = "build.zig";6pub const build_zig_basename = "build.zig";
4pub const Manifest = @import("Package/Manifest.zig");7pub const Manifest = @import("Package/Manifest.zig");
58
9pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
10pub const multihash_hex_digest_len = 2 * multihash_len;
11pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
12
13/// A user-readable, file system safe hash that identifies an exact package
14/// snapshot, including file contents.
15///
16/// This data structure can be used to store the legacy hash format too. Legacy
17/// hash format is scheduled to be removed after 0.14.0 is tagged.
18pub const Hash = struct {
19 /// Maximum size of a package hash. Unused bytes at the end are
20 /// filled with zeroes.
21 bytes: [max_len]u8,
22
23 pub const Algo = std.crypto.hash.sha2.Sha256;
24 pub const Digest = [Algo.digest_length]u8;
25
26 pub const max_len = 32 + 1 + 32 + 1 + 12;
27
28 pub fn fromSlice(s: []const u8) Hash {
29 assert(s.len <= max_len);
30 var result: Hash = undefined;
31 @memcpy(result.bytes[0..s.len], s);
32 @memset(result.bytes[s.len..], 0);
33 return result;
34 }
35
36 pub fn toSlice(ph: *const Hash) []const u8 {
37 var end: usize = ph.bytes.len;
38 while (true) {
39 end -= 1;
40 if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
41 }
42 }
43
44 pub fn eql(a: *const Hash, b: *const Hash) bool {
45 return std.mem.eql(u8, &a.bytes, &b.bytes);
46 }
47
48 /// Distinguishes whether the legacy multihash format is being stored here.
49 pub fn isOld(h: *const Hash) bool {
50 if (h.bytes.len < 2) return false;
51 const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
52 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
53 if (h.toSlice().len != multihash_hex_digest_len) return false;
54 return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
55 }
56
57 test isOld {
58 const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
59 try std.testing.expect(h.isOld());
60 }
61
62 /// Produces "$name-$semver-$sizedhash".
63 /// * name is the name field from build.zig.zon, truncated at 32 bytes and must
64 /// be a valid zig identifier
65 /// * semver is the version field from build.zig.zon, truncated at 32 bytes
66 /// * sizedhash is the following 9-byte array, base64 encoded using -_ to make
67 /// it filesystem safe:
68 /// - (4 bytes) LE u32 total decompressed size in bytes
69 /// - (5 bytes) truncated SHA-256 of hashed files of the package
70 ///
71 /// example: "nasm-2.16.1-2-BWdcABvF_jM1"
72 pub fn init(digest: Digest, name: []const u8, ver: []const u8, size: u32) Hash {
73 var result: Hash = undefined;
74 var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
75 buf.appendSliceAssumeCapacity(name[0..@min(name.len, 32)]);
76 buf.appendAssumeCapacity('-');
77 buf.appendSliceAssumeCapacity(ver[0..@min(ver.len, 32)]);
78 buf.appendAssumeCapacity('-');
79 var sizedhash: [9]u8 = undefined;
80 std.mem.writeInt(u32, sizedhash[0..4], size, .little);
81 sizedhash[4..].* = digest[0..5].*;
82 _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(12), &sizedhash);
83 @memset(buf.unusedCapacitySlice(), 0);
84 return result;
85 }
86
87 /// Produces "$hashiname-N-$sizedhash". For packages that lack "build.zig.zon" metadata.
88 /// * hashiname is [5..][0..24] bytes of the SHA-256, urlsafe-base64-encoded, for a total of 32 bytes encoded
89 /// * the semver section is replaced with a hardcoded N which stands for
90 /// "naked". It acts as a version number so that any future updates to the
91 /// hash format can tell this hash format apart. Note that "N" is an
92 /// invalid semver.
93 /// * sizedhash is the same as in `init`.
94 ///
95 /// The hash is broken up this way so that "sizedhash" can be calculated
96 /// exactly the same way in both cases, and so that "name" and "hashiname" can
97 /// be used interchangeably in both cases.
98 pub fn initNaked(digest: Digest, size: u32) Hash {
99 var name: [32]u8 = undefined;
100 _ = std.base64.url_safe_no_pad.Encoder.encode(&name, digest[5..][0..24]);
101 return init(digest, &name, "N", size);
102 }
103};
104
105pub const MultihashFunction = enum(u16) {
106 identity = 0x00,
107 sha1 = 0x11,
108 @"sha2-256" = 0x12,
109 @"sha2-512" = 0x13,
110 @"sha3-512" = 0x14,
111 @"sha3-384" = 0x15,
112 @"sha3-256" = 0x16,
113 @"sha3-224" = 0x17,
114 @"sha2-384" = 0x20,
115 @"sha2-256-trunc254-padded" = 0x1012,
116 @"sha2-224" = 0x1013,
117 @"sha2-512-224" = 0x1014,
118 @"sha2-512-256" = 0x1015,
119 @"blake2b-256" = 0xb220,
120 _,
121};
122
123pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
124 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
125 else => @compileError("unreachable"),
126};
127
128pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
129 const hex_charset = std.fmt.hex_charset;
130
131 var result: MultiHashHexDigest = undefined;
132
133 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
134 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
135
136 result[2] = hex_charset[Hash.Algo.digest_length >> 4];
137 result[3] = hex_charset[Hash.Algo.digest_length & 15];
138
139 for (digest, 0..) |byte, i| {
140 result[4 + i * 2] = hex_charset[byte >> 4];
141 result[5 + i * 2] = hex_charset[byte & 15];
142 }
143 return result;
144}
145
146comptime {
147 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
148 // values are small enough to be contained in the one-byte encoding.
149 assert(@intFromEnum(multihash_function) < 127);
150 assert(Hash.Algo.digest_length < 127);
151}
152
153test Hash {
154 const example_digest: Hash.Digest = .{
155 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
156 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
157 };
158 const result: Hash = .init(example_digest, "nasm", "2.16.1-2", 10 * 1024 * 1024);
159 try std.testing.expectEqualStrings("nasm-2.16.1-2-AACgAMf1cbe0", result.toSlice());
160}
161
6test {162test {
7 _ = Fetch;163 _ = Fetch;
8}164}
src/Package/Fetch.zig+92-62
...@@ -56,7 +56,7 @@ package_root: Cache.Path,...@@ -56,7 +56,7 @@ package_root: Cache.Path,
56error_bundle: ErrorBundle.Wip,56error_bundle: ErrorBundle.Wip,
57manifest: ?Manifest,57manifest: ?Manifest,
58manifest_ast: std.zig.Ast,58manifest_ast: std.zig.Ast,
59actual_hash: Manifest.Digest,59computed_hash: ComputedHash,
60/// Fetch logic notices whether a package has a build.zig file and sets this flag.60/// Fetch logic notices whether a package has a build.zig file and sets this flag.
61has_build_zig: bool,61has_build_zig: bool,
62/// Indicates whether the task aborted due to an out-of-memory condition.62/// Indicates whether the task aborted due to an out-of-memory condition.
...@@ -116,8 +116,8 @@ pub const JobQueue = struct {...@@ -116,8 +116,8 @@ pub const JobQueue = struct {
116 /// as lazy.116 /// as lazy.
117 unlazy_set: UnlazySet = .{},117 unlazy_set: UnlazySet = .{},
118118
119 pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);119 pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
120 pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);120 pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
121121
122 pub fn deinit(jq: *JobQueue) void {122 pub fn deinit(jq: *JobQueue) void {
123 if (jq.all_fetches.items.len == 0) return;123 if (jq.all_fetches.items.len == 0) return;
...@@ -160,22 +160,24 @@ pub const JobQueue = struct {...@@ -160,22 +160,24 @@ pub const JobQueue = struct {
160160
161 // Ensure the generated .zig file is deterministic.161 // Ensure the generated .zig file is deterministic.
162 jq.table.sortUnstable(@as(struct {162 jq.table.sortUnstable(@as(struct {
163 keys: []const Manifest.MultiHashHexDigest,163 keys: []const Package.Hash,
164 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {164 pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
165 return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);165 return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
166 }166 }
167 }, .{ .keys = keys }));167 }, .{ .keys = keys }));
168168
169 for (keys, jq.table.values()) |hash, fetch| {169 for (keys, jq.table.values()) |*hash, fetch| {
170 if (fetch == jq.all_fetches.items[0]) {170 if (fetch == jq.all_fetches.items[0]) {
171 // The first one is a dummy package for the current project.171 // The first one is a dummy package for the current project.
172 continue;172 continue;
173 }173 }
174174
175 const hash_slice = hash.toSlice();
176
175 try buf.writer().print(177 try buf.writer().print(
176 \\ pub const {} = struct {{178 \\ pub const {} = struct {{
177 \\179 \\
178 , .{std.zig.fmtId(&hash)});180 , .{std.zig.fmtId(hash_slice)});
179181
180 lazy: {182 lazy: {
181 switch (fetch.lazy_status) {183 switch (fetch.lazy_status) {
...@@ -207,7 +209,7 @@ pub const JobQueue = struct {...@@ -207,7 +209,7 @@ pub const JobQueue = struct {
207 try buf.writer().print(209 try buf.writer().print(
208 \\ pub const build_zig = @import("{}");210 \\ pub const build_zig = @import("{}");
209 \\211 \\
210 , .{std.zig.fmtEscapes(&hash)});212 , .{std.zig.fmtEscapes(hash_slice)});
211 }213 }
212214
213 if (fetch.manifest) |*manifest| {215 if (fetch.manifest) |*manifest| {
...@@ -219,7 +221,7 @@ pub const JobQueue = struct {...@@ -219,7 +221,7 @@ pub const JobQueue = struct {
219 const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;221 const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
220 try buf.writer().print(222 try buf.writer().print(
221 " .{{ \"{}\", \"{}\" }},\n",223 " .{{ \"{}\", \"{}\" }},\n",
222 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },224 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
223 );225 );
224 }226 }
225227
...@@ -251,7 +253,7 @@ pub const JobQueue = struct {...@@ -251,7 +253,7 @@ pub const JobQueue = struct {
251 const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;253 const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
252 try buf.writer().print(254 try buf.writer().print(
253 " .{{ \"{}\", \"{}\" }},\n",255 " .{{ \"{}\", \"{}\" }},\n",
254 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },256 .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
255 );257 );
256 }258 }
257 try buf.appendSlice("};\n");259 try buf.appendSlice("};\n");
...@@ -283,7 +285,7 @@ pub const Location = union(enum) {...@@ -283,7 +285,7 @@ pub const Location = union(enum) {
283 url: []const u8,285 url: []const u8,
284 /// If this is null it means the user omitted the hash field from a dependency.286 /// If this is null it means the user omitted the hash field from a dependency.
285 /// It will be an error but the logic should still fetch and print the discovered hash.287 /// It will be an error but the logic should still fetch and print the discovered hash.
286 hash: ?Manifest.MultiHashHexDigest,288 hash: ?Package.Hash,
287 };289 };
288};290};
289291
...@@ -325,9 +327,11 @@ pub fn run(f: *Fetch) RunError!void {...@@ -325,9 +327,11 @@ pub fn run(f: *Fetch) RunError!void {
325 // "p/$hash/foo", with possibly more directories after "foo".327 // "p/$hash/foo", with possibly more directories after "foo".
326 // We want to fail unless the resolved relative path has a328 // We want to fail unless the resolved relative path has a
327 // prefix of "p/$hash/".329 // prefix of "p/$hash/".
328 const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
329 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;330 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
330 const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len];331 const parent_sub_path = f.parent_package_root.sub_path;
332 const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse
333 parent_sub_path.len;
334 const expected_prefix = parent_sub_path[prefix_len..end];
331 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {335 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
332 return f.fail(336 return f.fail(
333 f.location_tok,337 f.location_tok,
...@@ -367,9 +371,13 @@ pub fn run(f: *Fetch) RunError!void {...@@ -367,9 +371,13 @@ pub fn run(f: *Fetch) RunError!void {
367 },371 },
368 };372 };
369373
370 const s = fs.path.sep_str;
371 if (remote.hash) |expected_hash| {374 if (remote.hash) |expected_hash| {
372 const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;375 var prefixed_pkg_sub_path_buffer: [100]u8 = undefined;
376 prefixed_pkg_sub_path_buffer[0] = 'p';
377 prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
378 const hash_slice = expected_hash.toSlice();
379 @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
380 const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
373 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;381 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
374 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];382 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
375 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {383 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
...@@ -437,7 +445,7 @@ fn runResource(...@@ -437,7 +445,7 @@ fn runResource(
437 f: *Fetch,445 f: *Fetch,
438 uri_path: []const u8,446 uri_path: []const u8,
439 resource: *Resource,447 resource: *Resource,
440 remote_hash: ?Manifest.MultiHashHexDigest,448 remote_hash: ?Package.Hash,
441) RunError!void {449) RunError!void {
442 defer resource.deinit();450 defer resource.deinit();
443 const arena = f.arena.allocator();451 const arena = f.arena.allocator();
...@@ -499,7 +507,7 @@ fn runResource(...@@ -499,7 +507,7 @@ fn runResource(
499 // Empty directories have already been omitted by `unpackResource`.507 // Empty directories have already been omitted by `unpackResource`.
500 // Compute the package hash based on the remaining files in the temporary508 // Compute the package hash based on the remaining files in the temporary
501 // directory.509 // directory.
502 f.actual_hash = try computeHash(f, pkg_path, filter);510 f.computed_hash = try computeHash(f, pkg_path, filter);
503511
504 break :blk if (unpack_result.root_dir.len > 0)512 break :blk if (unpack_result.root_dir.len > 0)
505 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })513 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
...@@ -507,6 +515,8 @@ fn runResource(...@@ -507,6 +515,8 @@ fn runResource(
507 tmp_dir_sub_path;515 tmp_dir_sub_path;
508 };516 };
509517
518 const computed_package_hash = computedPackageHash(f);
519
510 // Rename the temporary directory into the global zig package cache520 // Rename the temporary directory into the global zig package cache
511 // directory. If the hash already exists, delete the temporary directory521 // directory. If the hash already exists, delete the temporary directory
512 // and leave the zig package cache directory untouched as it may be in use522 // and leave the zig package cache directory untouched as it may be in use
...@@ -515,7 +525,7 @@ fn runResource(...@@ -515,7 +525,7 @@ fn runResource(
515525
516 f.package_root = .{526 f.package_root = .{
517 .root_dir = cache_root,527 .root_dir = cache_root,
518 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),528 .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
519 };529 };
520 renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {530 renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
521 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});531 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
...@@ -534,13 +544,22 @@ fn runResource(...@@ -534,13 +544,22 @@ fn runResource(
534 // Validate the computed hash against the expected hash. If invalid, this544 // Validate the computed hash against the expected hash. If invalid, this
535 // job is done.545 // job is done.
536546
537 const actual_hex = Manifest.hexDigest(f.actual_hash);
538 if (remote_hash) |declared_hash| {547 if (remote_hash) |declared_hash| {
539 if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {548 if (declared_hash.isOld()) {
540 return f.fail(f.hash_tok, try eb.printString(549 const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
541 "hash mismatch: manifest declares {s} but the fetched package has {s}",550 if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
542 .{ declared_hash, actual_hex },551 return f.fail(f.hash_tok, try eb.printString(
543 ));552 "hash mismatch: manifest declares {s} but the fetched package has {s}",
553 .{ declared_hash.toSlice(), actual_hex },
554 ));
555 }
556 } else {
557 if (!computed_package_hash.eql(&declared_hash)) {
558 return f.fail(f.hash_tok, try eb.printString(
559 "hash mismatch: manifest declares {s} but the fetched package has {s}",
560 .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
561 ));
562 }
544 }563 }
545 } else if (!f.omit_missing_hash_error) {564 } else if (!f.omit_missing_hash_error) {
546 const notes_len = 1;565 const notes_len = 1;
...@@ -551,7 +570,7 @@ fn runResource(...@@ -551,7 +570,7 @@ fn runResource(
551 });570 });
552 const notes_start = try eb.reserveNotes(notes_len);571 const notes_start = try eb.reserveNotes(notes_len);
553 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{572 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
554 .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),573 .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
555 }));574 }));
556 return error.FetchFailed;575 return error.FetchFailed;
557 }576 }
...@@ -562,6 +581,16 @@ fn runResource(...@@ -562,6 +581,16 @@ fn runResource(
562 return queueJobsForDeps(f);581 return queueJobsForDeps(f);
563}582}
564583
584pub fn computedPackageHash(f: *const Fetch) Package.Hash {
585 const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
586 if (f.manifest) |man| {
587 var version_buffer: [32]u8 = undefined;
588 const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
589 return .init(f.computed_hash.digest, man.name, version, saturated_size);
590 }
591 return .initNaked(f.computed_hash.digest, saturated_size);
592}
593
565/// `computeHash` gets a free check for the existence of `build.zig`, but when594/// `computeHash` gets a free check for the existence of `build.zig`, but when
566/// not computing a hash, we need to do a syscall to check for it.595/// not computing a hash, we need to do a syscall to check for it.
567fn checkBuildFileExistence(f: *Fetch) RunError!void {596fn checkBuildFileExistence(f: *Fetch) RunError!void {
...@@ -673,9 +702,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -673,9 +702,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
673 .url = url,702 .url = url,
674 .hash = h: {703 .hash = h: {
675 const h = dep.hash orelse break :h null;704 const h = dep.hash orelse break :h null;
676 const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;705 const pkg_hash: Package.Hash = .fromSlice(h);
677 const multihash_digest = h[0..digest_len].*;706 const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
678 const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
679 if (gop.found_existing) {707 if (gop.found_existing) {
680 if (!dep.lazy) {708 if (!dep.lazy) {
681 gop.value_ptr.*.lazy_status = .eager;709 gop.value_ptr.*.lazy_status = .eager;
...@@ -683,15 +711,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -683,15 +711,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
683 continue;711 continue;
684 }712 }
685 gop.value_ptr.* = new_fetch;713 gop.value_ptr.* = new_fetch;
686 break :h multihash_digest;714 break :h pkg_hash;
687 },715 },
688 } },716 } },
689 .path => |rel_path| l: {717 .path => |rel_path| l: {
690 // This might produce an invalid path, which is checked for718 // This might produce an invalid path, which is checked for
691 // at the beginning of run().719 // at the beginning of run().
692 const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);720 const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
693 const multihash_digest = relativePathDigest(new_root, cache_root);721 const pkg_hash = relativePathDigest(new_root, cache_root);
694 const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);722 const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
695 if (gop.found_existing) {723 if (gop.found_existing) {
696 if (!dep.lazy) {724 if (!dep.lazy) {
697 gop.value_ptr.*.lazy_status = .eager;725 gop.value_ptr.*.lazy_status = .eager;
...@@ -724,7 +752,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -724,7 +752,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
724 .error_bundle = undefined,752 .error_bundle = undefined,
725 .manifest = null,753 .manifest = null,
726 .manifest_ast = undefined,754 .manifest_ast = undefined,
727 .actual_hash = undefined,755 .computed_hash = undefined,
728 .has_build_zig = false,756 .has_build_zig = false,
729 .oom_flag = false,757 .oom_flag = false,
730 .latest_commit = null,758 .latest_commit = null,
...@@ -746,11 +774,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -746,11 +774,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
746 }774 }
747}775}
748776
749pub fn relativePathDigest(777pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
750 pkg_root: Cache.Path,778 var hasher = Package.Hash.Algo.init(.{});
751 cache_root: Cache.Directory,
752) Manifest.MultiHashHexDigest {
753 var hasher = Manifest.Hash.init(.{});
754 // This hash is a tuple of:779 // This hash is a tuple of:
755 // * whether it relative to the global cache directory or to the root package780 // * whether it relative to the global cache directory or to the root package
756 // * the relative file path from there to the build root of the package781 // * the relative file path from there to the build root of the package
...@@ -759,7 +784,7 @@ pub fn relativePathDigest(...@@ -759,7 +784,7 @@ pub fn relativePathDigest(
759 else784 else
760 &package_hash_prefix_project);785 &package_hash_prefix_project);
761 hasher.update(pkg_root.sub_path);786 hasher.update(pkg_root.sub_path);
762 return Manifest.hexDigest(hasher.finalResult());787 return .fromSlice(&hasher.finalResult());
763}788}
764789
765pub fn workerRun(f: *Fetch, prog_name: []const u8) void {790pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
...@@ -1387,11 +1412,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void...@@ -1387,11 +1412,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void
1387 }1412 }
1388}1413}
13891414
1390pub fn renameTmpIntoCache(1415pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
1391 cache_dir: fs.Dir,
1392 tmp_dir_sub_path: []const u8,
1393 dest_dir_sub_path: []const u8,
1394) !void {
1395 assert(dest_dir_sub_path[1] == fs.path.sep);1416 assert(dest_dir_sub_path[1] == fs.path.sep);
1396 var handled_missing_dir = false;1417 var handled_missing_dir = false;
1397 while (true) {1418 while (true) {
...@@ -1417,16 +1438,17 @@ pub fn renameTmpIntoCache(...@@ -1417,16 +1438,17 @@ pub fn renameTmpIntoCache(
1417 }1438 }
1418}1439}
14191440
1441const ComputedHash = struct {
1442 digest: Package.Hash.Digest,
1443 total_size: u64,
1444};
1445
1420/// Assumes that files not included in the package have already been filtered1446/// Assumes that files not included in the package have already been filtered
1421/// prior to calling this function. This ensures that files not protected by1447/// prior to calling this function. This ensures that files not protected by
1422/// the hash are not present on the file system. Empty directories are *not1448/// the hash are not present on the file system. Empty directories are *not
1423/// hashed* and must not be present on the file system when calling this1449/// hashed* and must not be present on the file system when calling this
1424/// function.1450/// function.
1425fn computeHash(1451fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
1426 f: *Fetch,
1427 pkg_path: Cache.Path,
1428 filter: Filter,
1429) RunError!Manifest.Digest {
1430 // All the path name strings need to be in memory for sorting.1452 // All the path name strings need to be in memory for sorting.
1431 const arena = f.arena.allocator();1453 const arena = f.arena.allocator();
1432 const gpa = f.arena.child_allocator;1454 const gpa = f.arena.child_allocator;
...@@ -1449,6 +1471,9 @@ fn computeHash(...@@ -1449,6 +1471,9 @@ fn computeHash(
1449 var walker = try root_dir.walk(gpa);1471 var walker = try root_dir.walk(gpa);
1450 defer walker.deinit();1472 defer walker.deinit();
14511473
1474 // Total number of bytes of file contents included in the package.
1475 var total_size: u64 = 0;
1476
1452 {1477 {
1453 // The final hash will be a hash of each file hashed independently. This1478 // The final hash will be a hash of each file hashed independently. This
1454 // allows hashing in parallel.1479 // allows hashing in parallel.
...@@ -1506,6 +1531,7 @@ fn computeHash(...@@ -1506,6 +1531,7 @@ fn computeHash(
1506 .kind = kind,1531 .kind = kind,
1507 .hash = undefined, // to be populated by the worker1532 .hash = undefined, // to be populated by the worker
1508 .failure = undefined, // to be populated by the worker1533 .failure = undefined, // to be populated by the worker
1534 .size = undefined, // to be populated by the worker
1509 };1535 };
1510 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });1536 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
1511 try all_files.append(hashed_file);1537 try all_files.append(hashed_file);
...@@ -1544,7 +1570,7 @@ fn computeHash(...@@ -1544,7 +1570,7 @@ fn computeHash(
15441570
1545 std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);1571 std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
15461572
1547 var hasher = Manifest.Hash.init(.{});1573 var hasher = Package.Hash.Algo.init(.{});
1548 var any_failures = false;1574 var any_failures = false;
1549 for (all_files.items) |hashed_file| {1575 for (all_files.items) |hashed_file| {
1550 hashed_file.failure catch |err| {1576 hashed_file.failure catch |err| {
...@@ -1556,6 +1582,7 @@ fn computeHash(...@@ -1556,6 +1582,7 @@ fn computeHash(
1556 });1582 });
1557 };1583 };
1558 hasher.update(&hashed_file.hash);1584 hasher.update(&hashed_file.hash);
1585 total_size += hashed_file.size;
1559 }1586 }
1560 for (deleted_files.items) |deleted_file| {1587 for (deleted_files.items) |deleted_file| {
1561 deleted_file.failure catch |err| {1588 deleted_file.failure catch |err| {
...@@ -1580,7 +1607,10 @@ fn computeHash(...@@ -1580,7 +1607,10 @@ fn computeHash(
1580 };1607 };
1581 }1608 }
15821609
1583 return hasher.finalResult();1610 return .{
1611 .digest = hasher.finalResult(),
1612 .total_size = total_size,
1613 };
1584}1614}
15851615
1586fn dumpHashInfo(all_files: []const *const HashedFile) !void {1616fn dumpHashInfo(all_files: []const *const HashedFile) !void {
...@@ -1609,8 +1639,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {...@@ -1609,8 +1639,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
16091639
1610fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {1640fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1611 var buf: [8000]u8 = undefined;1641 var buf: [8000]u8 = undefined;
1612 var hasher = Manifest.Hash.init(.{});1642 var hasher = Package.Hash.Algo.init(.{});
1613 hasher.update(hashed_file.normalized_path);1643 hasher.update(hashed_file.normalized_path);
1644 var file_size: u64 = 0;
16141645
1615 switch (hashed_file.kind) {1646 switch (hashed_file.kind) {
1616 .file => {1647 .file => {
...@@ -1622,6 +1653,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1622,6 +1653,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1622 while (true) {1653 while (true) {
1623 const bytes_read = try file.read(&buf);1654 const bytes_read = try file.read(&buf);
1624 if (bytes_read == 0) break;1655 if (bytes_read == 0) break;
1656 file_size += bytes_read;
1625 hasher.update(buf[0..bytes_read]);1657 hasher.update(buf[0..bytes_read]);
1626 file_header.update(buf[0..bytes_read]);1658 file_header.update(buf[0..bytes_read]);
1627 }1659 }
...@@ -1641,6 +1673,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void...@@ -1641,6 +1673,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
1641 },1673 },
1642 }1674 }
1643 hasher.final(&hashed_file.hash);1675 hasher.final(&hashed_file.hash);
1676 hashed_file.size = file_size;
1644}1677}
16451678
1646fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {1679fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
...@@ -1667,9 +1700,10 @@ const DeletedFile = struct {...@@ -1667,9 +1700,10 @@ const DeletedFile = struct {
1667const HashedFile = struct {1700const HashedFile = struct {
1668 fs_path: []const u8,1701 fs_path: []const u8,
1669 normalized_path: []const u8,1702 normalized_path: []const u8,
1670 hash: Manifest.Digest,1703 hash: Package.Hash.Digest,
1671 failure: Error!void,1704 failure: Error!void,
1672 kind: Kind,1705 kind: Kind,
1706 size: u64,
16731707
1674 const Error =1708 const Error =
1675 fs.File.OpenError ||1709 fs.File.OpenError ||
...@@ -1744,12 +1778,8 @@ const Filter = struct {...@@ -1744,12 +1778,8 @@ const Filter = struct {
1744 }1778 }
1745};1779};
17461780
1747pub fn depDigest(1781pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
1748 pkg_root: Cache.Path,1782 if (dep.hash) |h| return .fromSlice(h);
1749 cache_root: Cache.Directory,
1750 dep: Manifest.Dependency,
1751) ?Manifest.MultiHashHexDigest {
1752 if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*;
17531783
1754 switch (dep.location) {1784 switch (dep.location) {
1755 .url => return null,1785 .url => return null,
...@@ -2137,7 +2167,7 @@ test "tarball with excluded duplicate paths" {...@@ -2137,7 +2167,7 @@ test "tarball with excluded duplicate paths" {
2137 defer fb.deinit();2167 defer fb.deinit();
2138 try fetch.run();2168 try fetch.run();
21392169
2140 const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);2170 const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2141 try std.testing.expectEqualStrings(2171 try std.testing.expectEqualStrings(
2142 "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",2172 "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
2143 &hex_digest,2173 &hex_digest,
...@@ -2181,7 +2211,7 @@ test "tarball without root folder" {...@@ -2181,7 +2211,7 @@ test "tarball without root folder" {
2181 defer fb.deinit();2211 defer fb.deinit();
2182 try fetch.run();2212 try fetch.run();
21832213
2184 const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);2214 const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2185 try std.testing.expectEqualStrings(2215 try std.testing.expectEqualStrings(
2186 "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",2216 "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
2187 &hex_digest,2217 &hex_digest,
...@@ -2222,7 +2252,7 @@ test "set executable bit based on file content" {...@@ -2222,7 +2252,7 @@ test "set executable bit based on file content" {
2222 try fetch.run();2252 try fetch.run();
2223 try std.testing.expectEqualStrings(2253 try std.testing.expectEqualStrings(
2224 "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",2254 "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
2225 &Manifest.hexDigest(fetch.actual_hash),2255 &Package.multiHashHexDigest(fetch.computed_hash.digest),
2226 );2256 );
22272257
2228 var out = try fb.packageDir();2258 var out = try fb.packageDir();
...@@ -2304,7 +2334,7 @@ const TestFetchBuilder = struct {...@@ -2304,7 +2334,7 @@ const TestFetchBuilder = struct {
2304 .error_bundle = undefined,2334 .error_bundle = undefined,
2305 .manifest = null,2335 .manifest = null,
2306 .manifest_ast = undefined,2336 .manifest_ast = undefined,
2307 .actual_hash = undefined,2337 .computed_hash = undefined,
2308 .has_build_zig = false,2338 .has_build_zig = false,
2309 .oom_flag = false,2339 .oom_flag = false,
2310 .latest_commit = null,2340 .latest_commit = null,
src/Package/Manifest.zig+3-66
...@@ -5,15 +5,10 @@ const Allocator = std.mem.Allocator;...@@ -5,15 +5,10 @@ const Allocator = std.mem.Allocator;
5const assert = std.debug.assert;5const assert = std.debug.assert;
6const Ast = std.zig.Ast;6const Ast = std.zig.Ast;
7const testing = std.testing;7const testing = std.testing;
8const hex_charset = std.fmt.hex_charset;8const Package = @import("../Package.zig");
99
10pub const max_bytes = 10 * 1024 * 1024;10pub const max_bytes = 10 * 1024 * 1024;
11pub const basename = "build.zig.zon";11pub const basename = "build.zig.zon";
12pub const Hash = std.crypto.hash.sha2.Sha256;
13pub const Digest = [Hash.digest_length]u8;
14pub const multihash_len = 1 + 1 + Hash.digest_length;
15pub const multihash_hex_digest_len = 2 * multihash_len;
16pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
1712
18pub const Dependency = struct {13pub const Dependency = struct {
19 location: Location,14 location: Location,
...@@ -38,35 +33,6 @@ pub const ErrorMessage = struct {...@@ -38,35 +33,6 @@ pub const ErrorMessage = struct {
38 off: u32,33 off: u32,
39};34};
4035
41pub const MultihashFunction = enum(u16) {
42 identity = 0x00,
43 sha1 = 0x11,
44 @"sha2-256" = 0x12,
45 @"sha2-512" = 0x13,
46 @"sha3-512" = 0x14,
47 @"sha3-384" = 0x15,
48 @"sha3-256" = 0x16,
49 @"sha3-224" = 0x17,
50 @"sha2-384" = 0x20,
51 @"sha2-256-trunc254-padded" = 0x1012,
52 @"sha2-224" = 0x1013,
53 @"sha2-512-224" = 0x1014,
54 @"sha2-512-256" = 0x1015,
55 @"blake2b-256" = 0xb220,
56 _,
57};
58
59pub const multihash_function: MultihashFunction = switch (Hash) {
60 std.crypto.hash.sha2.Sha256 => .@"sha2-256",
61 else => @compileError("unreachable"),
62};
63comptime {
64 // We avoid unnecessary uleb128 code in hexDigest by asserting here the
65 // values are small enough to be contained in the one-byte encoding.
66 assert(@intFromEnum(multihash_function) < 127);
67 assert(Hash.digest_length < 127);
68}
69
70name: []const u8,36name: []const u8,
71version: std.SemanticVersion,37version: std.SemanticVersion,
72version_node: Ast.Node.Index,38version_node: Ast.Node.Index,
...@@ -164,22 +130,6 @@ pub fn copyErrorsIntoBundle(...@@ -164,22 +130,6 @@ pub fn copyErrorsIntoBundle(
164 }130 }
165}131}
166132
167pub fn hexDigest(digest: Digest) MultiHashHexDigest {
168 var result: MultiHashHexDigest = undefined;
169
170 result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
171 result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
172
173 result[2] = hex_charset[Hash.digest_length >> 4];
174 result[3] = hex_charset[Hash.digest_length & 15];
175
176 for (digest, 0..) |byte, i| {
177 result[4 + i * 2] = hex_charset[byte >> 4];
178 result[5 + i * 2] = hex_charset[byte & 15];
179 }
180 return result;
181}
182
183const Parse = struct {133const Parse = struct {
184 gpa: Allocator,134 gpa: Allocator,
185 ast: Ast,135 ast: Ast,
...@@ -421,21 +371,8 @@ const Parse = struct {...@@ -421,21 +371,8 @@ const Parse = struct {
421 const tok = main_tokens[node];371 const tok = main_tokens[node];
422 const h = try parseString(p, node);372 const h = try parseString(p, node);
423373
424 if (h.len >= 2) {374 if (h.len > Package.Hash.max_len) {
425 const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {375 return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len});
426 return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
427 @errorName(err),
428 });
429 };
430 if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
431 return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
432 }
433 }
434
435 if (h.len != multihash_hex_digest_len) {
436 return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
437 multihash_hex_digest_len, h.len,
438 });
439 }376 }
440377
441 return h;378 return h;
src/main.zig+18-17
...@@ -5197,7 +5197,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5197,7 +5197,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5197 .error_bundle = undefined,5197 .error_bundle = undefined,
5198 .manifest = null,5198 .manifest = null,
5199 .manifest_ast = undefined,5199 .manifest_ast = undefined,
5200 .actual_hash = undefined,5200 .computed_hash = undefined,
5201 .has_build_zig = true,5201 .has_build_zig = true,
5202 .oom_flag = false,5202 .oom_flag = false,
5203 .latest_commit = null,5203 .latest_commit = null,
...@@ -5244,13 +5244,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5244,13 +5244,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5244 const hashes = job_queue.table.keys();5244 const hashes = job_queue.table.keys();
5245 const fetches = job_queue.table.values();5245 const fetches = job_queue.table.values();
5246 try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));5246 try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
5247 for (hashes, fetches) |hash, f| {5247 for (hashes, fetches) |*hash, f| {
5248 if (f == &fetch) {5248 if (f == &fetch) {
5249 // The first one is a dummy package for the current project.5249 // The first one is a dummy package for the current project.
5250 continue;5250 continue;
5251 }5251 }
5252 if (!f.has_build_zig)5252 if (!f.has_build_zig)
5253 continue;5253 continue;
5254 const hash_slice = hash.toSlice();
5254 const m = try Package.Module.create(arena, .{5255 const m = try Package.Module.create(arena, .{
5255 .global_cache_directory = global_cache_directory,5256 .global_cache_directory = global_cache_directory,
5256 .paths = .{5257 .paths = .{
...@@ -5260,7 +5261,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5260,7 +5261,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5260 .fully_qualified_name = try std.fmt.allocPrint(5261 .fully_qualified_name = try std.fmt.allocPrint(
5261 arena,5262 arena,
5262 "root.@dependencies.{s}",5263 "root.@dependencies.{s}",
5263 .{&hash},5264 .{hash_slice},
5264 ),5265 ),
5265 .cc_argv = &.{},5266 .cc_argv = &.{},
5266 .inherited = .{},5267 .inherited = .{},
...@@ -5269,7 +5270,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5269,7 +5270,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5269 .builtin_mod = builtin_mod,5270 .builtin_mod = builtin_mod,
5270 .builtin_modules = null, // `builtin_mod` is specified5271 .builtin_modules = null, // `builtin_mod` is specified
5271 });5272 });
5272 const hash_cloned = try arena.dupe(u8, &hash);5273 const hash_cloned = try arena.dupe(u8, hash_slice);
5273 deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);5274 deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
5274 f.module = m;5275 f.module = m;
5275 }5276 }
...@@ -5385,23 +5386,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {...@@ -5385,23 +5386,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
5385 var any_errors = false;5386 var any_errors = false;
5386 while (it.next()) |hash| {5387 while (it.next()) |hash| {
5387 if (hash.len == 0) continue;5388 if (hash.len == 0) continue;
5388 const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len;5389 if (hash.len > Package.Hash.max_len) {
5389 if (hash.len != digest_len) {5390 std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
5390 std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{5391 hash.len, hash,
5391 hash.len, digest_len, hash,
5392 });5392 });
5393 any_errors = true;5393 any_errors = true;
5394 continue;5394 continue;
5395 }5395 }
5396 try unlazy_set.put(arena, hash[0..digest_len].*, {});5396 try unlazy_set.put(arena, .fromSlice(hash), {});
5397 }5397 }
5398 if (any_errors) process.exit(3);5398 if (any_errors) process.exit(3);
5399 if (system_pkg_dir_path) |p| {5399 if (system_pkg_dir_path) |p| {
5400 // In this mode, the system needs to provide these packages; they5400 // In this mode, the system needs to provide these packages; they
5401 // cannot be fetched by Zig.5401 // cannot be fetched by Zig.
5402 for (unlazy_set.keys()) |hash| {5402 for (unlazy_set.keys()) |*hash| {
5403 std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{5403 std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
5404 p, hash,5404 p, hash.toSlice(),
5405 });5405 });
5406 }5406 }
5407 std.log.info("remote package fetching disabled due to --system mode", .{});5407 std.log.info("remote package fetching disabled due to --system mode", .{});
...@@ -7097,7 +7097,7 @@ fn cmdFetch(...@@ -7097,7 +7097,7 @@ fn cmdFetch(
7097 .error_bundle = undefined,7097 .error_bundle = undefined,
7098 .manifest = null,7098 .manifest = null,
7099 .manifest_ast = undefined,7099 .manifest_ast = undefined,
7100 .actual_hash = undefined,7100 .computed_hash = undefined,
7101 .has_build_zig = false,7101 .has_build_zig = false,
7102 .oom_flag = false,7102 .oom_flag = false,
7103 .latest_commit = null,7103 .latest_commit = null,
...@@ -7117,14 +7117,15 @@ fn cmdFetch(...@@ -7117,14 +7117,15 @@ fn cmdFetch(
7117 process.exit(1);7117 process.exit(1);
7118 }7118 }
71197119
7120 const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);7120 const package_hash = fetch.computedPackageHash();
7121 const package_hash_slice = package_hash.toSlice();
71217122
7122 root_prog_node.end();7123 root_prog_node.end();
7123 root_prog_node = .{ .index = .none };7124 root_prog_node = .{ .index = .none };
71247125
7125 const name = switch (save) {7126 const name = switch (save) {
7126 .no => {7127 .no => {
7127 try io.getStdOut().writeAll(hex_digest ++ "\n");7128 try io.getStdOut().writer().print("{s}\n", .{package_hash_slice});
7128 return cleanExit();7129 return cleanExit();
7129 },7130 },
7130 .yes, .exact => |name| name: {7131 .yes, .exact => |name| name: {
...@@ -7194,7 +7195,7 @@ fn cmdFetch(...@@ -7194,7 +7195,7 @@ fn cmdFetch(
7194 \\ }}7195 \\ }}
7195 , .{7196 , .{
7196 std.zig.fmtEscapes(saved_path_or_url),7197 std.zig.fmtEscapes(saved_path_or_url),
7197 std.zig.fmtEscapes(&hex_digest),7198 std.zig.fmtEscapes(package_hash_slice),
7198 });7199 });
71997200
7200 const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{7201 const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{
...@@ -7213,7 +7214,7 @@ fn cmdFetch(...@@ -7213,7 +7214,7 @@ fn cmdFetch(
7213 if (dep.hash) |h| {7214 if (dep.hash) |h| {
7214 switch (dep.location) {7215 switch (dep.location) {
7215 .url => |u| {7216 .url => |u| {
7216 if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) {7217 if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
7217 std.log.info("existing dependency named '{s}' is up-to-date", .{name});7218 std.log.info("existing dependency named '{s}' is up-to-date", .{name});
7218 process.exit(0);7219 process.exit(0);
7219 }7220 }
...@@ -7230,7 +7231,7 @@ fn cmdFetch(...@@ -7230,7 +7231,7 @@ fn cmdFetch(
7230 const hash_replace = try std.fmt.allocPrint(7231 const hash_replace = try std.fmt.allocPrint(
7231 arena,7232 arena,
7232 "\"{}\"",7233 "\"{}\"",
7233 .{std.zig.fmtEscapes(&hex_digest)},7234 .{std.zig.fmtEscapes(package_hash_slice)},
7234 );7235 );
72357236
7236 warn("overwriting existing dependency named '{s}'", .{name});7237 warn("overwriting existing dependency named '{s}'", .{name});