authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-06-03 13:44:56-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-06-03 13:44:56-07:00
log25cf0b59d86cfba7bafec7d9aeae31e843a5417e
tree0272098b8ac61279105151e8f24ee4499a189b97
parent7fc3fb955abf3d510079ad5e42027b90e897dc20

enough code to get some examples working

2 files changed, 102 insertions(+), 7 deletions(-)

src/Package/Fetch.zig+22-7
......@@ -447,7 +447,7 @@ fn runResource(
447447 const rand_int = std.crypto.random.int(u64);
448448 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
449449
450 const package_sub_path = blk: {
450 const package_sub_path, const size = blk: {
451451 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
452452 var tmp_directory: Cache.Directory = .{
453453 .path = tmp_directory_path,
......@@ -499,12 +499,14 @@ fn runResource(
499499 // Empty directories have already been omitted by `unpackResource`.
500500 // Compute the package hash based on the remaining files in the temporary
501501 // directory.
502 f.actual_hash = try computeHash(f, pkg_path, filter);
502 f.actual_hash, const size = try computeHash(f, pkg_path, filter);
503503
504 break :blk if (unpack_result.root_dir.len > 0)
504 const pkg_sub_path = if (unpack_result.root_dir.len > 0)
505505 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
506506 else
507507 tmp_dir_sub_path;
508
509 break :blk .{ pkg_sub_path, size };
508510 };
509511
510512 // Rename the temporary directory into the global zig package cache
......@@ -513,6 +515,14 @@ fn runResource(
513515 // by the system. This is done even if the hash is invalid, in case the
514516 // package with the different hash is used in the future.
515517
518 const name = f.manifest.?.name;
519 const semver = f.manifest.?.version;
520 const new_hash = Package.Manifest.NewHashDecoded.init(name, semver, @intCast(size), f.actual_hash);
521 const encoded = new_hash.encode();
522 std.debug.print("new hash: {s}\n", .{encoded.toSlice()});
523
524 if (true) std.process.exit(0);
525
516526 f.package_root = .{
517527 .root_dir = cache_root,
518528 .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
......@@ -1420,7 +1430,7 @@ fn computeHash(
14201430 f: *Fetch,
14211431 pkg_path: Cache.Path,
14221432 filter: Filter,
1423) RunError!Manifest.Digest {
1433) RunError!struct { Manifest.Digest, u64 } {
14241434 // All the path name strings need to be in memory for sorting.
14251435 const arena = f.arena.allocator();
14261436 const gpa = f.arena.child_allocator;
......@@ -1499,6 +1509,7 @@ fn computeHash(
14991509 .normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
15001510 .kind = kind,
15011511 .hash = undefined, // to be populated by the worker
1512 .size = undefined, // to be populated by the worker
15021513 .failure = undefined, // to be populated by the worker
15031514 };
15041515 thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
......@@ -1540,6 +1551,7 @@ fn computeHash(
15401551
15411552 var hasher = Manifest.Hash.init(.{});
15421553 var any_failures = false;
1554 var total_size: u64 = 0;
15431555 for (all_files.items) |hashed_file| {
15441556 hashed_file.failure catch |err| {
15451557 any_failures = true;
......@@ -1550,6 +1562,7 @@ fn computeHash(
15501562 });
15511563 };
15521564 hasher.update(&hashed_file.hash);
1565 total_size += hashed_file.size;
15531566 }
15541567 for (deleted_files.items) |deleted_file| {
15551568 deleted_file.failure catch |err| {
......@@ -1574,7 +1587,7 @@ fn computeHash(
15741587 };
15751588 }
15761589
1577 return hasher.finalResult();
1590 return .{ hasher.finalResult(), total_size };
15781591}
15791592
15801593fn dumpHashInfo(all_files: []const *const HashedFile) !void {
......@@ -1604,20 +1617,20 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
16041617fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
16051618 var buf: [8000]u8 = undefined;
16061619 var hasher = Manifest.Hash.init(.{});
1620 var size: u64 = 0;
16071621 hasher.update(hashed_file.normalized_path);
16081622
16091623 switch (hashed_file.kind) {
16101624 .file => {
16111625 var file = try dir.openFile(hashed_file.fs_path, .{});
16121626 defer file.close();
1613 // Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
1614 hasher.update(&.{ 0, 0 });
16151627 var file_header: FileHeader = .{};
16161628 while (true) {
16171629 const bytes_read = try file.read(&buf);
16181630 if (bytes_read == 0) break;
16191631 hasher.update(buf[0..bytes_read]);
16201632 file_header.update(buf[0..bytes_read]);
1633 size += bytes_read;
16211634 }
16221635 if (file_header.isExecutable()) {
16231636 try setExecutable(file);
......@@ -1635,6 +1648,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
16351648 },
16361649 }
16371650 hasher.final(&hashed_file.hash);
1651 hashed_file.size = size;
16381652}
16391653
16401654fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
......@@ -1662,6 +1676,7 @@ const HashedFile = struct {
16621676 fs_path: []const u8,
16631677 normalized_path: []const u8,
16641678 hash: Manifest.Digest,
1679 size: u64,
16651680 failure: Error!void,
16661681 kind: Kind,
16671682
src/Package/Manifest.zig+80
......@@ -6,6 +6,84 @@ pub const multihash_len = 1 + 1 + Hash.digest_length;
66pub const multihash_hex_digest_len = 2 * multihash_len;
77pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
88
9pub const NewHashDecoded = extern struct {
10 name: [16]u8,
11 semver: [16]u8,
12 size: u32 align(1),
13 hash: [5]u8,
14
15 pub fn init(name: []const u8, semver: std.SemanticVersion, size: u32, hash: Digest) NewHashDecoded {
16 var result: NewHashDecoded = undefined;
17
18 const name_len = @min(name.len, result.name.len);
19 @memcpy(result.name[0..name_len], name[0..name_len]);
20 @memset(result.name[name_len..], 0);
21
22 if (std.fmt.bufPrint(&result.semver, "{}", .{semver})) |slice| {
23 @memset(result.semver[slice.len..], 0);
24 } else |err| switch (err) {
25 error.NoSpaceLeft => {
26 result.semver[result.semver.len - 1] = '-';
27 },
28 }
29
30 result.size = size;
31
32 @memcpy(&result.hash, hash[0..result.hash.len]);
33
34 return result;
35 }
36
37 pub fn getName(h: *const NewHashDecoded) []const u8 {
38 const len = mem.indexOfScalar(u8, &h.name, 0) orelse h.name.len;
39 return h.name[0..len];
40 }
41
42 pub fn getSemVer(h: *const NewHashDecoded) []const u8 {
43 const len = mem.indexOfScalar(u8, &h.semver, 0) orelse h.semver.len;
44 return h.semver[0..len];
45 }
46
47 pub fn encode(h: NewHashDecoded) NewHashEncoded {
48 var result: NewHashEncoded = .{ .bytes = undefined };
49 var i: usize = 0;
50
51 const name = h.getName();
52 @memcpy(result.bytes[i..][0..name.len], name);
53 i += name.len;
54
55 result.bytes[i] = '-';
56 i += 1;
57
58 const semver = h.getSemVer();
59 @memcpy(result.bytes[i..][0..semver.len], semver);
60 i += semver.len;
61
62 result.bytes[i] = '-';
63 i += 1;
64
65 const hash_len = 5; // TODO `h.hash.len`
66 var decoded_bytes: [hash_len + @sizeOf(u32)]u8 = undefined;
67 std.mem.writeInt(u32, decoded_bytes[0..@sizeOf(u32)], h.size, .little);
68 @memcpy(decoded_bytes[@sizeOf(u32)..], &h.hash);
69
70 i += (std.fs.base64_encoder.encode(result.bytes[i..], &decoded_bytes)).len;
71
72 @memset(result.bytes[i..], 0);
73 return result;
74 }
75};
76
77pub const new_hash_encoded_max_len = std.fs.base64_encoder.calcSize(@sizeOf(NewHashDecoded));
78pub const NewHashEncoded = extern struct {
79 bytes: [new_hash_encoded_max_len]u8,
80
81 pub fn toSlice(h: *const NewHashEncoded) []const u8 {
82 const len = mem.indexOfScalar(u8, &h.bytes, 0) orelse h.bytes.len;
83 return h.bytes[0..len];
84 }
85};
86
987pub const Dependency = struct {
1088 location: Location,
1189 location_tok: Ast.TokenIndex,
......@@ -591,12 +669,14 @@ const Parse = struct {
591669};
592670
593671const Manifest = @This();
672const builtin = @import("builtin");
594673const std = @import("std");
595674const mem = std.mem;
596675const Allocator = std.mem.Allocator;
597676const assert = std.debug.assert;
598677const Ast = std.zig.Ast;
599678const testing = std.testing;
679const native_endian = builtin.cpu.arch.endian();
600680
601681test "basic" {
602682 const gpa = testing.allocator;