authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-01 23:05:01-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-02 17:02:25-07:00
logef9966c9855dd855afda767f212abec6e5a36307
tree4f23d4468f3d9d44c16868becad6b0c6bb9083d8
parent309c53295f26999065e4dc76cef4d90f8d85fb38

introduce the 'zig fetch' command + symlink support

zig fetch [options] <url> zig fetch [options] <path> Fetches a package which is found at <url> or <path> into the global cache directory, printing the package hash to stdout. Closes #16972 Related to #14280 Additionally, this commit: * Adds uncompressed .tar support to package fetching * Introduces symlink support to package fetching

4 files changed, 302 insertions(+), 107 deletions(-)

lib/std/tar.zig+1-1
......@@ -210,7 +210,7 @@ pub fn pipeToFileSystem(dir: std.fs.Dir, reader: anytype, options: Options) !voi
210210 while (true) {
211211 const temp = try buffer.readChunk(reader, @intCast(rounded_file_size + 512 - file_off));
212212 if (temp.len == 0) return error.UnexpectedEndOfStream;
213 const slice = temp[0..@as(usize, @intCast(@min(file_size - file_off, temp.len)))];
213 const slice = temp[0..@intCast(@min(file_size - file_off, temp.len))];
214214 try file.writeAll(slice);
215215
216216 file_off += slice.len;
src/Package.zig+141-95
......@@ -15,10 +15,10 @@ const Compilation = @import("Compilation.zig");
1515const Module = @import("Module.zig");
1616const Cache = std.Build.Cache;
1717const build_options = @import("build_options");
18const Manifest = @import("Manifest.zig");
1918const git = @import("git.zig");
2019const computePackageHash = @import("Package/hash.zig").compute;
2120
21pub const Manifest = @import("Manifest.zig");
2222pub const Table = std.StringHashMapUnmanaged(*Package);
2323
2424root_src_directory: Compilation.Directory,
......@@ -454,8 +454,8 @@ pub fn createFilePkg(
454454 return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
455455}
456456
457const Report = struct {
458 ast: *const std.zig.Ast,
457pub const Report = struct {
458 ast: ?*const std.zig.Ast,
459459 directory: Compilation.Directory,
460460 error_bundle: *std.zig.ErrorBundle.Wip,
461461
......@@ -465,6 +465,7 @@ const Report = struct {
465465 comptime fmt_string: []const u8,
466466 fmt_args: anytype,
467467 ) error{ PackageFetchFailed, OutOfMemory } {
468 const ast = report.ast orelse main.fatal(fmt_string, fmt_args);
468469 const gpa = report.error_bundle.gpa;
469470
470471 const file_path = try report.directory.join(gpa, &.{Manifest.basename});
......@@ -473,7 +474,7 @@ const Report = struct {
473474 const msg = try std.fmt.allocPrint(gpa, fmt_string, fmt_args);
474475 defer gpa.free(msg);
475476
476 try addErrorMessage(report.ast.*, file_path, report.error_bundle, 0, .{
477 try addErrorMessage(ast.*, file_path, report.error_bundle, 0, .{
477478 .tok = tok,
478479 .off = 0,
479480 .msg = msg,
......@@ -482,6 +483,18 @@ const Report = struct {
482483 return error.PackageFetchFailed;
483484 }
484485
486 fn addErrorWithNotes(
487 report: Report,
488 notes_len: u32,
489 msg: Manifest.ErrorMessage,
490 ) error{OutOfMemory}!void {
491 const ast = report.ast orelse main.fatal("{s}", .{msg.msg});
492 const gpa = report.error_bundle.gpa;
493 const file_path = try report.directory.join(gpa, &.{Manifest.basename});
494 defer gpa.free(file_path);
495 return addErrorMessage(ast.*, file_path, report.error_bundle, notes_len, msg);
496 }
497
485498 fn addErrorMessage(
486499 ast: std.zig.Ast,
487500 file_path: []const u8,
......@@ -508,7 +521,7 @@ const Report = struct {
508521 }
509522};
510523
511const FetchLocation = union(enum) {
524pub const FetchLocation = union(enum) {
512525 /// The relative path to a file or directory.
513526 /// This may be a file that requires unpacking (such as a .tar.gz),
514527 /// or the path to the root directory of a package.
......@@ -517,30 +530,27 @@ const FetchLocation = union(enum) {
517530 http_request: std.Uri,
518531 git_request: std.Uri,
519532
520 pub fn init(gpa: Allocator, dep: Manifest.Dependency, root_dir: Compilation.Directory, report: Report) !FetchLocation {
533 pub fn init(
534 gpa: Allocator,
535 dep: Manifest.Dependency,
536 root_dir: Compilation.Directory,
537 report: Report,
538 ) !FetchLocation {
521539 switch (dep.location) {
522540 .url => |url| {
523541 const uri = std.Uri.parse(url) catch |err| switch (err) {
524542 error.UnexpectedCharacter => return report.fail(dep.location_tok, "failed to parse dependency location as URI", .{}),
525543 else => return err,
526544 };
527 if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
528 return report.fail(dep.location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
529 } else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
530 return .{ .http_request = uri };
531 } else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
532 return .{ .git_request = uri };
533 } else {
534 return report.fail(dep.location_tok, "Unsupported URL scheme: {s}", .{uri.scheme});
535 }
545 return initUri(uri, dep.location_tok, report);
536546 },
537547 .path => |path| {
538548 if (fs.path.isAbsolute(path)) {
539 return report.fail(dep.location_tok, "Absolute paths are not allowed. Use a relative path instead", .{});
549 return report.fail(dep.location_tok, "absolute paths are not allowed. Use a relative path instead", .{});
540550 }
541551
542552 const is_dir = isDirectory(root_dir, path) catch |err| switch (err) {
543 error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{path}),
553 error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{path}),
544554 else => return err,
545555 };
546556
......@@ -552,9 +562,21 @@ const FetchLocation = union(enum) {
552562 }
553563 }
554564
565 pub fn initUri(uri: std.Uri, location_tok: std.zig.Ast.TokenIndex, report: Report) !FetchLocation {
566 if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
567 return report.fail(location_tok, "'file' scheme is not allowed for URLs. Use '.path' instead", .{});
568 } else if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) {
569 return .{ .http_request = uri };
570 } else if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or ascii.eqlIgnoreCase(uri.scheme, "git+https")) {
571 return .{ .git_request = uri };
572 } else {
573 return report.fail(location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
574 }
575 }
576
555577 pub fn deinit(f: *FetchLocation, gpa: Allocator) void {
556578 switch (f.*) {
557 inline .file, .directory => |path| gpa.free(path),
579 .file, .directory => |path| gpa.free(path),
558580 .http_request, .git_request => {},
559581 }
560582 f.* = undefined;
......@@ -565,7 +587,7 @@ const FetchLocation = union(enum) {
565587 gpa: Allocator,
566588 root_dir: Compilation.Directory,
567589 http_client: *std.http.Client,
568 dep: Manifest.Dependency,
590 dep_location_tok: std.zig.Ast.TokenIndex,
569591 report: Report,
570592 ) !ReadableResource {
571593 switch (f) {
......@@ -588,7 +610,7 @@ const FetchLocation = union(enum) {
588610 try req.wait();
589611
590612 if (req.response.status != .ok) {
591 return report.fail(dep.location_tok, "Expected response status '200 OK' got '{} {s}'", .{
613 return report.fail(dep_location_tok, "expected response status '200 OK' got '{} {s}'", .{
592614 @intFromEnum(req.response.status),
593615 req.response.status.phrase() orelse "",
594616 });
......@@ -607,7 +629,7 @@ const FetchLocation = union(enum) {
607629 session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) {
608630 error.Redirected => {
609631 defer gpa.free(redirect_uri);
610 return report.fail(dep.location_tok, "Repository moved to {s}", .{redirect_uri});
632 return report.fail(dep_location_tok, "repository moved to {s}", .{redirect_uri});
611633 },
612634 else => |other| return other,
613635 };
......@@ -634,19 +656,16 @@ const FetchLocation = union(enum) {
634656 break :want_oid ref.peeled orelse ref.oid;
635657 }
636658 }
637 return report.fail(dep.location_tok, "Ref not found: {s}", .{want_ref});
659 return report.fail(dep_location_tok, "ref not found: {s}", .{want_ref});
638660 };
639661 if (uri.fragment == null) {
640 const file_path = try report.directory.join(gpa, &.{Manifest.basename});
641 defer gpa.free(file_path);
642
643 const eb = report.error_bundle;
644662 const notes_len = 1;
645 try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
646 .tok = dep.location_tok,
663 try report.addErrorWithNotes(notes_len, .{
664 .tok = dep_location_tok,
647665 .off = 0,
648666 .msg = "url field is missing an explicit ref",
649667 });
668 const eb = report.error_bundle;
650669 const notes_start = try eb.reserveNotes(notes_len);
651670 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
652671 .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid) }),
......@@ -669,12 +688,13 @@ const FetchLocation = union(enum) {
669688 }
670689};
671690
672const ReadableResource = struct {
691pub const ReadableResource = struct {
673692 path: []const u8,
674693 resource: union(enum) {
675694 file: fs.File,
676695 http_request: std.http.Client.Request,
677696 git_fetch_stream: git.Session.FetchStream,
697 dir: fs.IterableDir,
678698 },
679699
680700 /// Unpack the package into the global cache directory.
......@@ -685,12 +705,12 @@ const ReadableResource = struct {
685705 allocator: Allocator,
686706 thread_pool: *ThreadPool,
687707 global_cache_directory: Compilation.Directory,
688 dep: Manifest.Dependency,
708 dep_location_tok: std.zig.Ast.TokenIndex,
689709 report: Report,
690710 pkg_prog_node: *std.Progress.Node,
691711 ) !PackageLocation {
692712 switch (rr.resource) {
693 inline .file, .http_request, .git_fetch_stream => |*r| {
713 inline .file, .http_request, .git_fetch_stream, .dir => |*r, tag| {
694714 const s = fs.path.sep_str;
695715 const rand_int = std.crypto.random.int(u64);
696716 const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
......@@ -710,45 +730,58 @@ const ReadableResource = struct {
710730 };
711731 defer tmp_directory.closeAndFree(allocator);
712732
713 const opt_content_length = try rr.getSize();
714
715 var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
716 .child_reader = r.reader(),
717 .prog_node = pkg_prog_node,
718 .unit = if (opt_content_length) |content_length| unit: {
719 const kib = content_length / 1024;
720 const mib = kib / 1024;
721 if (mib > 0) {
722 pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
723 pkg_prog_node.setUnit("MiB");
724 break :unit .mib;
725 } else {
726 pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
727 pkg_prog_node.setUnit("KiB");
728 break :unit .kib;
733 if (tag != .dir) {
734 const opt_content_length = try rr.getSize();
735
736 var prog_reader: ProgressReader(@TypeOf(r.reader())) = .{
737 .child_reader = r.reader(),
738 .prog_node = pkg_prog_node,
739 .unit = if (opt_content_length) |content_length| unit: {
740 const kib = content_length / 1024;
741 const mib = kib / 1024;
742 if (mib > 0) {
743 pkg_prog_node.setEstimatedTotalItems(@intCast(mib));
744 pkg_prog_node.setUnit("MiB");
745 break :unit .mib;
746 } else {
747 pkg_prog_node.setEstimatedTotalItems(@intCast(@max(1, kib)));
748 pkg_prog_node.setUnit("KiB");
749 break :unit .kib;
750 }
751 } else .any,
752 };
753
754 switch (try rr.getFileType(dep_location_tok, report)) {
755 .tar => try unpackTarball(prog_reader.reader(), tmp_directory.handle),
756 .@"tar.gz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, std.compress.gzip),
757 .@"tar.xz" => try unpackTarballCompressed(allocator, prog_reader, tmp_directory.handle, std.compress.xz),
758 .git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle),
759 }
760 } else {
761 // Recursive directory copy.
762 var it = try r.walk(allocator);
763 defer it.deinit();
764 while (try it.next()) |entry| {
765 switch (entry.kind) {
766 .directory => try tmp_directory.handle.makePath(entry.path),
767 .file => try r.dir.copyFile(
768 entry.path,
769 tmp_directory.handle,
770 entry.path,
771 .{},
772 ),
773 .sym_link => {
774 var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
775 const link_name = try r.dir.readLink(entry.path, &buf);
776 // TODO: if this would create a symlink to outside
777 // the destination directory, fail with an error instead.
778 try tmp_directory.handle.symLink(link_name, entry.path, .{});
779 },
780 else => return error.IllegalFileTypeInPackage,
729781 }
730 } else .any,
731 };
732 pkg_prog_node.context.refresh();
733
734 switch (try rr.getFileType(dep, report)) {
735 .@"tar.gz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.gzip),
736 // I have not checked what buffer sizes the xz decompression implementation uses
737 // by default, so the same logic applies for buffering the reader as for gzip.
738 .@"tar.xz" => try unpackTarball(allocator, prog_reader, tmp_directory.handle, std.compress.xz),
739 .git_pack => try unpackGitPack(allocator, &prog_reader, git.parseOid(rr.path) catch unreachable, tmp_directory.handle),
782 }
740783 }
741784
742 // Unpack completed - stop showing amount as progress
743 pkg_prog_node.setEstimatedTotalItems(0);
744 pkg_prog_node.setCompletedItems(0);
745 pkg_prog_node.context.refresh();
746
747 // TODO: delete files not included in the package prior to computing the package hash.
748 // for example, if the ini file has directives to include/not include certain files,
749 // apply those rules directly to the filesystem right here. This ensures that files
750 // not protected by the hash are not present on the file system.
751
752785 break :h try computePackageHash(thread_pool, .{ .dir = tmp_directory.handle });
753786 };
754787
......@@ -769,6 +802,7 @@ const ReadableResource = struct {
769802 }
770803
771804 const FileType = enum {
805 tar,
772806 @"tar.gz",
773807 @"tar.xz",
774808 git_pack,
......@@ -780,21 +814,28 @@ const ReadableResource = struct {
780814 // TODO: Handle case of chunked content-length
781815 .http_request => |req| return req.response.content_length,
782816 .git_fetch_stream => |stream| return stream.request.response.content_length,
817 .dir => unreachable,
783818 }
784819 }
785820
786 pub fn getFileType(rr: ReadableResource, dep: Manifest.Dependency, report: Report) !FileType {
821 pub fn getFileType(
822 rr: ReadableResource,
823 dep_location_tok: std.zig.Ast.TokenIndex,
824 report: Report,
825 ) !FileType {
787826 switch (rr.resource) {
788827 .file => {
789828 return fileTypeFromPath(rr.path) orelse
790 return report.fail(dep.location_tok, "Unknown file type", .{});
829 return report.fail(dep_location_tok, "unknown file type", .{});
791830 },
792831 .http_request => |req| {
793832 const content_type = req.response.headers.getFirstValue("Content-Type") orelse
794 return report.fail(dep.location_tok, "Missing 'Content-Type' header", .{});
833 return report.fail(dep_location_tok, "missing 'Content-Type' header", .{});
795834
796835 // If the response has a different content type than the URI indicates, override
797836 // the previously assumed file type.
837 if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) return .tar;
838
798839 return if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
799840 ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
800841 ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
......@@ -805,22 +846,21 @@ const ReadableResource = struct {
805846 // support gitlab tarball urls such as https://gitlab.com/<namespace>/<project>/-/archive/<sha>/<project>-<sha>.tar.gz
806847 // whose content-disposition header is: 'attachment; filename="<project>-<sha>.tar.gz"'
807848 const content_disposition = req.response.headers.getFirstValue("Content-Disposition") orelse
808 return report.fail(dep.location_tok, "Missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
849 return report.fail(dep_location_tok, "missing 'Content-Disposition' header for Content-Type=application/octet-stream", .{});
809850 break :ty getAttachmentType(content_disposition) orelse
810 return report.fail(dep.location_tok, "Unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
811 } else return report.fail(dep.location_tok, "Unrecognized value for 'Content-Type' header: {s}", .{content_type});
851 return report.fail(dep_location_tok, "unsupported 'Content-Disposition' header value: '{s}' for Content-Type=application/octet-stream", .{content_disposition});
852 } else return report.fail(dep_location_tok, "unrecognized value for 'Content-Type' header: {s}", .{content_type});
812853 },
813854 .git_fetch_stream => return .git_pack,
855 .dir => unreachable,
814856 }
815857 }
816858
817859 fn fileTypeFromPath(file_path: []const u8) ?FileType {
818 return if (ascii.endsWithIgnoreCase(file_path, ".tar.gz"))
819 .@"tar.gz"
820 else if (ascii.endsWithIgnoreCase(file_path, ".tar.xz"))
821 .@"tar.xz"
822 else
823 null;
860 if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
861 if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
862 if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
863 return null;
824864 }
825865
826866 fn getAttachmentType(content_disposition: []const u8) ?FileType {
......@@ -847,6 +887,7 @@ const ReadableResource = struct {
847887 .file => |file| file.close(),
848888 .http_request => |*req| req.deinit(),
849889 .git_fetch_stream => |*stream| stream.deinit(),
890 .dir => |*dir| dir.close(),
850891 }
851892 rr.* = undefined;
852893 }
......@@ -908,7 +949,7 @@ fn ProgressReader(comptime ReaderType: type) type {
908949 }
909950 },
910951 }
911 self.prog_node.context.maybeRefresh();
952 self.prog_node.activate();
912953 return amt;
913954 }
914955
......@@ -993,7 +1034,7 @@ fn getDirectoryModule(
9931034 if (all_modules.get(hex_digest)) |mod| return .{ mod.?, true };
9941035
9951036 var pkg_dir = directory.handle.openDir(fetch_location.directory, .{}) catch |err| switch (err) {
996 error.FileNotFound => return report.fail(dep.location_tok, "File not found: {s}", .{fetch_location.directory}),
1037 error.FileNotFound => return report.fail(dep.location_tok, "file not found: {s}", .{fetch_location.directory}),
9971038 else => |e| return e,
9981039 };
9991040 defer pkg_dir.close();
......@@ -1032,12 +1073,18 @@ fn fetchAndUnpack(
10321073 var pkg_prog_node = root_prog_node.start(name_for_prog, 0);
10331074 defer pkg_prog_node.end();
10341075 pkg_prog_node.activate();
1035 pkg_prog_node.context.refresh();
10361076
1037 var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep, report);
1077 var readable_resource = try fetch_location.fetch(gpa, directory, http_client, dep.location_tok, report);
10381078 defer readable_resource.deinit(gpa);
10391079
1040 var package_location = try readable_resource.unpack(gpa, thread_pool, global_cache_directory, dep, report, &pkg_prog_node);
1080 var package_location = try readable_resource.unpack(
1081 gpa,
1082 thread_pool,
1083 global_cache_directory,
1084 dep.location_tok,
1085 report,
1086 &pkg_prog_node,
1087 );
10411088 defer package_location.deinit(gpa);
10421089
10431090 const actual_hex = Manifest.hexDigest(package_location.hash);
......@@ -1048,16 +1095,13 @@ fn fetchAndUnpack(
10481095 });
10491096 }
10501097 } else {
1051 const file_path = try report.directory.join(gpa, &.{Manifest.basename});
1052 defer gpa.free(file_path);
1053
1054 const eb = report.error_bundle;
10551098 const notes_len = 1;
1056 try Report.addErrorMessage(report.ast.*, file_path, eb, notes_len, .{
1099 try report.addErrorWithNotes(notes_len, .{
10571100 .tok = dep.location_tok,
10581101 .off = 0,
10591102 .msg = "dependency is missing hash field",
10601103 });
1104 const eb = report.error_bundle;
10611105 const notes_start = try eb.reserveNotes(notes_len);
10621106 eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
10631107 .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
......@@ -1080,18 +1124,22 @@ fn fetchAndUnpack(
10801124 return module;
10811125}
10821126
1083fn unpackTarball(
1127fn unpackTarballCompressed(
10841128 gpa: Allocator,
10851129 reader: anytype,
10861130 out_dir: fs.Dir,
1087 comptime compression: type,
1131 comptime Compression: type,
10881132) !void {
10891133 var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
10901134
1091 var decompress = try compression.decompress(gpa, br.reader());
1135 var decompress = try Compression.decompress(gpa, br.reader());
10921136 defer decompress.deinit();
10931137
1094 try std.tar.pipeToFileSystem(out_dir, decompress.reader(), .{
1138 return unpackTarball(decompress.reader(), out_dir);
1139}
1140
1141fn unpackTarball(reader: anytype, out_dir: fs.Dir) !void {
1142 try std.tar.pipeToFileSystem(out_dir, reader, .{
10951143 .strip_components = 1,
10961144 // TODO: we would like to set this to executable_bit_only, but two
10971145 // things need to happen before that:
......@@ -1126,7 +1174,6 @@ fn unpackGitPack(
11261174 var index_prog_node = reader.prog_node.start("Index pack", 0);
11271175 defer index_prog_node.end();
11281176 index_prog_node.activate();
1129 index_prog_node.context.refresh();
11301177 var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
11311178 try git.indexPack(gpa, pack_file, index_buffered_writer.writer());
11321179 try index_buffered_writer.flush();
......@@ -1137,7 +1184,6 @@ fn unpackGitPack(
11371184 var checkout_prog_node = reader.prog_node.start("Checkout", 0);
11381185 defer checkout_prog_node.end();
11391186 checkout_prog_node.activate();
1140 checkout_prog_node.context.refresh();
11411187 var repository = try git.Repository.init(gpa, pack_file, index_file);
11421188 defer repository.deinit();
11431189 try repository.checkout(out_dir, want_oid);
src/Package/hash.zig+33-11
......@@ -16,6 +16,11 @@ pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_
1616 defer arena_instance.deinit();
1717 const arena = arena_instance.allocator();
1818
19 // TODO: delete files not included in the package prior to computing the package hash.
20 // for example, if the ini file has directives to include/not include certain files,
21 // apply those rules directly to the filesystem right here. This ensures that files
22 // not protected by the hash are not present on the file system.
23
1924 // Collect all files, recursively, then sort.
2025 var all_files = std.ArrayList(*HashedFile).init(gpa);
2126 defer all_files.deinit();
......@@ -30,16 +35,18 @@ pub fn compute(thread_pool: *ThreadPool, pkg_dir: fs.IterableDir) ![Hash.digest_
3035 defer wait_group.wait();
3136
3237 while (try walker.next()) |entry| {
33 switch (entry.kind) {
38 const kind: HashedFile.Kind = switch (entry.kind) {
3439 .directory => continue,
35 .file => {},
40 .file => .file,
41 .sym_link => .sym_link,
3642 else => return error.IllegalFileTypeInPackage,
37 }
43 };
3844 const hashed_file = try arena.create(HashedFile);
3945 const fs_path = try arena.dupe(u8, entry.path);
4046 hashed_file.* = .{
4147 .fs_path = fs_path,
4248 .normalized_path = try normalizePath(arena, fs_path),
49 .kind = kind,
4350 .hash = undefined, // to be populated by the worker
4451 .failure = undefined, // to be populated by the worker
4552 };
......@@ -70,8 +77,15 @@ const HashedFile = struct {
7077 normalized_path: []const u8,
7178 hash: [Hash.digest_length]u8,
7279 failure: Error!void,
80 kind: Kind,
7381
74 const Error = fs.File.OpenError || fs.File.ReadError || fs.File.StatError;
82 const Error =
83 fs.File.OpenError ||
84 fs.File.ReadError ||
85 fs.File.StatError ||
86 fs.Dir.ReadLinkError;
87
88 const Kind = enum { file, sym_link };
7589
7690 fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
7791 _ = context;
......@@ -104,15 +118,23 @@ fn workerHashFile(dir: fs.Dir, hashed_file: *HashedFile, wg: *WaitGroup) void {
104118
105119fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
106120 var buf: [8000]u8 = undefined;
107 var file = try dir.openFile(hashed_file.fs_path, .{});
108 defer file.close();
109121 var hasher = Hash.init(.{});
110122 hasher.update(hashed_file.normalized_path);
111 hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
112 while (true) {
113 const bytes_read = try file.read(&buf);
114 if (bytes_read == 0) break;
115 hasher.update(buf[0..bytes_read]);
123 switch (hashed_file.kind) {
124 .file => {
125 var file = try dir.openFile(hashed_file.fs_path, .{});
126 defer file.close();
127 hasher.update(&.{ 0, @intFromBool(try isExecutable(file)) });
128 while (true) {
129 const bytes_read = try file.read(&buf);
130 if (bytes_read == 0) break;
131 hasher.update(buf[0..bytes_read]);
132 }
133 },
134 .sym_link => {
135 const link_name = try dir.readLink(hashed_file.fs_path, &buf);
136 hasher.update(link_name);
137 },
116138 }
117139 hasher.final(&hashed_file.hash);
118140}
src/main.zig+127
......@@ -84,6 +84,7 @@ const normal_usage =
8484 \\Commands:
8585 \\
8686 \\ build Build project from build.zig
87 \\ fetch Copy a package into global cache and print its hash
8788 \\ init-exe Initialize a `zig build` application in the cwd
8889 \\ init-lib Initialize a `zig build` library in the cwd
8990 \\
......@@ -303,6 +304,8 @@ pub fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
303304 return cmdFmt(gpa, arena, cmd_args);
304305 } else if (mem.eql(u8, cmd, "objcopy")) {
305306 return @import("objcopy.zig").cmdObjCopy(gpa, arena, cmd_args);
307 } else if (mem.eql(u8, cmd, "fetch")) {
308 return cmdFetch(gpa, arena, cmd_args);
306309 } else if (mem.eql(u8, cmd, "libc")) {
307310 return cmdLibC(gpa, cmd_args);
308311 } else if (mem.eql(u8, cmd, "init-exe")) {
......@@ -6589,3 +6592,127 @@ fn parseRcIncludes(arg: []const u8) Compilation.RcIncludes {
65896592 return std.meta.stringToEnum(Compilation.RcIncludes, arg) orelse
65906593 fatal("unsupported rc includes type: '{s}'", .{arg});
65916594}
6595
6596pub const usage_fetch =
6597 \\Usage: zig fetch [options] <url>
6598 \\Usage: zig fetch [options] <path>
6599 \\
6600 \\ Copy a package into the global cache and print its hash.
6601 \\
6602 \\Options:
6603 \\ -h, --help Print this help and exit
6604 \\ --global-cache-dir [path] Override path to global Zig cache directory
6605 \\
6606;
6607
6608fn cmdFetch(
6609 gpa: Allocator,
6610 arena: Allocator,
6611 args: []const []const u8,
6612) !void {
6613 var opt_url: ?[]const u8 = null;
6614 var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
6615
6616 {
6617 var i: usize = 0;
6618 while (i < args.len) : (i += 1) {
6619 const arg = args[i];
6620 if (mem.startsWith(u8, arg, "-")) {
6621 if (mem.eql(u8, arg, "-h") or mem.eql(u8, arg, "--help")) {
6622 const stdout = io.getStdOut().writer();
6623 try stdout.writeAll(usage_fetch);
6624 return cleanExit();
6625 } else if (mem.eql(u8, arg, "--global-cache-dir")) {
6626 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
6627 i += 1;
6628 override_global_cache_dir = args[i];
6629 continue;
6630 } else {
6631 fatal("unrecognized parameter: '{s}'", .{arg});
6632 }
6633 } else if (opt_url != null) {
6634 fatal("unexpected extra parameter: '{s}'", .{arg});
6635 } else {
6636 opt_url = arg;
6637 }
6638 }
6639 }
6640
6641 const url = opt_url orelse fatal("missing url or path parameter", .{});
6642
6643 var thread_pool: ThreadPool = undefined;
6644 try thread_pool.init(.{ .allocator = gpa });
6645 defer thread_pool.deinit();
6646
6647 var http_client: std.http.Client = .{ .allocator = gpa };
6648 defer http_client.deinit();
6649
6650 var progress: std.Progress = .{ .dont_print_on_dumb = true };
6651 const root_prog_node = progress.start("Fetch", 0);
6652 defer root_prog_node.end();
6653
6654 var report: Package.Report = .{
6655 .ast = null,
6656 .directory = undefined,
6657 .error_bundle = undefined,
6658 };
6659
6660 var global_cache_directory: Compilation.Directory = l: {
6661 const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
6662 break :l .{
6663 .handle = try fs.cwd().makeOpenPath(p, .{}),
6664 .path = p,
6665 };
6666 };
6667 defer global_cache_directory.handle.close();
6668
6669 var readable_resource: Package.ReadableResource = rr: {
6670 if (fs.cwd().openIterableDir(url, .{})) |dir| {
6671 break :rr .{
6672 .path = try gpa.dupe(u8, url),
6673 .resource = .{ .dir = dir },
6674 };
6675 } else |dir_err| {
6676 const file_err = if (dir_err == error.NotDir) e: {
6677 if (fs.cwd().openFile(url, .{})) |f| {
6678 break :rr .{
6679 .path = try gpa.dupe(u8, url),
6680 .resource = .{ .file = f },
6681 };
6682 } else |err| break :e err;
6683 } else dir_err;
6684
6685 const uri = std.Uri.parse(url) catch |uri_err| {
6686 fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
6687 url, @errorName(file_err), @errorName(uri_err),
6688 });
6689 };
6690 const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
6691 const cwd: Cache.Directory = .{
6692 .handle = fs.cwd(),
6693 .path = null,
6694 };
6695 break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
6696 }
6697 };
6698 defer readable_resource.deinit(gpa);
6699
6700 var package_location = try readable_resource.unpack(
6701 gpa,
6702 &thread_pool,
6703 global_cache_directory,
6704 0,
6705 report,
6706 root_prog_node,
6707 );
6708 defer package_location.deinit(gpa);
6709
6710 const hex_digest = Package.Manifest.hexDigest(package_location.hash);
6711
6712 progress.done = true;
6713 progress.refresh();
6714
6715 try io.getStdOut().writeAll(hex_digest ++ "\n");
6716
6717 return cleanExit();
6718}