authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-04-20 16:35:07-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-04-20 18:45:16-07:00
log3a9b16db446379d6fd22bd614e6741e1142b456a
treec3e48977ee030d03fcd4c6574115eec03f6fed8e
parentf887bea4d34da6e4dcfdf1eb18cca6afca3dc546

zig fetch: detect global vs local mode via --save

Makes `zig fetch` only fetch globally, just like it used to. However, if `--save` (or any variant) is used, then it also fetches locally. When fetching by path, the hash is always computed, recompressed tarball is always created, always overwrites any existing global cache entry. closes #31818 closes #31866 (only requires build.zig file when --save is passed)

2 files changed, 114 insertions(+), 104 deletions(-)

src/Package/Fetch.zig+70-68
......@@ -104,6 +104,12 @@ pub const LazyStatus = enum {
104104 unavailable,
105105};
106106
107pub const LocalStorage = struct {
108 cache_root: Cache.Path,
109 /// Path to "zig-pkg" inside the package in which the user ran `zig build`.
110 pkg_root: Cache.Path,
111};
112
107113/// Contains shared state among all `Fetch` tasks.
108114pub const JobQueue = struct {
109115 io: Io,
......@@ -122,9 +128,8 @@ pub const JobQueue = struct {
122128 /// This tracks `Fetch` tasks as well as recompression tasks.
123129 group: Io.Group = .init,
124130 global_cache: Cache.Directory,
125 local_cache: Cache.Path,
126 /// Path to "zig-pkg" inside the package in which the user ran `zig build`.
127 root_pkg_path: Cache.Path,
131 /// If `null`, indicates fetch globally only.
132 local_storage: ?*const LocalStorage,
128133 /// If true then, no fetching occurs, and:
129134 /// * The `global_cache` directory is assumed to be the direct parent
130135 /// directory of on-disk packages rather than having the "p/" directory
......@@ -341,7 +346,7 @@ pub const JobQueue = struct {
341346 );
342347 }
343348
344 fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {
349 fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Cache.Path) Io.Cancelable!void {
345350 const pkg_hash_slice = package_hash.toSlice();
346351
347352 const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
......@@ -359,7 +364,7 @@ pub const JobQueue = struct {
359364 defer arena_instance.deinit();
360365 const arena = arena_instance.allocator();
361366
362 recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) {
367 recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) {
363368 error.Canceled => |e| return e,
364369 error.ReadFailed => comptime unreachable,
365370 error.WriteFailed => comptime unreachable,
......@@ -372,6 +377,7 @@ pub const JobQueue = struct {
372377 arena: Allocator,
373378 dest_path: Cache.Path,
374379 pkg_hash_slice: []const u8,
380 package_root: Cache.Path,
375381 prog_node: std.Progress.Node,
376382 ) !void {
377383 const gpa = jq.http_client.allocator;
......@@ -386,7 +392,7 @@ pub const JobQueue = struct {
386392 var scanned_files: std.ArrayList(ScannedFile) = .empty;
387393 defer scanned_files.deinit(gpa);
388394
389 var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true });
395 var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
390396 defer pkg_dir.close(io);
391397
392398 {
......@@ -513,7 +519,6 @@ pub fn run(f: *Fetch) RunError!void {
513519 const eb = &f.error_bundle;
514520 const arena = f.arena.allocator();
515521 const gpa = f.arena.child_allocator;
516 const local_cache_root = job_queue.local_cache;
517522
518523 try eb.init(gpa);
519524
......@@ -534,32 +539,16 @@ pub fn run(f: *Fetch) RunError!void {
534539 );
535540 // Packages fetched by URL may not use relative paths to escape outside the
536541 // fetched package directory from within the package cache.
537 if (pkg_root.root_dir.eql(local_cache_root.root_dir)) {
538 // `parent_package_root.sub_path` contains a path like this:
539 // "p/$hash", or
540 // "p/$hash/foo", with possibly more directories after "foo".
541 // We want to fail unless the resolved relative path has a
542 // prefix of "p/$hash/".
543 const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len;
544 const parent_sub_path = f.parent_package_root.sub_path;
545 const end = find_end: {
546 if (parent_sub_path.len > prefix_len) {
547 // Use `isSep` instead of `indexOfScalarPos` to account for
548 // Windows accepting both `\` and `/` as path separators.
549 for (parent_sub_path[prefix_len..], prefix_len..) |c, i| {
550 if (std.fs.path.isSep(c)) break :find_end i;
551 }
552 }
553 break :find_end parent_sub_path.len;
554 };
555 const expected_prefix = parent_sub_path[0..end];
556 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
557 return f.fail(
558 f.location_tok,
559 try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
560 );
561 }
562 }
542
543 // This code path is only reachable recursively and the sub_path
544 // will already have been resolved to no longer have extra ".." or
545 // "." components.
546 assert(job_queue.local_storage != null);
547 assert(pkg_root.root_dir.eql(f.parent_package_root.root_dir));
548 if (!std.mem.startsWith(u8, pkg_root.sub_path, f.parent_package_root.sub_path)) return f.fail(
549 f.location_tok,
550 try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
551 );
563552 f.package_root = pkg_root;
564553 try loadManifest(f, pkg_root);
565554 if (!f.has_build_zig) try checkBuildFileExistence(f);
......@@ -610,31 +599,33 @@ pub fn run(f: *Fetch) RunError!void {
610599 return queueJobsForDeps(f);
611600 }
612601
613 const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice());
614 if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
615 assert(f.lazy_status != .unavailable);
616 f.package_root = package_root;
617 try loadManifest(f, f.package_root);
618 try checkBuildFileExistence(f);
619 if (!job_queue.recursive) return;
620 return queueJobsForDeps(f);
621 } else |err| switch (err) {
622 error.FileNotFound => {
623 log.debug("FileNotFound: {f}", .{package_root});
624 if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
625 f.name_tok,
626 try eb.printString("package not found at '{f}'", .{package_root}),
627 );
628 },
629 error.Canceled => |e| return e,
630 else => |e| {
631 try eb.addRootErrorMessage(.{
632 .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
633 package_root, e,
634 }),
635 });
636 return error.FetchFailed;
637 },
602 if (job_queue.local_storage) |ls| {
603 const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
604 if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
605 assert(f.lazy_status != .unavailable);
606 f.package_root = package_root;
607 try loadManifest(f, f.package_root);
608 try checkBuildFileExistence(f);
609 if (!job_queue.recursive) return;
610 return queueJobsForDeps(f);
611 } else |err| switch (err) {
612 error.FileNotFound => {
613 log.debug("FileNotFound: {f}", .{package_root});
614 if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
615 f.name_tok,
616 try eb.printString("package not found at '{f}'", .{package_root}),
617 );
618 },
619 error.Canceled => |e| return e,
620 else => |e| {
621 try eb.addRootErrorMessage(.{
622 .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
623 package_root, e,
624 }),
625 });
626 return error.FetchFailed;
627 },
628 }
638629 }
639630
640631 // Check global cache before remote fetch.
......@@ -713,7 +704,14 @@ fn runResource(
713704 break :r x;
714705 };
715706 const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
716 const tmp_directory_path = try job_queue.root_pkg_path.join(arena, tmp_dir_sub_path);
707 const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
708 const tmp_directory_path: Cache.Path = if (job_queue.local_storage) |ls|
709 try ls.pkg_root.join(arena, tmp_dir_sub_path)
710 else
711 .{
712 .root_dir = job_queue.global_cache,
713 .sub_path = tmp_tmp_dir_sub_path,
714 };
717715
718716 const package_sub_path = blk: {
719717 var tmp_directory: Cache.Directory = .{
......@@ -772,19 +770,23 @@ fn runResource(
772770 // zig package directory untouched as it may be in use. This is done even
773771 // if the hash is invalid, in case the package with the different hash is
774772 // used in the future.
775 f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice());
776 renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
777 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
778 "unable to rename temporary directory {f} into package cache directory {f}: {t}",
779 .{ package_sub_path, f.package_root, err },
780 ) });
781 return error.FetchFailed;
782 };
773 if (job_queue.local_storage) |ls| {
774 f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
775 renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
776 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
777 "unable to rename temporary directory {f} into package cache directory {f}: {t}",
778 .{ package_sub_path, f.package_root, err },
779 ) });
780 return error.FetchFailed;
781 };
782 } else {
783 f.package_root = tmp_directory_path;
784 }
783785
784786 if (!disable_recompress) {
785787 // Spin off a task to recompress the tarball, with filtered files deleted, into
786788 // the global cache.
787 job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash });
789 job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
788790 }
789791
790792 // Remove temporary directory root if not already renamed to global cache.
src/main.zig+44-36
......@@ -5331,9 +5331,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
53315331 .parent = root_mod,
53325332 });
53335333
5334 var cleanup_build_dir: ?Io.Dir = null;
5335 defer if (cleanup_build_dir) |*dir| dir.close(io);
5336
53375334 if (dev.env.supports(.fetch_command)) {
53385335 const fetch_prog_node = root_prog_node.start("Fetch Packages", 0);
53395336 defer fetch_prog_node.end();
......@@ -5345,36 +5342,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
53455342 .io = io,
53465343 .http_client = &http_client,
53475344 .global_cache = dirs.global_cache,
5348 .local_cache = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5349 .root_pkg_path = if (override_pkg_dir) |cwd_rel_path| .initCwd(cwd_rel_path) else .{
5350 .root_dir = build_root.directory,
5351 .sub_path = "zig-pkg",
5345 .local_storage = &.{
5346 .cache_root = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5347 .pkg_root = if (override_pkg_dir) |p|
5348 .initCwd(p)
5349 else if (system_pkg_dir_path) |p|
5350 .initCwd(p)
5351 else
5352 .{
5353 .root_dir = build_root.directory,
5354 .sub_path = "zig-pkg",
5355 },
53525356 },
5353 .read_only = false,
53545357 .recursive = true,
53555358 .debug_hash = false,
53565359 .unlazy_set = unlazy_set,
53575360 .fork_set = fork_set,
53585361 .mode = fetch_mode,
53595362 .prog_node = fetch_prog_node,
5363 .read_only = system_pkg_dir_path != null,
53605364 };
53615365 defer job_queue.deinit();
53625366
5363 if (system_pkg_dir_path) |p| {
5364 const system_pkg_path: Path = .{
5365 .root_dir = .{
5366 .path = p,
5367 .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {
5368 fatal("unable to open system package directory '{s}': {t}", .{ p, err });
5369 },
5370 },
5371 .sub_path = "",
5372 };
5373 job_queue.global_cache = system_pkg_path.root_dir;
5374 job_queue.root_pkg_path = system_pkg_path;
5375 job_queue.read_only = true;
5376 cleanup_build_dir = job_queue.global_cache.handle;
5377 } else {
5367 if (system_pkg_dir_path == null) {
53785368 try http_client.initDefaultProxies(arena, environ_map);
53795369 }
53805370
......@@ -7041,7 +7031,8 @@ const usage_fetch =
70417031 \\Options:
70427032 \\ -h, --help Print this help and exit
70437033 \\ --global-cache-dir [path] Override path to global Zig cache directory
7044 \\ --pkg-dir [path] Override path to package directory
7034 \\ --cache-dir [path] Override path to local cache directory
7035 \\ --pkg-dir [path] Override path to local package directory
70457036 \\ --debug-hash Print verbose hash information to stdout
70467037 \\ --save Add the fetched package to build.zig.zon
70477038 \\ --save=[name] Add the fetched package to build.zig.zon as name
......@@ -7062,6 +7053,7 @@ fn cmdFetch(
70627053 const color: Color = .auto;
70637054 var opt_path_or_url: ?[]const u8 = null;
70647055 var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
7056 var override_local_cache_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_CACHE_DIR.get(environ_map);
70657057 var override_pkg_dir: ?[]const u8 = EnvVar.ZIG_LOCAL_PKG_DIR.get(environ_map);
70667058 var debug_hash: bool = false;
70677059 var save: union(enum) {
......@@ -7082,6 +7074,10 @@ fn cmdFetch(
70827074 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
70837075 i += 1;
70847076 override_global_cache_dir = args[i];
7077 } else if (mem.eql(u8, arg, "--cache-dir")) {
7078 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
7079 i += 1;
7080 override_local_cache_dir = args[i];
70857081 } else if (mem.eql(u8, arg, "--pkg-dir")) {
70867082 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
70877083 i += 1;
......@@ -7128,27 +7124,39 @@ fn cmdFetch(
71287124 };
71297125 defer global_cache_directory.handle.close(io);
71307126
7127 var local_storage: Package.Fetch.LocalStorage = undefined;
7128 var build_root: BuildRoot = undefined;
7129 var build_root_initialized = false;
7130 defer if (build_root_initialized) build_root.deinit(io);
7131
71317132 const cwd_path = try introspect.getResolvedCwd(io, arena);
71327133
7133 var build_root = try findBuildRoot(arena, io, .{
7134 .cwd_path = cwd_path,
7135 });
7136 defer build_root.deinit(io);
7134 const local_storage_ptr = switch (save) {
7135 .no => null,
7136 .yes, .exact => ls: {
7137 build_root = try findBuildRoot(arena, io, .{ .cwd_path = cwd_path });
7138 build_root_initialized = true;
7139
7140 local_storage = .{
7141 .cache_root = if (override_local_cache_dir) |p| .initCwd(p) else .{
7142 .root_dir = build_root.directory,
7143 .sub_path = ".zig-cache",
7144 },
7145 .pkg_root = if (override_pkg_dir) |p| .initCwd(p) else .{
7146 .root_dir = build_root.directory,
7147 .sub_path = "zig-pkg",
7148 },
7149 };
71377150
7138 const local_cache_path: Path = .{
7139 .root_dir = build_root.directory,
7140 .sub_path = ".zig-cache",
7151 break :ls &local_storage;
7152 },
71417153 };
71427154
71437155 var job_queue: Package.Fetch.JobQueue = .{
71447156 .io = io,
71457157 .http_client = &http_client,
71467158 .global_cache = global_cache_directory,
7147 .local_cache = local_cache_path,
7148 .root_pkg_path = if (override_pkg_dir) |cwd_rel_path| .initCwd(cwd_rel_path) else .{
7149 .root_dir = build_root.directory,
7150 .sub_path = "zig-pkg",
7151 },
7159 .local_storage = local_storage_ptr,
71527160 .recursive = false,
71537161 .read_only = false,
71547162 .debug_hash = debug_hash,