authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-06 09:41:28+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2026-02-06 09:41:28+01:00
logd84a638e8b6ffeb95dfafef59e6305bd0e139d4e
tree15d475879933c9ccce5493222cd0b0b2ca261dba
parent076f7e5bd5389e159865d99e0e86edc905cffc42
parentd8171e8a2ee56e76bcd91f187d5ca5664b87bc83

Merge pull request 'fetch packages into project-local directory' (#31121) from project-local-deps into master

Reviewed-on: https://codeberg.org/ziglang/zig/pulls/31121

11 files changed, 360 insertions(+), 342 deletions(-)

lib/std/Io.zig+6
...@@ -1031,6 +1031,9 @@ pub const Group = struct {...@@ -1031,6 +1031,9 @@ pub const Group = struct {
1031 /// Once this function is called, there are resources associated with the1031 /// Once this function is called, there are resources associated with the
1032 /// group. To release those resources, `Group.await` or `Group.cancel` must1032 /// group. To release those resources, `Group.await` or `Group.cancel` must
1033 /// eventually be called.1033 /// eventually be called.
1034 ///
1035 /// If `error.Canceled` is returned from any operation this task performs,
1036 /// it is asserted that `function` returns `error.Canceled`.
1034 pub fn async(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void {1037 pub fn async(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) void {
1035 const Args = @TypeOf(args);1038 const Args = @TypeOf(args);
1036 const TypeErased = struct {1039 const TypeErased = struct {
...@@ -1050,6 +1053,9 @@ pub const Group = struct {...@@ -1050,6 +1053,9 @@ pub const Group = struct {
1050 /// Once this function is called, there are resources associated with the1053 /// Once this function is called, there are resources associated with the
1051 /// group. To release those resources, `Group.await` or `Group.cancel` must1054 /// group. To release those resources, `Group.await` or `Group.cancel` must
1052 /// eventually be called.1055 /// eventually be called.
1056 ///
1057 /// If `error.Canceled` is returned from any operation this task performs,
1058 /// it is asserted that `function` returns `error.Canceled`.
1053 pub fn concurrent(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) ConcurrentError!void {1059 pub fn concurrent(g: *Group, io: Io, function: anytype, args: std.meta.ArgsTuple(@TypeOf(function))) ConcurrentError!void {
1054 const Args = @TypeOf(args);1060 const Args = @TypeOf(args);
1055 const TypeErased = struct {1061 const TypeErased = struct {
lib/std/Io/Threaded.zig+35-45
...@@ -3191,29 +3191,24 @@ fn dirCreateDirPosix(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8, perm...@@ -3191,29 +3191,24 @@ fn dirCreateDirPosix(userdata: ?*anyopaque, dir: Dir, sub_path: []const u8, perm
3191 try syscall.checkCancel();3191 try syscall.checkCancel();
3192 continue;3192 continue;
3193 },3193 },
3194 else => |e| {3194 .ACCES => return syscall.fail(error.AccessDenied),
3195 syscall.finish();3195 .PERM => return syscall.fail(error.PermissionDenied),
3196 switch (e) {3196 .DQUOT => return syscall.fail(error.DiskQuota),
3197 .ACCES => return error.AccessDenied,3197 .EXIST => return syscall.fail(error.PathAlreadyExists),
3198 .BADF => |err| return errnoBug(err), // File descriptor used after closed.3198 .LOOP => return syscall.fail(error.SymLinkLoop),
3199 .PERM => return error.PermissionDenied,3199 .MLINK => return syscall.fail(error.LinkQuotaExceeded),
3200 .DQUOT => return error.DiskQuota,3200 .NAMETOOLONG => return syscall.fail(error.NameTooLong),
3201 .EXIST => return error.PathAlreadyExists,3201 .NOENT => return syscall.fail(error.FileNotFound),
3202 .FAULT => |err| return errnoBug(err),3202 .NOMEM => return syscall.fail(error.SystemResources),
3203 .LOOP => return error.SymLinkLoop,3203 .NOSPC => return syscall.fail(error.NoSpaceLeft),
3204 .MLINK => return error.LinkQuotaExceeded,3204 .NOTDIR => return syscall.fail(error.NotDir),
3205 .NAMETOOLONG => return error.NameTooLong,3205 .ROFS => return syscall.fail(error.ReadOnlyFileSystem),
3206 .NOENT => return error.FileNotFound,3206 // dragonfly: when dir_fd is unlinked from filesystem
3207 .NOMEM => return error.SystemResources,3207 .NOTCONN => return syscall.fail(error.FileNotFound),
3208 .NOSPC => return error.NoSpaceLeft,3208 .ILSEQ => return syscall.fail(error.BadPathName),
3209 .NOTDIR => return error.NotDir,3209 .BADF => |err| return syscall.errnoBug(err), // File descriptor used after closed.
3210 .ROFS => return error.ReadOnlyFileSystem,3210 .FAULT => |err| return syscall.errnoBug(err),
3211 // dragonfly: when dir_fd is unlinked from filesystem3211 else => |err| return syscall.unexpectedErrno(err),
3212 .NOTCONN => return error.FileNotFound,
3213 .ILSEQ => return error.BadPathName,
3214 else => |err| return posix.unexpectedErrno(err),
3215 }
3216 },
3217 }3212 }
3218 }3213 }
3219}3214}
...@@ -5261,28 +5256,23 @@ fn dirOpenDirPosix(...@@ -5261,28 +5256,23 @@ fn dirOpenDirPosix(
5261 try syscall.checkCancel();5256 try syscall.checkCancel();
5262 continue;5257 continue;
5263 },5258 },
5264 else => |e| {5259 .INVAL => return syscall.fail(error.BadPathName),
5265 syscall.finish();5260 .ACCES => return syscall.fail(error.AccessDenied),
5266 switch (e) {5261 .LOOP => return syscall.fail(error.SymLinkLoop),
5267 .FAULT => |err| return errnoBug(err),5262 .MFILE => return syscall.fail(error.ProcessFdQuotaExceeded),
5268 .INVAL => return error.BadPathName,5263 .NAMETOOLONG => return syscall.fail(error.NameTooLong),
5269 .BADF => |err| return errnoBug(err), // File descriptor used after closed.5264 .NFILE => return syscall.fail(error.SystemFdQuotaExceeded),
5270 .ACCES => return error.AccessDenied,5265 .NODEV => return syscall.fail(error.NoDevice),
5271 .LOOP => return error.SymLinkLoop,5266 .NOENT => return syscall.fail(error.FileNotFound),
5272 .MFILE => return error.ProcessFdQuotaExceeded,5267 .NOMEM => return syscall.fail(error.SystemResources),
5273 .NAMETOOLONG => return error.NameTooLong,5268 .NOTDIR => return syscall.fail(error.NotDir),
5274 .NFILE => return error.SystemFdQuotaExceeded,5269 .PERM => return syscall.fail(error.PermissionDenied),
5275 .NODEV => return error.NoDevice,5270 .NXIO => return syscall.fail(error.NoDevice),
5276 .NOENT => return error.FileNotFound,5271 .ILSEQ => return syscall.fail(error.BadPathName),
5277 .NOMEM => return error.SystemResources,5272 .FAULT => |err| return syscall.errnoBug(err),
5278 .NOTDIR => return error.NotDir,5273 .BADF => |err| return syscall.errnoBug(err), // File descriptor used after closed.
5279 .PERM => return error.PermissionDenied,5274 .BUSY => |err| return syscall.errnoBug(err), // O_EXCL not passed
5280 .BUSY => |err| return errnoBug(err), // O_EXCL not passed5275 else => |err| return syscall.unexpectedErrno(err),
5281 .NXIO => return error.NoDevice,
5282 .ILSEQ => return error.BadPathName,
5283 else => |err| return posix.unexpectedErrno(err),
5284 }
5285 },
5286 }5276 }
5287 }5277 }
5288}5278}
lib/std/Progress.zig+6
...@@ -325,6 +325,12 @@ pub const Node = struct {...@@ -325,6 +325,12 @@ pub const Node = struct {
325 return init(@enumFromInt(free_index), parent, name, estimated_total_items);325 return init(@enumFromInt(free_index), parent, name, estimated_total_items);
326 }326 }
327327
328 pub fn startFmt(node: Node, estimated_total_items: usize, comptime format: []const u8, args: anytype) Node {
329 var buffer: [max_name_len]u8 = undefined;
330 const name = std.fmt.bufPrint(&buffer, format, args) catch &buffer;
331 return Node.start(node, name, estimated_total_items);
332 }
333
328 /// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe.334 /// This is the same as calling `start` and then `end` on the returned `Node`. Thread-safe.
329 pub fn completeOne(n: Node) void {335 pub fn completeOne(n: Node) void {
330 const index = n.index.unwrap() orelse return;336 const index = n.index.unwrap() orelse return;
lib/std/compress/flate/Compress.zig+1-1
...@@ -267,7 +267,7 @@ pub const Options = struct {...@@ -267,7 +267,7 @@ pub const Options = struct {
267 pub const best = level_9;267 pub const best = level_9;
268};268};
269269
270/// It is asserted `buffer` is least `flate.max_history_len` bytes.270/// It is asserted `buffer` is least `flate.max_window_len` bytes.
271/// It is asserted `output` has a capacity of at least 8 bytes.271/// It is asserted `output` has a capacity of at least 8 bytes.
272pub fn init(272pub fn init(
273 output: *Writer,273 output: *Writer,
lib/std/zig.zig-1
...@@ -737,7 +737,6 @@ pub const EnvVar = enum {...@@ -737,7 +737,6 @@ pub const EnvVar = enum {
737 ZIG_BUILD_MULTILINE_ERRORS,737 ZIG_BUILD_MULTILINE_ERRORS,
738 ZIG_VERBOSE_LINK,738 ZIG_VERBOSE_LINK,
739 ZIG_VERBOSE_CC,739 ZIG_VERBOSE_CC,
740 ZIG_BTRFS_WORKAROUND,
741 ZIG_DEBUG_CMD,740 ZIG_DEBUG_CMD,
742 ZIG_IS_DETECTING_LIBC_PATHS,741 ZIG_IS_DETECTING_LIBC_PATHS,
743 ZIG_IS_TRYING_TO_NOT_CALL_ITSELF,742 ZIG_IS_TRYING_TO_NOT_CALL_ITSELF,
src/Package/Fetch.zig+265-242
...@@ -1,28 +1,34 @@...@@ -1,28 +1,34 @@
1//! Represents one independent job whose responsibility is to:1//! Represents one independent job whose responsibility is to:
2//!2//!
3//! 1. Check the global zig package cache to see if the hash already exists.3//! 1. Check the local zig package directory to see if the hash already exists.
4//! If so, load, parse, and validate the build.zig.zon file therein, and4//! If so, load, parse, and validate the build.zig.zon file therein, and
5//! goto step 8. Likewise if the location is a relative path, treat this5//! goto step 9. Likewise if the location is a relative path, treat this
6//! the same as a cache hit. Otherwise, proceed.6//! the same as a cache hit. Otherwise, proceed.
7//! 2. Fetch and unpack a URL into a temporary directory.7//! 2. Check the global package cache for a compressed tarball matching the
8//! 3. Load, parse, and validate the build.zig.zon file therein. It is allowed8//! hash. If it is found, unpack the contents into a temporary directory inside
9//! project local zig cache. Rename this directory into the local zig package
10//! directory and goto step 9, skipping step 10.
11//! 3. Fetch and unpack a URL into a temporary directory.
12//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
9//! for the file to be missing, in which case this fetched package is considered13//! for the file to be missing, in which case this fetched package is considered
10//! to be a "naked" package.14//! to be a "naked" package.
11//! 4. Apply inclusion rules of the build.zig.zon to the temporary directory by15//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
12//! deleting excluded files. If any files had errors for files that were16//! deleting excluded files. If any files had errors for files that were
13//! ultimately excluded, those errors should be ignored, such as failure to17//! ultimately excluded, those errors should be ignored, such as failure to
14//! create symlinks that weren't supposed to be included anyway.18//! create symlinks that weren't supposed to be included anyway.
15//! 5. Compute the package hash based on the remaining files in the temporary19//! 6. Compute the package hash based on the remaining files in the temporary
16//! directory.20//! directory.
17//! 6. Rename the temporary directory into the global zig package cache21//! 7. Rename the temporary directory into the local zig package directory. If
18//! directory. If the hash already exists, delete the temporary directory and22//! the hash already exists, delete the temporary directory and leave the zig
19//! leave the zig package cache directory untouched as it may be in use by the23//! package directory untouched as it may be in use. This is done even if
20//! system. This is done even if the hash is invalid, in case the package with24//! the hash is invalid, in case the package with the different hash is used
21//! the different hash is used in the future.25//! in the future.
22//! 7. Validate the computed hash against the expected hash. If invalid,26//! 8. Validate the computed hash against the expected hash. If invalid,
23//! this job is done.27//! this job is done.
24//! 8. Spawn a new fetch job for each dependency in the manifest file. Use28//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
25//! a mutex and a hash map so that redundant jobs do not get queued up.29//! a mutex and a hash map so that redundant jobs do not get queued up.
30//! 10.Compress the package directory and store it into the global package
31//! cache.
26//!32//!
27//! All of this must be done with only referring to the state inside this struct33//! All of this must be done with only referring to the state inside this struct
28//! because this work will be done in a dedicated thread.34//! because this work will be done in a dedicated thread.
...@@ -34,6 +40,7 @@ const native_os = builtin.os.tag;...@@ -34,6 +40,7 @@ const native_os = builtin.os.tag;
34const std = @import("std");40const std = @import("std");
35const Io = std.Io;41const Io = std.Io;
36const fs = std.fs;42const fs = std.fs;
43const log = std.log.scoped(.fetch);
37const assert = std.debug.assert;44const assert = std.debug.assert;
38const ascii = std.ascii;45const ascii = std.ascii;
39const Allocator = std.mem.Allocator;46const Allocator = std.mem.Allocator;
...@@ -60,16 +67,13 @@ omit_missing_hash_error: bool,...@@ -60,16 +67,13 @@ omit_missing_hash_error: bool,
60/// which specifies inclusion rules. This is intended to be true for the first67/// which specifies inclusion rules. This is intended to be true for the first
61/// fetch task and false for the recursive dependencies.68/// fetch task and false for the recursive dependencies.
62allow_missing_paths_field: bool,69allow_missing_paths_field: bool,
63allow_missing_fingerprint: bool,
64allow_name_string: bool,
65/// If true and URL points to a Git repository, will use the latest commit.70/// If true and URL points to a Git repository, will use the latest commit.
66use_latest_commit: bool,71use_latest_commit: bool,
6772
68// Above this are fields provided as inputs to `run`.73// Above this are fields provided as inputs to `run`.
69// Below this are fields populated by `run`.74// Below this are fields populated by `run`.
7075
71/// This will either be relative to `global_cache`, or to the build root of76/// Relative to the build root of the root package.
72/// the root package.
73package_root: Cache.Path,77package_root: Cache.Path,
74error_bundle: ErrorBundle.Wip,78error_bundle: ErrorBundle.Wip,
75manifest: ?Manifest,79manifest: ?Manifest,
...@@ -111,10 +115,15 @@ pub const JobQueue = struct {...@@ -111,10 +115,15 @@ pub const JobQueue = struct {
111 /// field contains references to all of them.115 /// field contains references to all of them.
112 /// Protected by `mutex`.116 /// Protected by `mutex`.
113 all_fetches: std.ArrayList(*Fetch) = .empty,117 all_fetches: std.ArrayList(*Fetch) = .empty,
118 prog_node: std.Progress.Node,
114119
115 http_client: *std.http.Client,120 http_client: *std.http.Client,
121 /// This tracks `Fetch` tasks as well as recompression tasks.
116 group: Io.Group = .init,122 group: Io.Group = .init,
117 global_cache: Cache.Directory,123 global_cache: Cache.Directory,
124 local_cache: Cache.Path,
125 /// Path to "zig-pkg" inside the package in which the user ran `zig build`.
126 root_pkg_path: Cache.Path,
118 /// If true then, no fetching occurs, and:127 /// If true then, no fetching occurs, and:
119 /// * The `global_cache` directory is assumed to be the direct parent128 /// * The `global_cache` directory is assumed to be the direct parent
120 /// directory of on-disk packages rather than having the "p/" directory129 /// directory of on-disk packages rather than having the "p/" directory
...@@ -129,7 +138,6 @@ pub const JobQueue = struct {...@@ -129,7 +138,6 @@ pub const JobQueue = struct {
129 /// two hashes of the same package do not match.138 /// two hashes of the same package do not match.
130 /// If this is true, `recursive` must be false.139 /// If this is true, `recursive` must be false.
131 debug_hash: bool,140 debug_hash: bool,
132 work_around_btrfs_bug: bool,
133 mode: Mode,141 mode: Mode,
134 /// Set of hashes that will be additionally fetched even if they are marked142 /// Set of hashes that will be additionally fetched even if they are marked
135 /// as lazy.143 /// as lazy.
...@@ -294,8 +302,121 @@ pub const JobQueue = struct {...@@ -294,8 +302,121 @@ pub const JobQueue = struct {
294 \\302 \\
295 );303 );
296 }304 }
305
306 fn recompress(jq: *JobQueue, package_hash: Package.Hash) Io.Cancelable!void {
307 const pkg_hash_slice = package_hash.toSlice();
308
309 const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
310 defer prog_node.end();
311
312 var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
313 const dest_path: Cache.Path = .{
314 .root_dir = jq.global_cache,
315 .sub_path = std.fmt.bufPrint(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
316 };
317
318 const gpa = jq.http_client.allocator;
319
320 var arena_instance = std.heap.ArenaAllocator.init(gpa);
321 defer arena_instance.deinit();
322 const arena = arena_instance.allocator();
323
324 recompressFallible(jq, arena, dest_path, pkg_hash_slice, prog_node) catch |err| switch (err) {
325 error.Canceled => |e| return e,
326 error.ReadFailed => comptime unreachable,
327 error.WriteFailed => comptime unreachable,
328 else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
329 };
330 }
331
332 fn recompressFallible(
333 jq: *JobQueue,
334 arena: Allocator,
335 dest_path: Cache.Path,
336 pkg_hash_slice: []const u8,
337 prog_node: std.Progress.Node,
338 ) !void {
339 const gpa = jq.http_client.allocator;
340 const io = jq.io;
341
342 // We have to walk the file system up front in order to sort the file
343 // list for determinism purposes. The hash of the recompressed file is
344 // not critical because the true hash is based on the content alone.
345 // However, if we want Zig users to be able to share cached package
346 // data with each other via peer-to-peer protocols, we benefit greatly
347 // from the data being identical on everyone's computers.
348 var scanned_files: std.ArrayList([]const u8) = .empty;
349 defer scanned_files.deinit(gpa);
350
351 var pkg_dir = try jq.root_pkg_path.openDir(io, pkg_hash_slice, .{ .iterate = true });
352 defer pkg_dir.close(io);
353
354 {
355 var walker = try pkg_dir.walk(gpa);
356 defer walker.deinit();
357
358 while (try walker.next(io)) |entry| {
359 switch (entry.kind) {
360 .directory => continue,
361 .file, .sym_link => {},
362 else => {
363 return error.IllegalFileType;
364 },
365 }
366 const entry_path = try arena.dupe(u8, entry.path);
367 try scanned_files.append(gpa, entry_path);
368 }
369
370 std.mem.sortUnstable([]const u8, scanned_files.items, {}, stringCmp);
371 }
372
373 prog_node.setEstimatedTotalItems(scanned_files.items.len);
374
375 var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
376 .make_path = true,
377 .replace = true,
378 });
379 defer atomic_file.deinit(io);
380
381 var file_write_buffer: [4096]u8 = undefined;
382 var file_writer = atomic_file.file.writer(io, &file_write_buffer);
383
384 var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
385 var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) {
386 error.WriteFailed => return file_writer.err.?,
387 };
388
389 var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
390 archiver.prefix = pkg_hash_slice;
391
392 var file_read_buffer: [4096]u8 = undefined;
393
394 for (scanned_files.items) |entry_path| {
395 var file = try pkg_dir.openFile(io, entry_path, .{});
396 defer file.close(io);
397 var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
398 archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) {
399 error.ReadFailed => return file_reader.err.?,
400 error.WriteFailed => return file_writer.err.?,
401 else => |e| return e,
402 };
403 prog_node.completeOne();
404 }
405
406 // intentionally omitting the pointless trailer
407 //try archiver.finish();
408 compress.writer.flush() catch |err| switch (err) {
409 error.WriteFailed => return file_writer.err.?,
410 };
411 try file_writer.flush();
412 try atomic_file.replace(io);
413 }
297};414};
298415
416fn stringCmp(_: void, lhs: []const u8, rhs: []const u8) bool {
417 return std.mem.lessThan(u8, lhs, rhs);
418}
419
299pub const Location = union(enum) {420pub const Location = union(enum) {
300 remote: Remote,421 remote: Remote,
301 /// A directory found inside the parent package.422 /// A directory found inside the parent package.
...@@ -326,11 +447,12 @@ pub const RunError = error{...@@ -326,11 +447,12 @@ pub const RunError = error{
326};447};
327448
328pub fn run(f: *Fetch) RunError!void {449pub fn run(f: *Fetch) RunError!void {
329 const io = f.job_queue.io;450 const job_queue = f.job_queue;
451 const io = job_queue.io;
330 const eb = &f.error_bundle;452 const eb = &f.error_bundle;
331 const arena = f.arena.allocator();453 const arena = f.arena.allocator();
332 const gpa = f.arena.child_allocator;454 const gpa = f.arena.child_allocator;
333 const cache_root = f.job_queue.global_cache;455 const local_cache_root = job_queue.local_cache;
334456
335 try eb.init(gpa);457 try eb.init(gpa);
336458
...@@ -351,13 +473,13 @@ pub fn run(f: *Fetch) RunError!void {...@@ -351,13 +473,13 @@ pub fn run(f: *Fetch) RunError!void {
351 );473 );
352 // Packages fetched by URL may not use relative paths to escape outside the474 // Packages fetched by URL may not use relative paths to escape outside the
353 // fetched package directory from within the package cache.475 // fetched package directory from within the package cache.
354 if (pkg_root.root_dir.eql(cache_root)) {476 if (pkg_root.root_dir.eql(local_cache_root.root_dir)) {
355 // `parent_package_root.sub_path` contains a path like this:477 // `parent_package_root.sub_path` contains a path like this:
356 // "p/$hash", or478 // "p/$hash", or
357 // "p/$hash/foo", with possibly more directories after "foo".479 // "p/$hash/foo", with possibly more directories after "foo".
358 // We want to fail unless the resolved relative path has a480 // We want to fail unless the resolved relative path has a
359 // prefix of "p/$hash/".481 // prefix of "p/$hash/".
360 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;482 const prefix_len: usize = if (job_queue.read_only) 0 else "p/".len;
361 const parent_sub_path = f.parent_package_root.sub_path;483 const parent_sub_path = f.parent_package_root.sub_path;
362 const end = find_end: {484 const end = find_end: {
363 if (parent_sub_path.len > prefix_len) {485 if (parent_sub_path.len > prefix_len) {
...@@ -380,21 +502,21 @@ pub fn run(f: *Fetch) RunError!void {...@@ -380,21 +502,21 @@ pub fn run(f: *Fetch) RunError!void {
380 f.package_root = pkg_root;502 f.package_root = pkg_root;
381 try loadManifest(f, pkg_root);503 try loadManifest(f, pkg_root);
382 if (!f.has_build_zig) try checkBuildFileExistence(f);504 if (!f.has_build_zig) try checkBuildFileExistence(f);
383 if (!f.job_queue.recursive) return;505 if (!job_queue.recursive) return;
384 return queueJobsForDeps(f);506 return queueJobsForDeps(f);
385 },507 },
386 .remote => |remote| remote,508 .remote => |remote| remote,
387 .path_or_url => |path_or_url| {509 .path_or_url => |path_or_url| {
388 if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| {510 if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| {
389 var resource: Resource = .{ .dir = dir };511 var resource: Resource = .{ .dir = dir };
390 return f.runResource(path_or_url, &resource, null);512 return f.runResource(path_or_url, &resource, null, false);
391 } else |dir_err| {513 } else |dir_err| {
392 var server_header_buffer: [init_resource_buffer_size]u8 = undefined;514 var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
393515
394 const file_err = if (dir_err == error.NotDir) e: {516 const file_err = if (dir_err == error.NotDir) e: {
395 if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| {517 if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| {
396 var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };518 var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };
397 return f.runResource(path_or_url, &resource, null);519 return f.runResource(path_or_url, &resource, null, false);
398 } else |err| break :e err;520 } else |err| break :e err;
399 } else dir_err;521 } else dir_err;
400522
...@@ -406,57 +528,73 @@ pub fn run(f: *Fetch) RunError!void {...@@ -406,57 +528,73 @@ pub fn run(f: *Fetch) RunError!void {
406 };528 };
407 var resource: Resource = undefined;529 var resource: Resource = undefined;
408 try f.initResource(uri, &resource, &server_header_buffer);530 try f.initResource(uri, &resource, &server_header_buffer);
409 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null);531 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false);
410 }532 }
411 },533 },
412 };534 };
413535
536 var resource_buffer: [init_resource_buffer_size]u8 = undefined;
537
414 if (remote.hash) |expected_hash| {538 if (remote.hash) |expected_hash| {
415 var prefixed_pkg_sub_path_buffer: [Package.Hash.max_len + 2]u8 = undefined;539 const package_root = try job_queue.root_pkg_path.join(arena, expected_hash.toSlice());
416 prefixed_pkg_sub_path_buffer[0] = 'p';540 if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
417 prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
418 const hash_slice = expected_hash.toSlice();
419 @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
420 const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
421 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
422 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
423 if (cache_root.handle.access(io, pkg_sub_path, .{})) |_| {
424 assert(f.lazy_status != .unavailable);541 assert(f.lazy_status != .unavailable);
425 f.package_root = .{542 f.package_root = package_root;
426 .root_dir = cache_root,
427 .sub_path = try arena.dupe(u8, pkg_sub_path),
428 };
429 try loadManifest(f, f.package_root);543 try loadManifest(f, f.package_root);
430 try checkBuildFileExistence(f);544 try checkBuildFileExistence(f);
431 if (!f.job_queue.recursive) return;545 if (!job_queue.recursive) return;
432 return queueJobsForDeps(f);546 return queueJobsForDeps(f);
433 } else |err| switch (err) {547 } else |err| switch (err) {
434 error.FileNotFound => {548 error.FileNotFound => {
435 switch (f.lazy_status) {549 log.debug("FileNotFound: {f}", .{package_root});
436 .eager => {},550 if (job_queue.read_only) return f.fail(
437 .available => if (!f.job_queue.unlazy_set.contains(expected_hash)) {
438 f.lazy_status = .unavailable;
439 return;
440 },
441 .unavailable => unreachable,
442 }
443 if (f.job_queue.read_only) return f.fail(
444 f.name_tok,551 f.name_tok,
445 try eb.printString("package not found at '{f}{s}'", .{552 try eb.printString("package not found at '{f}'", .{package_root}),
446 cache_root, pkg_sub_path,
447 }),
448 );553 );
449 },554 },
555 error.Canceled => |e| return e,
556 else => |e| {
557 try eb.addRootErrorMessage(.{
558 .msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
559 package_root, e,
560 }),
561 });
562 return error.FetchFailed;
563 },
564 }
565
566 // Check global cache before remote fetch.
567 const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()});
568 const cached_tarball_path: Cache.Path = .{
569 .root_dir = job_queue.global_cache,
570 .sub_path = cached_tarball_sub_path,
571 };
572 if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| {
573 log.debug("found global cached tarball {f}", .{cached_tarball_path});
574 var resource: Resource = .{ .file = file.reader(io, &resource_buffer) };
575 return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true);
576 } else |err| switch (err) {
577 error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}),
578 error.Canceled => |e| return e,
450 else => |e| {579 else => |e| {
451 try eb.addRootErrorMessage(.{580 try eb.addRootErrorMessage(.{
452 .msg = try eb.printString("unable to open global package cache directory '{f}{s}': {s}", .{581 .msg = try eb.printString("unable to open globally cached package {f}: {t}", .{
453 cache_root, pkg_sub_path, @errorName(e),582 cached_tarball_path, e,
454 }),583 }),
455 });584 });
456 return error.FetchFailed;585 return error.FetchFailed;
457 },586 },
458 }587 }
459 } else if (f.job_queue.read_only) {588
589 switch (f.lazy_status) {
590 .eager => {},
591 .available => if (!job_queue.unlazy_set.contains(expected_hash)) {
592 f.lazy_status = .unavailable;
593 return;
594 },
595 .unavailable => unreachable,
596 }
597 } else if (job_queue.read_only) {
460 try eb.addRootErrorMessage(.{598 try eb.addRootErrorMessage(.{
461 .msg = try eb.addString("dependency is missing hash field"),599 .msg = try eb.addString("dependency is missing hash field"),
462 .src_loc = try f.srcLoc(f.location_tok),600 .src_loc = try f.srcLoc(f.location_tok),
...@@ -465,15 +603,13 @@ pub fn run(f: *Fetch) RunError!void {...@@ -465,15 +603,13 @@ pub fn run(f: *Fetch) RunError!void {
465 }603 }
466604
467 // Fetch and unpack the remote into a temporary directory.605 // Fetch and unpack the remote into a temporary directory.
468
469 const uri = std.Uri.parse(remote.url) catch |err| return f.fail(606 const uri = std.Uri.parse(remote.url) catch |err| return f.fail(
470 f.location_tok,607 f.location_tok,
471 try eb.printString("invalid URI: {s}", .{@errorName(err)}),608 try eb.printString("invalid URI: {t}", .{err}),
472 );609 );
473 var buffer: [init_resource_buffer_size]u8 = undefined;
474 var resource: Resource = undefined;610 var resource: Resource = undefined;
475 try f.initResource(uri, &resource, &buffer);611 try f.initResource(uri, &resource, &resource_buffer);
476 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash);612 return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false);
477}613}
478614
479pub fn deinit(f: *Fetch) void {615pub fn deinit(f: *Fetch) void {
...@@ -487,30 +623,35 @@ fn runResource(...@@ -487,30 +623,35 @@ fn runResource(
487 uri_path: []const u8,623 uri_path: []const u8,
488 resource: *Resource,624 resource: *Resource,
489 remote_hash: ?Package.Hash,625 remote_hash: ?Package.Hash,
626 disable_recompress: bool,
490) RunError!void {627) RunError!void {
491 const io = f.job_queue.io;628 const job_queue = f.job_queue;
629 assert(!job_queue.read_only);
630
631 const io = job_queue.io;
492 defer resource.deinit(io);632 defer resource.deinit(io);
633
493 const arena = f.arena.allocator();634 const arena = f.arena.allocator();
494 const eb = &f.error_bundle;635 const eb = &f.error_bundle;
495 const s = fs.path.sep_str;636 const s = fs.path.sep_str;
496 const cache_root = f.job_queue.global_cache;637 const local_cache_root = job_queue.local_cache;
497 const rand_int = r: {638 const rand_int = r: {
498 var x: u64 = undefined;639 var x: u64 = undefined;
499 io.random(@ptrCast(&x));640 io.random(@ptrCast(&x));
500 break :r x;641 break :r x;
501 };642 };
502 const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int);643 const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int);
644 const tmp_directory_path = try local_cache_root.join(arena, tmp_dir_sub_path);
503645
504 const package_sub_path = blk: {646 const package_sub_path = blk: {
505 const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
506 var tmp_directory: Cache.Directory = .{647 var tmp_directory: Cache.Directory = .{
507 .path = tmp_directory_path,648 .path = tmp_directory_path.sub_path,
508 .handle = handle: {649 .handle = handle: {
509 const dir = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{650 const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
510 .open_options = .{ .iterate = true },651 .open_options = .{ .iterate = true },
511 }) catch |err| {652 }) catch |err| {
512 try eb.addRootErrorMessage(.{653 try eb.addRootErrorMessage(.{
513 .msg = try eb.printString("unable to create temporary directory '{s}': {t}", .{654 .msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
514 tmp_directory_path, err,655 tmp_directory_path, err,
515 }),656 }),
516 });657 });
...@@ -524,16 +665,7 @@ fn runResource(...@@ -524,16 +665,7 @@ fn runResource(
524 // Fetch and unpack a resource into a temporary directory.665 // Fetch and unpack a resource into a temporary directory.
525 var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);666 var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
526667
527 var pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };668 const pkg_path: Cache.Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
528
529 // Apply btrfs workaround if needed. Reopen tmp_directory.
530 if (native_os == .linux and f.job_queue.work_around_btrfs_bug) {
531 // https://github.com/ziglang/zig/issues/17095
532 pkg_path.root_dir.handle.close(io);
533 pkg_path.root_dir.handle = cache_root.handle.createDirPathOpen(io, tmp_dir_sub_path, .{
534 .open_options = .{ .iterate = true },
535 }) catch @panic("btrfs workaround failed");
536 }
537669
538 // Load, parse, and validate the unpacked build.zig.zon file. It is allowed670 // Load, parse, and validate the unpacked build.zig.zon file. It is allowed
539 // for the file to be missing, in which case this fetched package is671 // for the file to be missing, in which case this fetched package is
...@@ -555,36 +687,40 @@ fn runResource(...@@ -555,36 +687,40 @@ fn runResource(
555 // directory.687 // directory.
556 f.computed_hash = try computeHash(f, pkg_path, filter);688 f.computed_hash = try computeHash(f, pkg_path, filter);
557689
558 break :blk if (unpack_result.root_dir.len > 0)690 if (unpack_result.root_dir.len > 0)
559 try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })691 break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
560 else692
561 tmp_dir_sub_path;693 break :blk tmp_directory_path;
562 };694 };
563695
564 const computed_package_hash = computedPackageHash(f);696 const computed_package_hash = computedPackageHash(f);
565697
566 // Rename the temporary directory into the global zig package cache698 // Rename the temporary directory into the local zig package directory. If
567 // directory. If the hash already exists, delete the temporary directory699 // the hash already exists, delete the temporary directory and leave the
568 // and leave the zig package cache directory untouched as it may be in use700 // zig package directory untouched as it may be in use. This is done even
569 // by the system. This is done even if the hash is invalid, in case the701 // if the hash is invalid, in case the package with the different hash is
570 // package with the different hash is used in the future.702 // used in the future.
571703 f.package_root = try job_queue.root_pkg_path.join(arena, computed_package_hash.toSlice());
572 f.package_root = .{704 renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
573 .root_dir = cache_root,
574 .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
575 };
576 renameTmpIntoCache(io, cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
577 const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
578 const dest = try cache_root.join(arena, &.{f.package_root.sub_path});
579 try eb.addRootErrorMessage(.{ .msg = try eb.printString(705 try eb.addRootErrorMessage(.{ .msg = try eb.printString(
580 "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}",706 "unable to rename temporary directory {f} into package cache directory {f}: {t}",
581 .{ src, dest, @errorName(err) },707 .{ package_sub_path, f.package_root, err },
582 ) });708 ) });
583 return error.FetchFailed;709 return error.FetchFailed;
584 };710 };
711
712 if (!disable_recompress) {
713 // Spin off a task to recompress the tarball, with filtered files deleted, into
714 // the global cache.
715 job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash });
716 }
717
585 // Remove temporary directory root if not already renamed to global cache.718 // Remove temporary directory root if not already renamed to global cache.
586 if (!std.mem.eql(u8, package_sub_path, tmp_dir_sub_path)) {719 if (!package_sub_path.eql(tmp_directory_path)) {
587 cache_root.handle.deleteDir(io, tmp_dir_sub_path) catch {};720 tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) {
721 error.Canceled => |e| return e,
722 else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }),
723 };
588 }724 }
589725
590 // Validate the computed hash against the expected hash. If invalid, this726 // Validate the computed hash against the expected hash. If invalid, this
...@@ -624,7 +760,7 @@ fn runResource(...@@ -624,7 +760,7 @@ fn runResource(
624760
625 // Spawn a new fetch job for each dependency in the manifest file. Use761 // Spawn a new fetch job for each dependency in the manifest file. Use
626 // a mutex and a hash map so that redundant jobs do not get queued up.762 // a mutex and a hash map so that redundant jobs do not get queued up.
627 if (!f.job_queue.recursive) return;763 if (!job_queue.recursive) return;
628 return queueJobsForDeps(f);764 return queueJobsForDeps(f);
629}765}
630766
...@@ -651,8 +787,8 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void {...@@ -651,8 +787,8 @@ fn checkBuildFileExistence(f: *Fetch) RunError!void {
651 error.FileNotFound => {},787 error.FileNotFound => {},
652 else => |e| {788 else => |e| {
653 try eb.addRootErrorMessage(.{789 try eb.addRootErrorMessage(.{
654 .msg = try eb.printString("unable to access '{f}{s}': {s}", .{790 .msg = try eb.printString("unable to access '{f}{s}': {t}", .{
655 f.package_root, Package.build_zig_basename, @errorName(e),791 f.package_root, Package.build_zig_basename, e,
656 }),792 }),
657 });793 });
658 return error.FetchFailed;794 return error.FetchFailed;
...@@ -677,9 +813,7 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {...@@ -677,9 +813,7 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
677 else => |e| {813 else => |e| {
678 const file_path = try pkg_root.join(arena, Manifest.basename);814 const file_path = try pkg_root.join(arena, Manifest.basename);
679 try eb.addRootErrorMessage(.{815 try eb.addRootErrorMessage(.{
680 .msg = try eb.printString("unable to load package manifest '{f}': {s}", .{816 .msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ file_path, e }),
681 file_path, @errorName(e),
682 }),
683 });817 });
684 return error.FetchFailed;818 return error.FetchFailed;
685 },819 },
...@@ -698,8 +832,6 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {...@@ -698,8 +832,6 @@ fn loadManifest(f: *Fetch, pkg_root: Cache.Path) RunError!void {
698832
699 f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{833 f.manifest = try Manifest.parse(arena, ast.*, rng.interface(), .{
700 .allow_missing_paths_field = f.allow_missing_paths_field,834 .allow_missing_paths_field = f.allow_missing_paths_field,
701 .allow_missing_fingerprint = f.allow_missing_fingerprint,
702 .allow_name_string = f.allow_name_string,
703 });835 });
704 const manifest = &f.manifest.?;836 const manifest = &f.manifest.?;
705837
...@@ -817,8 +949,6 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -817,8 +949,6 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
817 .job_queue = f.job_queue,949 .job_queue = f.job_queue,
818 .omit_missing_hash_error = false,950 .omit_missing_hash_error = false,
819 .allow_missing_paths_field = true,951 .allow_missing_paths_field = true,
820 .allow_missing_fingerprint = true,
821 .allow_name_string = true,
822 .use_latest_commit = false,952 .use_latest_commit = false,
823953
824 .package_root = undefined,954 .package_root = undefined,
...@@ -1463,14 +1593,20 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void...@@ -1463,14 +1593,20 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void
1463 }1593 }
1464}1594}
14651595
1466pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {1596pub fn renameTmpIntoCache(io: Io, tmp_path: Cache.Path, dest_path: Cache.Path) !void {
1467 assert(dest_dir_sub_path[1] == fs.path.sep);
1468 var handled_missing_dir = false;1597 var handled_missing_dir = false;
1469 while (true) {1598 while (true) {
1470 cache_dir.rename(tmp_dir_sub_path, cache_dir, dest_dir_sub_path, io) catch |err| switch (err) {1599 Io.Dir.rename(
1600 tmp_path.root_dir.handle,
1601 tmp_path.sub_path,
1602 dest_path.root_dir.handle,
1603 dest_path.sub_path,
1604 io,
1605 ) catch |err| switch (err) {
1471 error.FileNotFound => {1606 error.FileNotFound => {
1472 if (handled_missing_dir) return err;1607 if (handled_missing_dir) return err;
1473 cache_dir.createDir(io, dest_dir_sub_path[0..1], .default_dir) catch |mkd_err| switch (mkd_err) {1608 const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
1609 dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) {
1474 error.PathAlreadyExists => handled_missing_dir = true,1610 error.PathAlreadyExists => handled_missing_dir = true,
1475 else => |e| return e,1611 else => |e| return e,
1476 };1612 };
...@@ -1478,9 +1614,11 @@ pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u...@@ -1478,9 +1614,11 @@ pub fn renameTmpIntoCache(io: Io, cache_dir: Io.Dir, tmp_dir_sub_path: []const u
1478 },1614 },
1479 error.DirNotEmpty, error.AccessDenied => {1615 error.DirNotEmpty, error.AccessDenied => {
1480 // Package has been already downloaded and may already be in use on the system.1616 // Package has been already downloaded and may already be in use on the system.
1481 cache_dir.deleteTree(io, tmp_dir_sub_path) catch {1617 tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) {
1618 error.Canceled => |e| return e,
1482 // Garbage files leftover in zig-cache/tmp/ is, as they say1619 // Garbage files leftover in zig-cache/tmp/ is, as they say
1483 // on Star Trek, "operating within normal parameters".1620 // on Star Trek, "operating within normal parameters".
1621 else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
1484 };1622 };
1485 },1623 },
1486 else => |e| return e,1624 else => |e| return e,
...@@ -2064,130 +2202,6 @@ const UnpackResult = struct {...@@ -2064,130 +2202,6 @@ const UnpackResult = struct {
2064 }2202 }
2065};2203};
20662204
2067test "tarball with duplicate paths" {
2068 // This tarball has duplicate path 'dir1/file1' to simulate case sensitve
2069 // file system on any file sytstem.
2070 //
2071 // duplicate_paths/
2072 // duplicate_paths/dir1/
2073 // duplicate_paths/dir1/file1
2074 // duplicate_paths/dir1/file1
2075 // duplicate_paths/build.zig.zon
2076 // duplicate_paths/src/
2077 // duplicate_paths/src/main.zig
2078 // duplicate_paths/src/root.zig
2079 // duplicate_paths/build.zig
2080 //
2081
2082 const gpa = std.testing.allocator;
2083 const io = std.testing.io;
2084 var tmp = std.testing.tmpDir(.{});
2085 defer tmp.cleanup();
2086
2087 const tarball_name = "duplicate_paths.tar.gz";
2088 try saveEmbedFile(io, tarball_name, tmp.dir);
2089 const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name });
2090 defer gpa.free(tarball_path);
2091
2092 // Run tarball fetch, expect to fail
2093 var fb: TestFetchBuilder = undefined;
2094 var fetch = try fb.build(gpa, io, tmp.dir, tarball_path);
2095 defer fb.deinit();
2096 try std.testing.expectError(error.FetchFailed, fetch.run());
2097
2098 try fb.expectFetchErrors(1,
2099 \\error: unable to unpack tarball
2100 \\ note: unable to create file 'dir1/file1': PathAlreadyExists
2101 \\
2102 );
2103}
2104
2105test "tarball with excluded duplicate paths" {
2106 // Same as previous tarball but has build.zig.zon wich excludes 'dir1'.
2107 //
2108 // .paths = .{
2109 // "build.zig",
2110 // "build.zig.zon",
2111 // "src",
2112 // }
2113 //
2114
2115 const gpa = std.testing.allocator;
2116 const io = std.testing.io;
2117 var tmp = std.testing.tmpDir(.{});
2118 defer tmp.cleanup();
2119
2120 const tarball_name = "duplicate_paths_excluded.tar.gz";
2121 try saveEmbedFile(io, tarball_name, tmp.dir);
2122 const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name });
2123 defer gpa.free(tarball_path);
2124
2125 // Run tarball fetch, should succeed
2126 var fb: TestFetchBuilder = undefined;
2127 var fetch = try fb.build(gpa, io, tmp.dir, tarball_path);
2128 defer fb.deinit();
2129 try fetch.run();
2130
2131 const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2132 try std.testing.expectEqualStrings(
2133 "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
2134 &hex_digest,
2135 );
2136
2137 const expected_files: []const []const u8 = &.{
2138 "build.zig",
2139 "build.zig.zon",
2140 "src/main.zig",
2141 "src/root.zig",
2142 };
2143 try fb.expectPackageFiles(expected_files);
2144}
2145
2146test "tarball without root folder" {
2147 // Tarball with root folder. Manifest excludes dir1 and dir2.
2148 //
2149 // build.zig
2150 // build.zig.zon
2151 // dir1/
2152 // dir1/file2
2153 // dir1/file1
2154 // dir2/
2155 // dir2/file2
2156 // src/
2157 // src/main.zig
2158 //
2159
2160 const gpa = std.testing.allocator;
2161 const io = std.testing.io;
2162
2163 var tmp = std.testing.tmpDir(.{});
2164 defer tmp.cleanup();
2165
2166 const tarball_name = "no_root.tar.gz";
2167 try saveEmbedFile(io, tarball_name, tmp.dir);
2168 const tarball_path = try std.fmt.allocPrint(gpa, ".zig-cache/tmp/{s}/{s}", .{ tmp.sub_path, tarball_name });
2169 defer gpa.free(tarball_path);
2170
2171 // Run tarball fetch, should succeed
2172 var fb: TestFetchBuilder = undefined;
2173 var fetch = try fb.build(gpa, io, tmp.dir, tarball_path);
2174 defer fb.deinit();
2175 try fetch.run();
2176
2177 const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
2178 try std.testing.expectEqualStrings(
2179 "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
2180 &hex_digest,
2181 );
2182
2183 const expected_files: []const []const u8 = &.{
2184 "build.zig",
2185 "build.zig.zon",
2186 "src/main.zig",
2187 };
2188 try fb.expectPackageFiles(expected_files);
2189}
2190
2191test "set executable bit based on file content" {2205test "set executable bit based on file content" {
2192 if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;2206 if (!Io.File.Permissions.has_executable_bit) return error.SkipZigTest;
2193 const gpa = std.testing.allocator;2207 const gpa = std.testing.allocator;
...@@ -2254,6 +2268,7 @@ fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void {...@@ -2254,6 +2268,7 @@ fn saveEmbedFile(io: Io, comptime tarball_name: []const u8, dir: Io.Dir) !void {
2254const TestFetchBuilder = struct {2268const TestFetchBuilder = struct {
2255 http_client: std.http.Client,2269 http_client: std.http.Client,
2256 global_cache_directory: Cache.Directory,2270 global_cache_directory: Cache.Directory,
2271 local_cache_path: Cache.Path,
2257 job_queue: Fetch.JobQueue,2272 job_queue: Fetch.JobQueue,
2258 fetch: Fetch,2273 fetch: Fetch,
22592274
...@@ -2264,20 +2279,30 @@ const TestFetchBuilder = struct {...@@ -2264,20 +2279,30 @@ const TestFetchBuilder = struct {
2264 cache_parent_dir: std.Io.Dir,2279 cache_parent_dir: std.Io.Dir,
2265 path_or_url: []const u8,2280 path_or_url: []const u8,
2266 ) !*Fetch {2281 ) !*Fetch {
2267 const cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{});2282 const global_cache_dir = try cache_parent_dir.createDirPathOpen(io, "zig-global-cache", .{});
2283 const package_root_dir = try cache_parent_dir.createDirPathOpen(io, "local-project-root", .{});
22682284
2269 self.http_client = .{ .allocator = allocator, .io = io };2285 self.http_client = .{ .allocator = allocator, .io = io };
2270 self.global_cache_directory = .{ .handle = cache_dir, .path = null };2286 self.global_cache_directory = .{ .handle = global_cache_dir, .path = "zig-global-cache" };
2287 self.local_cache_path = .{
2288 .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
2289 .sub_path = ".zig-cache",
2290 };
22712291
2272 self.job_queue = .{2292 self.job_queue = .{
2273 .io = io,2293 .io = io,
2274 .http_client = &self.http_client,2294 .http_client = &self.http_client,
2275 .global_cache = self.global_cache_directory,2295 .global_cache = self.global_cache_directory,
2296 .local_cache = self.local_cache_path,
2297 .root_pkg_path = .{
2298 .root_dir = .{ .handle = package_root_dir, .path = "local-project-root" },
2299 .sub_path = "zig-pkg",
2300 },
2276 .recursive = false,2301 .recursive = false,
2277 .read_only = false,2302 .read_only = false,
2278 .debug_hash = false,2303 .debug_hash = false,
2279 .work_around_btrfs_bug = false,
2280 .mode = .needed,2304 .mode = .needed,
2305 .prog_node = std.Progress.Node.none,
2281 };2306 };
22822307
2283 self.fetch = .{2308 self.fetch = .{
...@@ -2287,14 +2312,12 @@ const TestFetchBuilder = struct {...@@ -2287,14 +2312,12 @@ const TestFetchBuilder = struct {
2287 .hash_tok = .none,2312 .hash_tok = .none,
2288 .name_tok = 0,2313 .name_tok = 0,
2289 .lazy_status = .eager,2314 .lazy_status = .eager,
2290 .parent_package_root = Cache.Path{ .root_dir = Cache.Directory{ .handle = cache_dir, .path = null } },2315 .parent_package_root = .{ .root_dir = .{ .handle = package_root_dir, .path = null } },
2291 .parent_manifest_ast = null,2316 .parent_manifest_ast = null,
2292 .prog_node = std.Progress.Node.none,2317 .prog_node = std.Progress.Node.none,
2293 .job_queue = &self.job_queue,2318 .job_queue = &self.job_queue,
2294 .omit_missing_hash_error = true,2319 .omit_missing_hash_error = true,
2295 .allow_missing_paths_field = false,2320 .allow_missing_paths_field = false,
2296 .allow_missing_fingerprint = true, // so we can keep using the old testdata .tar.gz
2297 .allow_name_string = true, // so we can keep using the old testdata .tar.gz
2298 .use_latest_commit = true,2321 .use_latest_commit = true,
22992322
2300 .package_root = undefined,2323 .package_root = undefined,
src/Package/Fetch/testdata/duplicate_paths.tar.gz deleted
Binary files a/src/Package/Fetch/testdata/duplicate_paths.tar.gz and /dev/null differ
src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz deleted
Binary files a/src/Package/Fetch/testdata/duplicate_paths_excluded.tar.gz and /dev/null differ
src/Package/Fetch/testdata/no_root.tar.gz deleted
Binary files a/src/Package/Fetch/testdata/no_root.tar.gz and /dev/null differ
src/Package/Manifest.zig+7-27
...@@ -49,10 +49,6 @@ arena_state: std.heap.ArenaAllocator.State,...@@ -49,10 +49,6 @@ arena_state: std.heap.ArenaAllocator.State,
4949
50pub const ParseOptions = struct {50pub const ParseOptions = struct {
51 allow_missing_paths_field: bool = false,51 allow_missing_paths_field: bool = false,
52 /// Deprecated, to be removed after 0.14.0 is tagged.
53 allow_name_string: bool = true,
54 /// Deprecated, to be removed after 0.14.0 is tagged.
55 allow_missing_fingerprint: bool = true,
56};52};
5753
58pub const Error = Allocator.Error;54pub const Error = Allocator.Error;
...@@ -77,8 +73,6 @@ pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) E...@@ -77,8 +73,6 @@ pub fn parse(gpa: Allocator, ast: Ast, rng: std.Random, options: ParseOptions) E
77 .dependencies_node = .none,73 .dependencies_node = .none,
78 .paths = .{},74 .paths = .{},
79 .allow_missing_paths_field = options.allow_missing_paths_field,75 .allow_missing_paths_field = options.allow_missing_paths_field,
80 .allow_name_string = options.allow_name_string,
81 .allow_missing_fingerprint = options.allow_missing_fingerprint,
82 .minimum_zig_version = null,76 .minimum_zig_version = null,
83 .buf = .{},77 .buf = .{},
84 };78 };
...@@ -151,8 +145,6 @@ const Parse = struct {...@@ -151,8 +145,6 @@ const Parse = struct {
151 dependencies_node: Ast.Node.OptionalIndex,145 dependencies_node: Ast.Node.OptionalIndex,
152 paths: std.StringArrayHashMapUnmanaged(void),146 paths: std.StringArrayHashMapUnmanaged(void),
153 allow_missing_paths_field: bool,147 allow_missing_paths_field: bool,
154 allow_name_string: bool,
155 allow_missing_fingerprint: bool,
156 minimum_zig_version: ?std.SemanticVersion,148 minimum_zig_version: ?std.SemanticVersion,
157149
158 const InnerError = error{ ParseFailure, OutOfMemory };150 const InnerError = error{ ParseFailure, OutOfMemory };
...@@ -221,12 +213,10 @@ const Parse = struct {...@@ -221,12 +213,10 @@ const Parse = struct {
221 });213 });
222 }214 }
223 p.id = n.id;215 p.id = n.id;
224 } else if (!p.allow_missing_fingerprint) {216 } else {
225 try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{217 try appendError(p, main_token, "missing top-level 'fingerprint' field; suggested value: 0x{x}", .{
226 Package.Fingerprint.generate(rng, p.name).int(),218 Package.Fingerprint.generate(rng, p.name).int(),
227 });219 });
228 } else {
229 p.id = 0;
230 }220 }
231 }221 }
232222
...@@ -395,19 +385,6 @@ const Parse = struct {...@@ -395,19 +385,6 @@ const Parse = struct {
395 const ast = p.ast;385 const ast = p.ast;
396 const main_token = ast.nodeMainToken(node);386 const main_token = ast.nodeMainToken(node);
397387
398 if (p.allow_name_string and ast.nodeTag(node) == .string_literal) {
399 const name = try parseString(p, node);
400 if (!std.zig.isValidId(name))
401 return fail(p, main_token, "name must be a valid bare zig identifier (hint: switch from string to enum literal)", .{});
402
403 if (name.len > max_name_len)
404 return fail(p, main_token, "name '{f}' exceeds max length of {d}", .{
405 std.zig.fmtId(name), max_name_len,
406 });
407
408 return name;
409 }
410
411 if (ast.nodeTag(node) != .enum_literal)388 if (ast.nodeTag(node) != .enum_literal)
412 return fail(p, main_token, "expected enum literal", .{});389 return fail(p, main_token, "expected enum literal", .{});
413390
...@@ -606,7 +583,8 @@ test "basic" {...@@ -606,7 +583,8 @@ test "basic" {
606583
607 const example =584 const example =
608 \\.{585 \\.{
609 \\ .name = "foo",586 \\ .name = .foo,
587 \\ .fingerprint = 0x8c736521490b23df,
610 \\ .version = "3.2.1",588 \\ .version = "3.2.1",
611 \\ .paths = .{""},589 \\ .paths = .{""},
612 \\ .dependencies = .{590 \\ .dependencies = .{
...@@ -656,7 +634,8 @@ test "minimum_zig_version" {...@@ -656,7 +634,8 @@ test "minimum_zig_version" {
656634
657 const example =635 const example =
658 \\.{636 \\.{
659 \\ .name = "foo",637 \\ .name = .foo,
638 \\ .fingerprint = 0x8c736521490b23df,
660 \\ .version = "3.2.1",639 \\ .version = "3.2.1",
661 \\ .paths = .{""},640 \\ .paths = .{""},
662 \\ .minimum_zig_version = "0.11.1",641 \\ .minimum_zig_version = "0.11.1",
...@@ -690,7 +669,8 @@ test "minimum_zig_version - invalid version" {...@@ -690,7 +669,8 @@ test "minimum_zig_version - invalid version" {
690669
691 const example =670 const example =
692 \\.{671 \\.{
693 \\ .name = "foo",672 \\ .name = .foo,
673 \\ .fingerprint = 0x8c736521490b23df,
694 \\ .version = "3.2.1",674 \\ .version = "3.2.1",
695 \\ .minimum_zig_version = "X.11.1",675 \\ .minimum_zig_version = "X.11.1",
696 \\ .paths = .{""},676 \\ .paths = .{""},
src/main.zig+40-26
...@@ -5098,8 +5098,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,...@@ -5098,8 +5098,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5098 }5098 }
5099 }5099 }
51005100
5101 const work_around_btrfs_bug = native_os == .linux and
5102 EnvVar.ZIG_BTRFS_WORKAROUND.isSet(environ_map);
5103 const root_prog_node = std.Progress.start(io, .{5101 const root_prog_node = std.Progress.start(io, .{
5104 .disable_printing = (color == .off),5102 .disable_printing = (color == .off),
5105 .root_name = "Compile Build Script",5103 .root_name = "Compile Build Script",
...@@ -5241,24 +5239,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,...@@ -5241,24 +5239,29 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5241 .io = io,5239 .io = io,
5242 .http_client = &http_client,5240 .http_client = &http_client,
5243 .global_cache = dirs.global_cache,5241 .global_cache = dirs.global_cache,
5242 .local_cache = .{ .root_dir = dirs.local_cache, .sub_path = "" },
5243 .root_pkg_path = .{ .root_dir = build_root.directory, .sub_path = "zig-pkg" },
5244 .read_only = false,5244 .read_only = false,
5245 .recursive = true,5245 .recursive = true,
5246 .debug_hash = false,5246 .debug_hash = false,
5247 .work_around_btrfs_bug = work_around_btrfs_bug,
5248 .unlazy_set = unlazy_set,5247 .unlazy_set = unlazy_set,
5249 .mode = fetch_mode,5248 .mode = fetch_mode,
5249 .prog_node = fetch_prog_node,
5250 };5250 };
5251 defer job_queue.deinit();5251 defer job_queue.deinit();
52525252
5253 if (system_pkg_dir_path) |p| {5253 if (system_pkg_dir_path) |p| {
5254 job_queue.global_cache = .{5254 const system_pkg_path: Path = .{
5255 .path = p,5255 .root_dir = .{
5256 .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {5256 .path = p,
5257 fatal("unable to open system package directory '{s}': {s}", .{5257 .handle = Io.Dir.cwd().openDir(io, p, .{}) catch |err| {
5258 p, @errorName(err),5258 fatal("unable to open system package directory '{s}': {t}", .{ p, err });
5259 });5259 },
5260 },5260 },
5261 .sub_path = "",
5261 };5262 };
5263 job_queue.global_cache = system_pkg_path.root_dir;
5264 job_queue.root_pkg_path = system_pkg_path;
5262 job_queue.read_only = true;5265 job_queue.read_only = true;
5263 cleanup_build_dir = job_queue.global_cache.handle;5266 cleanup_build_dir = job_queue.global_cache.handle;
5264 } else {5267 } else {
...@@ -5283,8 +5286,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,...@@ -5283,8 +5286,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, io: Io, args: []const []const u8,
5283 .job_queue = &job_queue,5286 .job_queue = &job_queue,
5284 .omit_missing_hash_error = true,5287 .omit_missing_hash_error = true,
5285 .allow_missing_paths_field = false,5288 .allow_missing_paths_field = false,
5286 .allow_missing_fingerprint = false,
5287 .allow_name_string = false,
5288 .use_latest_commit = false,5289 .use_latest_commit = false,
52895290
5290 .package_root = undefined,5291 .package_root = undefined,
...@@ -6938,8 +6939,6 @@ fn cmdFetch(...@@ -6938,8 +6939,6 @@ fn cmdFetch(
6938 dev.check(.fetch_command);6939 dev.check(.fetch_command);
69396940
6940 const color: Color = .auto;6941 const color: Color = .auto;
6941 const work_around_btrfs_bug = native_os == .linux and
6942 EnvVar.ZIG_BTRFS_WORKAROUND.isSet(environ_map);
6943 var opt_path_or_url: ?[]const u8 = null;6942 var opt_path_or_url: ?[]const u8 = null;
6944 var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);6943 var override_global_cache_dir: ?[]const u8 = EnvVar.ZIG_GLOBAL_CACHE_DIR.get(environ_map);
6945 var debug_hash: bool = false;6944 var debug_hash: bool = false;
...@@ -7003,15 +7002,32 @@ fn cmdFetch(...@@ -7003,15 +7002,32 @@ fn cmdFetch(
7003 };7002 };
7004 defer global_cache_directory.handle.close(io);7003 defer global_cache_directory.handle.close(io);
70057004
7005 const cwd_path = try introspect.getResolvedCwd(io, arena);
7006
7007 var build_root = try findBuildRoot(arena, io, .{
7008 .cwd_path = cwd_path,
7009 });
7010 defer build_root.deinit(io);
7011
7012 const local_cache_path: Path = .{
7013 .root_dir = build_root.directory,
7014 .sub_path = ".zig-cache",
7015 };
7016
7006 var job_queue: Package.Fetch.JobQueue = .{7017 var job_queue: Package.Fetch.JobQueue = .{
7007 .io = io,7018 .io = io,
7008 .http_client = &http_client,7019 .http_client = &http_client,
7009 .global_cache = global_cache_directory,7020 .global_cache = global_cache_directory,
7021 .local_cache = local_cache_path,
7022 .root_pkg_path = .{
7023 .root_dir = build_root.directory,
7024 .sub_path = "zig-pkg",
7025 },
7010 .recursive = false,7026 .recursive = false,
7011 .read_only = false,7027 .read_only = false,
7012 .debug_hash = debug_hash,7028 .debug_hash = debug_hash,
7013 .work_around_btrfs_bug = work_around_btrfs_bug,
7014 .mode = .all,7029 .mode = .all,
7030 .prog_node = root_prog_node,
7015 };7031 };
7016 defer job_queue.deinit();7032 defer job_queue.deinit();
70177033
...@@ -7028,8 +7044,6 @@ fn cmdFetch(...@@ -7028,8 +7044,6 @@ fn cmdFetch(
7028 .job_queue = &job_queue,7044 .job_queue = &job_queue,
7029 .omit_missing_hash_error = true,7045 .omit_missing_hash_error = true,
7030 .allow_missing_paths_field = false,7046 .allow_missing_paths_field = false,
7031 .allow_missing_fingerprint = true,
7032 .allow_name_string = true,
7033 .use_latest_commit = true,7047 .use_latest_commit = true,
70347048
7035 .package_root = undefined,7049 .package_root = undefined,
...@@ -7077,13 +7091,6 @@ fn cmdFetch(...@@ -7077,13 +7091,6 @@ fn cmdFetch(
7077 },7091 },
7078 };7092 };
70797093
7080 const cwd_path = try introspect.getResolvedCwd(io, arena);
7081
7082 var build_root = try findBuildRoot(arena, io, .{
7083 .cwd_path = cwd_path,
7084 });
7085 defer build_root.deinit(io);
7086
7087 // The name to use in case the manifest file needs to be created now.7094 // The name to use in case the manifest file needs to be created now.
7088 const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);7095 const init_root_name = fs.path.basename(build_root.directory.path orelse cwd_path);
7089 var manifest, var ast = try loadManifest(gpa, arena, io, .{7096 var manifest, var ast = try loadManifest(gpa, arena, io, .{
...@@ -7247,18 +7254,25 @@ fn createDependenciesModule(...@@ -7247,18 +7254,25 @@ fn createDependenciesModule(
7247 defer tmp_dir.close(io);7254 defer tmp_dir.close(io);
7248 try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source });7255 try tmp_dir.writeFile(io, .{ .sub_path = basename, .data = source });
7249 }7256 }
7257 const tmp_dir_path: Path = .{
7258 .root_dir = dirs.local_cache,
7259 .sub_path = tmp_dir_sub_path,
7260 };
72507261
7251 var hh: Cache.HashHelper = .{};7262 var hh: Cache.HashHelper = .{};
7252 hh.addBytes(build_options.version);7263 hh.addBytes(build_options.version);
7253 hh.addBytes(source);7264 hh.addBytes(source);
7254 const hex_digest = hh.final();7265 const hex_digest = hh.final();
72557266
7256 const o_dir_sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest);7267 const o_dir_path: Path = .{
7257 try Package.Fetch.renameTmpIntoCache(io, dirs.local_cache.handle, tmp_dir_sub_path, o_dir_sub_path);7268 .root_dir = dirs.local_cache,
7269 .sub_path = try arena.dupe(u8, "o" ++ fs.path.sep_str ++ hex_digest),
7270 };
7271 try Package.Fetch.renameTmpIntoCache(io, tmp_dir_path, o_dir_path);
72587272
7259 const deps_mod = try Package.Module.create(arena, .{7273 const deps_mod = try Package.Module.create(arena, .{
7260 .paths = .{7274 .paths = .{
7261 .root = try .fromRoot(arena, dirs, .local_cache, o_dir_sub_path),7275 .root = try .fromRoot(arena, dirs, .local_cache, o_dir_path.sub_path),
7262 .root_src_path = basename,7276 .root_src_path = basename,
7263 },7277 },
7264 .fully_qualified_name = "root.@dependencies",7278 .fully_qualified_name = "root.@dependencies",