authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-08 14:33:39-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-10-08 16:54:31-07:00
log47a413361dd6702dd0412ed58003e9ca9d8ba928
tree84a34f34c649e765c3bedc74f105b287ffe6cc45
parent1fd95fc00519ee44de94fa5b73ca7a3285b92149

Package.Fetch: fix handling of relative paths

3 files changed, 54 insertions(+), 37 deletions(-)

lib/std/Build/Cache.zig+1-1
......@@ -61,7 +61,7 @@ pub const Directory = struct {
6161 writer: anytype,
6262 ) !void {
6363 _ = options;
64 if (fmt_string.len != 0) fmt.invalidFmtError(fmt, self);
64 if (fmt_string.len != 0) fmt.invalidFmtError(fmt_string, self);
6565 if (self.path) |p| {
6666 try writer.writeAll(p);
6767 try writer.writeAll(fs.path.sep_str);
src/Package/Fetch.zig+46-35
......@@ -200,7 +200,7 @@ pub const JobQueue = struct {
200200pub const Location = union(enum) {
201201 remote: Remote,
202202 /// A directory found inside the parent package.
203 relative_path: []const u8,
203 relative_path: Package.Path,
204204 /// Recursive Fetch tasks will never use this Location, but it may be
205205 /// passed in by the CLI. Indicates the file contents here should be copied
206206 /// into the global package cache. It may be a file relative to the cwd or
......@@ -239,8 +239,8 @@ pub fn run(f: *Fetch) RunError!void {
239239 // relative path, treat this the same as a cache hit. Otherwise, proceed.
240240
241241 const remote = switch (f.location) {
242 .relative_path => |sub_path| {
243 if (fs.path.isAbsolute(sub_path)) return f.fail(
242 .relative_path => |pkg_root| {
243 if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
244244 f.location_tok,
245245 try eb.addString("expected path relative to build root; found absolute path"),
246246 );
......@@ -248,31 +248,19 @@ pub fn run(f: *Fetch) RunError!void {
248248 f.hash_tok,
249249 try eb.addString("path-based dependencies are not hashed"),
250250 );
251 f.package_root = try f.parent_package_root.resolvePosix(arena, sub_path);
252 if (std.mem.startsWith(u8, f.package_root.sub_path, "../")) {
251 if (std.mem.startsWith(u8, pkg_root.sub_path, "../")) {
253252 return f.fail(
254253 f.location_tok,
255 try eb.addString("dependency path outside package"),
254 try eb.printString("dependency path outside project: '{}{s}'", .{
255 pkg_root.root_dir, pkg_root.sub_path,
256 }),
256257 );
257258 }
258 try loadManifest(f, f.package_root);
259 f.package_root = pkg_root;
260 try loadManifest(f, pkg_root);
259261 try checkBuildFileExistence(f);
260262 if (!f.job_queue.recursive) return;
261 // Package hashes are used as unique identifiers for packages, so
262 // we still need one for relative paths.
263 const digest = h: {
264 var hasher = Manifest.Hash.init(.{});
265 // This hash is a tuple of:
266 // * whether it relative to the global cache directory or to the root package
267 // * the relative file path from there to the build root of the package
268 hasher.update(if (f.package_root.root_dir.eql(cache_root))
269 &package_hash_prefix_cached
270 else
271 &package_hash_prefix_project);
272 hasher.update(f.package_root.sub_path);
273 break :h hasher.finalResult();
274 };
275 return queueJobsForDeps(f, Manifest.hexDigest(digest));
263 return queueJobsForDeps(f);
276264 },
277265 .remote => |remote| remote,
278266 .path_or_url => |path_or_url| {
......@@ -310,7 +298,7 @@ pub fn run(f: *Fetch) RunError!void {
310298 try loadManifest(f, f.package_root);
311299 try checkBuildFileExistence(f);
312300 if (!f.job_queue.recursive) return;
313 return queueJobsForDeps(f, expected_hash);
301 return queueJobsForDeps(f);
314302 } else |err| switch (err) {
315303 error.FileNotFound => {},
316304 else => |e| {
......@@ -450,7 +438,7 @@ fn runResource(
450438 // Spawn a new fetch job for each dependency in the manifest file. Use
451439 // a mutex and a hash map so that redundant jobs do not get queued up.
452440 if (!f.job_queue.recursive) return;
453 return queueJobsForDeps(f, actual_hex);
441 return queueJobsForDeps(f);
454442}
455443
456444/// `computeHash` gets a free check for the existence of `build.zig`, but when
......@@ -534,27 +522,29 @@ fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void {
534522 }
535523}
536524
537fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void {
525fn queueJobsForDeps(f: *Fetch) RunError!void {
538526 assert(f.job_queue.recursive);
539527
540528 // If the package does not have a build.zig.zon file then there are no dependencies.
541529 const manifest = f.manifest orelse return;
542530
543531 const new_fetches = nf: {
544 const deps = manifest.dependencies.values();
532 const parent_arena = f.arena.allocator();
545533 const gpa = f.arena.child_allocator;
534 const cache_root = f.job_queue.global_cache;
535 const deps = manifest.dependencies.values();
546536 // Grab the new tasks into a temporary buffer so we can unlock that mutex
547537 // as fast as possible.
548538 // This overallocates any fetches that get skipped by the `continue` in the
549539 // loop below.
550 const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len);
540 const new_fetches = try parent_arena.alloc(Fetch, deps.len);
551541 var new_fetch_index: usize = 0;
552542
553543 f.job_queue.mutex.lock();
554544 defer f.job_queue.mutex.unlock();
555545
556546 try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
557 try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1));
547 try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
558548
559549 // There are four cases here:
560550 // * Correct hash is provided by manifest.
......@@ -564,12 +554,8 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void
564554 // * Hash is not provided by manifest.
565555 // - Hash missing error emitted; `queueJobsForDeps` is not called.
566556 // * path-based location is used without a hash.
567 // - We need to add `hash` to the table now.
568 switch (f.location) {
569 .remote => assert(f.job_queue.table.get(hash) == f),
570 .relative_path => f.job_queue.table.putAssumeCapacityNoClobber(hash, f),
571 .path_or_url => unreachable,
572 }
557 // - Hash is added to the table based on the path alone before
558 // calling run(); no need to add it again.
573559
574560 for (deps) |dep| {
575561 const new_fetch = &new_fetches[new_fetch_index];
......@@ -586,7 +572,16 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void
586572 break :h multihash_digest;
587573 },
588574 } },
589 .path => |path| .{ .relative_path = path },
575 .path => |rel_path| l: {
576 // This might produce an invalid path, which is checked for
577 // at the beginning of run().
578 const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
579 const multihash_digest = relativePathDigest(new_root, cache_root);
580 const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
581 if (gop.found_existing) continue;
582 gop.value_ptr.* = new_fetch;
583 break :l .{ .relative_path = new_root };
584 },
590585 };
591586 new_fetch_index += 1;
592587 f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
......@@ -630,6 +625,22 @@ fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void
630625 }
631626}
632627
628pub fn relativePathDigest(
629 pkg_root: Package.Path,
630 cache_root: Cache.Directory,
631) Manifest.MultiHashHexDigest {
632 var hasher = Manifest.Hash.init(.{});
633 // This hash is a tuple of:
634 // * whether it relative to the global cache directory or to the root package
635 // * the relative file path from there to the build root of the package
636 hasher.update(if (pkg_root.root_dir.eql(cache_root))
637 &package_hash_prefix_cached
638 else
639 &package_hash_prefix_project);
640 hasher.update(pkg_root.sub_path);
641 return Manifest.hexDigest(hasher.finalResult());
642}
643
633644pub fn workerRun(f: *Fetch) void {
634645 defer f.job_queue.wait_group.finish();
635646 run(f) catch |err| switch (err) {
src/main.zig+7-1
......@@ -4851,10 +4851,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
48514851 defer job_queue.deinit();
48524852
48534853 try job_queue.all_fetches.ensureUnusedCapacity(gpa, 1);
4854 try job_queue.table.ensureUnusedCapacity(gpa, 1);
48544855
48554856 var fetch: Package.Fetch = .{
48564857 .arena = std.heap.ArenaAllocator.init(gpa),
4857 .location = .{ .relative_path = "" },
4858 .location = .{ .relative_path = build_mod.root },
48584859 .location_tok = 0,
48594860 .hash_tok = 0,
48604861 .parent_package_root = build_mod.root,
......@@ -4874,6 +4875,11 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
48744875 };
48754876 job_queue.all_fetches.appendAssumeCapacity(&fetch);
48764877
4878 job_queue.table.putAssumeCapacityNoClobber(
4879 Package.Fetch.relativePathDigest(build_mod.root, global_cache_directory),
4880 &fetch,
4881 );
4882
48774883 job_queue.wait_group.start();
48784884 try job_queue.thread_pool.spawn(Package.Fetch.workerRun, .{&fetch});
48794885 job_queue.wait_group.wait();