Commit d0bcc390e8f61ada470b524e3fd203c1af521a99 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-06 17:05:58-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-10-08 16:54:31-07:00
log	d0bcc390e8f61ada470b524e3fd203c1af521a99
tree	14503dd0b9cb5b3666b6c397f3449c7931bbcb08
parent	88bbec8f9b2f8f023a0177c204f51b8ac0aee83a

get `zig fetch` working with the new system

* start renaming "package" to "module" (see #14307)
  - build system gains `main_mod_path` and `main_pkg_path` is still
    there but it is deprecated.
* eliminate the object-oriented memory management style of what was
  previously `*Package`. Now it is `*Package.Module` and all pointers
  point to externally managed memory.
* fixes to get the new Fetch.zig code working. The previous commit was
  work-in-progress. There are still two commented out code paths, the
  one that leads to `Compilation.create` and the one for `zig build`
  that fetches the entire dependency tree and creates the required
  modules for the build runner.

12 files changed, 883 insertions(+), 750 deletions(-)

build.zig
lib
- std
  - Build.zig
  - Build
    Cache.zig
    Step
    Compile.zig
src

build.zig+1-1

...	@@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void {	...	@@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void {
88	.name = "check-case",	88	.name = "check-case",
89	.root_source_file = .{ .path = "test/src/Cases.zig" },	89	.root_source_file = .{ .path = "test/src/Cases.zig" },
90	.optimize = optimize,	90	.optimize = optimize,
91	.main_pkg_path = .{ .path = "." },	91	.main_mod_path = .{ .path = "." },
92	});	92	});
93	check_case_exe.stack_size = stack_size;	93	check_case_exe.stack_size = stack_size;
94	check_case_exe.single_threaded = single_threaded;	94	check_case_exe.single_threaded = single_threaded;

lib/std/Build.zig+20-5

...	@@ -634,6 +634,9 @@ pub const ExecutableOptions = struct {	...	@@ -634,6 +634,9 @@ pub const ExecutableOptions = struct {
634	use_llvm: ?bool = null,	634	use_llvm: ?bool = null,
635	use_lld: ?bool = null,	635	use_lld: ?bool = null,
636	zig_lib_dir: ?LazyPath = null,	636	zig_lib_dir: ?LazyPath = null,
		637	main_mod_path: ?LazyPath = null,
		638
		639	/// Deprecated; use `main_mod_path`.
637	main_pkg_path: ?LazyPath = null,	640	main_pkg_path: ?LazyPath = null,
638	};	641	};
639		642
...	@@ -652,7 +655,7 @@ pub fn addExecutable(b: Build, options: ExecutableOptions) Step.Compile {	...	@@ -652,7 +655,7 @@ pub fn addExecutable(b: Build, options: ExecutableOptions) Step.Compile {
652	.use_llvm = options.use_llvm,	655	.use_llvm = options.use_llvm,
653	.use_lld = options.use_lld,	656	.use_lld = options.use_lld,
654	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,	657	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
655	.main_pkg_path = options.main_pkg_path,	658	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
656	});	659	});
657	}	660	}
658		661
...	@@ -667,6 +670,9 @@ pub const ObjectOptions = struct {	...	@@ -667,6 +670,9 @@ pub const ObjectOptions = struct {
667	use_llvm: ?bool = null,	670	use_llvm: ?bool = null,
668	use_lld: ?bool = null,	671	use_lld: ?bool = null,
669	zig_lib_dir: ?LazyPath = null,	672	zig_lib_dir: ?LazyPath = null,
		673	main_mod_path: ?LazyPath = null,
		674
		675	/// Deprecated; use `main_mod_path`.
670	main_pkg_path: ?LazyPath = null,	676	main_pkg_path: ?LazyPath = null,
671	};	677	};
672		678
...	@@ -683,7 +689,7 @@ pub fn addObject(b: Build, options: ObjectOptions) Step.Compile {	...	@@ -683,7 +689,7 @@ pub fn addObject(b: Build, options: ObjectOptions) Step.Compile {
683	.use_llvm = options.use_llvm,	689	.use_llvm = options.use_llvm,
684	.use_lld = options.use_lld,	690	.use_lld = options.use_lld,
685	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,	691	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
686	.main_pkg_path = options.main_pkg_path,	692	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
687	});	693	});
688	}	694	}
689		695
...	@@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct {	...	@@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct {
699	use_llvm: ?bool = null,	705	use_llvm: ?bool = null,
700	use_lld: ?bool = null,	706	use_lld: ?bool = null,
701	zig_lib_dir: ?LazyPath = null,	707	zig_lib_dir: ?LazyPath = null,
		708	main_mod_path: ?LazyPath = null,
		709
		710	/// Deprecated; use `main_mod_path`.
702	main_pkg_path: ?LazyPath = null,	711	main_pkg_path: ?LazyPath = null,
703	};	712	};
704		713
...	@@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: Build, options: SharedLibraryOptions) Step.Compile	...	@@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: Build, options: SharedLibraryOptions) Step.Compile
717	.use_llvm = options.use_llvm,	726	.use_llvm = options.use_llvm,
718	.use_lld = options.use_lld,	727	.use_lld = options.use_lld,
719	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,	728	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
720	.main_pkg_path = options.main_pkg_path,	729	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
721	});	730	});
722	}	731	}
723		732
...	@@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct {	...	@@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct {
733	use_llvm: ?bool = null,	742	use_llvm: ?bool = null,
734	use_lld: ?bool = null,	743	use_lld: ?bool = null,
735	zig_lib_dir: ?LazyPath = null,	744	zig_lib_dir: ?LazyPath = null,
		745	main_mod_path: ?LazyPath = null,
		746
		747	/// Deprecated; use `main_mod_path`.
736	main_pkg_path: ?LazyPath = null,	748	main_pkg_path: ?LazyPath = null,
737	};	749	};
738		750
...	@@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: Build, options: StaticLibraryOptions) Step.Compile	...	@@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: Build, options: StaticLibraryOptions) Step.Compile
751	.use_llvm = options.use_llvm,	763	.use_llvm = options.use_llvm,
752	.use_lld = options.use_lld,	764	.use_lld = options.use_lld,
753	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,	765	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
754	.main_pkg_path = options.main_pkg_path,	766	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
755	});	767	});
756	}	768	}
757		769
...	@@ -769,6 +781,9 @@ pub const TestOptions = struct {	...	@@ -769,6 +781,9 @@ pub const TestOptions = struct {
769	use_llvm: ?bool = null,	781	use_llvm: ?bool = null,
770	use_lld: ?bool = null,	782	use_lld: ?bool = null,
771	zig_lib_dir: ?LazyPath = null,	783	zig_lib_dir: ?LazyPath = null,
		784	main_mod_path: ?LazyPath = null,
		785
		786	/// Deprecated; use `main_mod_path`.
772	main_pkg_path: ?LazyPath = null,	787	main_pkg_path: ?LazyPath = null,
773	};	788	};
774		789
...	@@ -787,7 +802,7 @@ pub fn addTest(b: Build, options: TestOptions) Step.Compile {	...	@@ -787,7 +802,7 @@ pub fn addTest(b: Build, options: TestOptions) Step.Compile {
787	.use_llvm = options.use_llvm,	802	.use_llvm = options.use_llvm,
788	.use_lld = options.use_lld,	803	.use_lld = options.use_lld,
789	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,	804	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
790	.main_pkg_path = options.main_pkg_path,	805	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
791	});	806	});
792	}	807	}
793		808

lib/std/Build/Cache.zig+11

...	@@ -9,6 +9,13 @@ pub const Directory = struct {	...	@@ -9,6 +9,13 @@ pub const Directory = struct {
9	path: ?[]const u8,	9	path: ?[]const u8,
10	handle: fs.Dir,	10	handle: fs.Dir,
11		11
		12	pub fn cwd() Directory {
		13	return .{
		14	.path = null,
		15	.handle = fs.cwd(),
		16	};
		17	}
		18
12	pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {	19	pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
13	if (self.path) \|p\| {	20	if (self.path) \|p\| {
14	// TODO clean way to do this with only 1 allocation	21	// TODO clean way to do this with only 1 allocation
...	@@ -53,6 +60,10 @@ pub const Directory = struct {	...	@@ -53,6 +60,10 @@ pub const Directory = struct {
53	try writer.writeAll(fs.path.sep_str);	60	try writer.writeAll(fs.path.sep_str);
54	}	61	}
55	}	62	}
		63
		64	pub fn eql(self: Directory, other: Directory) bool {
		65	return self.handle.fd == other.handle.fd;
		66	}
56	};	67	};
57		68
58	gpa: Allocator,	69	gpa: Allocator,

lib/std/Build/Step/Compile.zig+9-6

...	@@ -68,7 +68,7 @@ c_std: std.Build.CStd,	...	@@ -68,7 +68,7 @@ c_std: std.Build.CStd,
68	/// Set via options; intended to be read-only after that.	68	/// Set via options; intended to be read-only after that.
69	zig_lib_dir: ?LazyPath,	69	zig_lib_dir: ?LazyPath,
70	/// Set via options; intended to be read-only after that.	70	/// Set via options; intended to be read-only after that.
71	main_pkg_path: ?LazyPath,	71	main_mod_path: ?LazyPath,
72	exec_cmd_args: ?[]const ?[]const u8,	72	exec_cmd_args: ?[]const ?[]const u8,
73	filter: ?[]const u8,	73	filter: ?[]const u8,
74	test_evented_io: bool = false,	74	test_evented_io: bool = false,
...	@@ -316,6 +316,9 @@ pub const Options = struct {	...	@@ -316,6 +316,9 @@ pub const Options = struct {
316	use_llvm: ?bool = null,	316	use_llvm: ?bool = null,
317	use_lld: ?bool = null,	317	use_lld: ?bool = null,
318	zig_lib_dir: ?LazyPath = null,	318	zig_lib_dir: ?LazyPath = null,
		319	main_mod_path: ?LazyPath = null,
		320
		321	/// deprecated; use `main_mod_path`.
319	main_pkg_path: ?LazyPath = null,	322	main_pkg_path: ?LazyPath = null,
320	};	323	};
321		324
...	@@ -480,7 +483,7 @@ pub fn create(owner: std.Build, options: Options) Compile {	...	@@ -480,7 +483,7 @@ pub fn create(owner: std.Build, options: Options) Compile {
480	.installed_headers = ArrayList(*Step).init(owner.allocator),	483	.installed_headers = ArrayList(*Step).init(owner.allocator),
481	.c_std = std.Build.CStd.C99,	484	.c_std = std.Build.CStd.C99,
482	.zig_lib_dir = null,	485	.zig_lib_dir = null,
483	.main_pkg_path = null,	486	.main_mod_path = null,
484	.exec_cmd_args = null,	487	.exec_cmd_args = null,
485	.filter = options.filter,	488	.filter = options.filter,
486	.test_runner = options.test_runner,	489	.test_runner = options.test_runner,
...	@@ -515,8 +518,8 @@ pub fn create(owner: std.Build, options: Options) Compile {	...	@@ -515,8 +518,8 @@ pub fn create(owner: std.Build, options: Options) Compile {
515	lp.addStepDependencies(&self.step);	518	lp.addStepDependencies(&self.step);
516	}	519	}
517		520
518	if (options.main_pkg_path) \|lp\| {	521	if (options.main_mod_path orelse options.main_pkg_path) \|lp\| {
519	self.main_pkg_path = lp.dupe(self.step.owner);	522	self.main_mod_path = lp.dupe(self.step.owner);
520	lp.addStepDependencies(&self.step);	523	lp.addStepDependencies(&self.step);
521	}	524	}
522		525
...	@@ -1998,8 +2001,8 @@ fn make(step: Step, prog_node: std.Progress.Node) !void {	...	@@ -1998,8 +2001,8 @@ fn make(step: Step, prog_node: std.Progress.Node) !void {
1998	try zig_args.append(dir.getPath(b));	2001	try zig_args.append(dir.getPath(b));
1999	}	2002	}
2000		2003
2001	if (self.main_pkg_path) \|dir\| {	2004	if (self.main_mod_path) \|dir\| {
2002	try zig_args.append("--main-pkg-path");	2005	try zig_args.append("--main-mod-path");
2003	try zig_args.append(dir.getPath(b));	2006	try zig_args.append(dir.getPath(b));
2004	}	2007	}
2005		2008

src/Compilation.zig+84-86

 @@ -273,8 +273,8 @@ const Job = union(enum) {
     /// The source file containing the Decl has been updated, and so the
     /// Decl may need its line number information updated in the debug info.
     update_line_number: Module.Decl.Index,
-    /// The main source file for the package needs to be analyzed.
+    /// The main source file for the module needs to be analyzed.
-    analyze_pkg: *Package,
+    analyze_mod: *Package.Module,
     /// one of the glibc static objects
     glibc_crt_file: glibc.CRTFile,
 @@ -414,7 +414,7 @@ pub const MiscTask = enum {
     compiler_rt,
     libssp,
     zig_libc,
-    analyze_pkg,
+    analyze_mod,
     @"musl crti.o",
     @"musl crtn.o",
 @@ -544,7 +544,7 @@ pub const InitOptions = struct {
     global_cache_directory: Directory,
     target: Target,
     root_name: []const u8,
-    main_pkg: ?*Package,
+    main_mod: ?*Package.Module,
     output_mode: std.builtin.OutputMode,
     thread_pool: *ThreadPool,
     dynamic_linker: ?[]const u8 = null,
 @@ -736,53 +736,53 @@ pub const InitOptions = struct {
     pdb_out_path: ?[]const u8 = null,
 };
-fn addPackageTableToCacheHash(
+fn addModuleTableToCacheHash(
     hash: *Cache.HashHelper,
     arena: *std.heap.ArenaAllocator,
-    pkg_table: Package.Table,
+    mod_table: Package.Module.Deps,
-    seen_table: *std.AutoHashMap(*Package, void),
+    seen_table: *std.AutoHashMap(*Package.Module, void),
     hash_type: union(enum) { path_bytes, files: *Cache.Manifest },
 ) (error{OutOfMemory} || std.os.GetCwdError)!void {
     const allocator = arena.allocator();
-    const packages = try allocator.alloc(Package.Table.KV, pkg_table.count());
+    const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count());
     {
         // Copy over the hashmap entries to our slice
-        var table_it = pkg_table.iterator();
+        var table_it = mod_table.iterator();
         var idx: usize = 0;
         while (table_it.next()) |entry| : (idx += 1) {
-            packages[idx] = .{
+            modules[idx] = .{
                 .key = entry.key_ptr.*,
                 .value = entry.value_ptr.*,
             };
         }
     }
     // Sort the slice by package name
-    mem.sort(Package.Table.KV, packages, {}, struct {
+    mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct {
-        fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool {
+        fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool {
             return std.mem.lessThan(u8, lhs.key, rhs.key);
         }
     }.lessThan);
-    for (packages) |pkg| {
+    for (modules) |mod| {
-        if ((try seen_table.getOrPut(pkg.value)).found_existing) continue;
+        if ((try seen_table.getOrPut(mod.value)).found_existing) continue;
         // Finally insert the package name and path to the cache hash.
-        hash.addBytes(pkg.key);
+        hash.addBytes(mod.key);
         switch (hash_type) {
             .path_bytes => {
-                hash.addBytes(pkg.value.root_src_path);
+                hash.addBytes(mod.value.root_src_path);
-                hash.addOptionalBytes(pkg.value.root_src_directory.path);
+                hash.addOptionalBytes(mod.value.root_src_directory.path);
             },
             .files => |man| {
-                const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{
+                const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{
-                    pkg.value.root_src_path,
+                    mod.value.root_src_path,
                 });
                 _ = try man.addFile(pkg_zig_file, null);
             },
         }
-        // Recurse to handle the package's dependencies
+        // Recurse to handle the module's dependencies
-        try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type);
+        try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type);
     }
 }
 @@ -839,7 +839,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 break :blk true;
             // If we have no zig code to compile, no need for LLVM.
-            if (options.main_pkg == null)
+            if (options.main_mod == null)
                 break :blk false;
             // If LLVM does not support the target, then we can't use it.
 @@ -869,7 +869,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         // compiler state, the second clause here can be removed so that incremental
         // cache mode is used for LLVM backend too. We need some fuzz testing before
         // that can be enabled.
-        const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching)
+        const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching)
             CacheMode.whole
         else
             options.cache_mode;
 @@ -925,7 +925,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             if (use_llvm) {
                 // If stage1 generates an object file, self-hosted linker is not
                 // yet sophisticated enough to handle that.
-                break :blk options.main_pkg != null;
+                break :blk options.main_mod != null;
             }
             break :blk false;
 @@ -1210,7 +1210,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
         if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
         // TODO audit this and make sure everything is in it
-        const module: ?*Module = if (options.main_pkg) |main_pkg| blk: {
+        const module: ?*Module = if (options.main_mod) |main_mod| blk: {
             // Options that are specific to zig source files, that cannot be
             // modified between incremental updates.
             var hash = cache.hash;
 @@ -1223,11 +1223,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                     // do want to namespace different source file names because they are
                     // likely different compilations and therefore this would be likely to
                     // cause cache hits.
-                    hash.addBytes(main_pkg.root_src_path);
+                    hash.addBytes(main_mod.root_src_path);
-                    hash.addOptionalBytes(main_pkg.root_src_directory.path);
+                    hash.addOptionalBytes(main_mod.root.root_dir.path);
+                    hash.addBytes(main_mod.root.sub_path);
                     {
-                        var seen_table = std.AutoHashMap(*Package, void).init(arena);
+                        var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
-                        try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes);
+                        try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes);
                     }
                 },
                 .whole => {
 @@ -1283,34 +1284,31 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
             };
-            const builtin_pkg = try Package.createWithDir(
+            const builtin_mod = try Package.Module.create(arena, .{
-                gpa,
+                .root = .{ .root_dir = zig_cache_artifact_directory },
-                zig_cache_artifact_directory,
+                .root_src_path = "builtin.zig",
-                null,
+            });
-                "builtin.zig",
-            );
-            errdefer builtin_pkg.destroy(gpa);
-            // When you're testing std, the main module is std. In that case, we'll just set the std
+            // When you're testing std, the main module is std. In that case,
-            // module to the main one, since avoiding the errors caused by duplicating it is more
+            // we'll just set the std module to the main one, since avoiding
-            // effort than it's worth.
+            // the errors caused by duplicating it is more effort than it's
-            const main_pkg_is_std = m: {
+            // worth.
+            const main_mod_is_std = m: {
                 const std_path = try std.fs.path.resolve(arena, &[_][]const u8{
                     options.zig_lib_directory.path orelse ".",
                     "std",
                     "std.zig",
                 });
-                defer arena.free(std_path);
                 const main_path = try std.fs.path.resolve(arena, &[_][]const u8{
-                    main_pkg.root_src_directory.path orelse ".",
+                    main_mod.root.root_dir.path orelse ".",
-                    main_pkg.root_src_path,
+                    main_mod.root.sub_path,
+                    main_mod.root_src_path,
                 });
-                defer arena.free(main_path);
                 break :m mem.eql(u8, main_path, std_path);
             };
-            const std_pkg = if (main_pkg_is_std)
+            const std_mod = if (main_mod_is_std)
-                main_pkg
+                main_mod
             else
                 try Package.createWithDir(
                     gpa,
 @@ -1319,16 +1317,16 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                     "std.zig",
                 );
-            errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa);
+            errdefer if (!main_mod_is_std) std_mod.destroy(gpa);
-            const root_pkg = if (options.is_test) root_pkg: {
+            const root_mod = if (options.is_test) root_mod: {
                 const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: {
                     const test_dir = std.fs.path.dirname(test_runner);
                     const basename = std.fs.path.basename(test_runner);
                     const pkg = try Package.create(gpa, test_dir, basename);
-                    // copy package table from main_pkg to root_pkg
+                    // copy module table from main_mod to root_mod
-                    pkg.table = try main_pkg.table.clone(gpa);
+                    pkg.deps = try main_mod.deps.clone(gpa);
                     break :test_pkg pkg;
                 } else try Package.createWithDir(
                     gpa,
 @@ -1338,26 +1336,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
                 );
                 errdefer test_pkg.destroy(gpa);
-                break :root_pkg test_pkg;
+                break :root_mod test_pkg;
-            } else main_pkg;
+            } else main_mod;
-            errdefer if (options.is_test) root_pkg.destroy(gpa);
+            errdefer if (options.is_test) root_mod.destroy(gpa);
-            const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: {
+            const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: {
-                break :compiler_rt_pkg try Package.createWithDir(
+                break :compiler_rt_mod try Package.createWithDir(
                     gpa,
                     options.zig_lib_directory,
                     null,
                     "compiler_rt.zig",
                 );
             } else null;
-            errdefer if (compiler_rt_pkg) |p| p.destroy(gpa);
+            errdefer if (compiler_rt_mod) |p| p.destroy(gpa);
-            try main_pkg.add(gpa, "builtin", builtin_pkg);
+            try main_mod.add(gpa, "builtin", builtin_mod);
-            try main_pkg.add(gpa, "root", root_pkg);
+            try main_mod.add(gpa, "root", root_mod);
-            try main_pkg.add(gpa, "std", std_pkg);
+            try main_mod.add(gpa, "std", std_mod);
-            if (compiler_rt_pkg) |p| {
+            if (compiler_rt_mod) |p| {
-                try main_pkg.add(gpa, "compiler_rt", p);
+                try main_mod.add(gpa, "compiler_rt", p);
             }
             // Pre-open the directory handles for cached ZIR code so that it does not need
 @@ -1395,8 +1393,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
             module.* = .{
                 .gpa = gpa,
                 .comp = comp,
-                .main_pkg = main_pkg,
+                .main_mod = main_mod,
-                .root_pkg = root_pkg,
+                .root_mod = root_mod,
                 .zig_cache_artifact_directory = zig_cache_artifact_directory,
                 .global_zir_cache = global_zir_cache,
                 .local_zir_cache = local_zir_cache,
 @@ -2005,8 +2003,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director
     // This is only for cleanup purposes; Module.deinit calls close
     // on the handle of zig_cache_artifact_directory.
     if (comp.bin_file.options.module) |module| {
-        const builtin_pkg = module.main_pkg.table.get("builtin").?;
+        const builtin_mod = module.main_mod.deps.get("builtin").?;
-        module.zig_cache_artifact_directory = builtin_pkg.root_src_directory;
+        module.zig_cache_artifact_directory = builtin_mod.root_src_directory;
     }
 }
 @@ -2148,8 +2146,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
         // Make sure std.zig is inside the import_table. We unconditionally need
         // it for start.zig.
-        const std_pkg = module.main_pkg.table.get("std").?;
+        const std_mod = module.main_mod.deps.get("std").?;
-        _ = try module.importPkg(std_pkg);
+        _ = try module.importPkg(std_mod);
         // Normally we rely on importing std to in turn import the root source file
         // in the start code, but when using the stage1 backend that won't happen,
 @@ -2158,11 +2156,11 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
         // Likewise, in the case of `zig test`, the test runner is the root source file,
         // and so there is nothing to import the main file.
         if (comp.bin_file.options.is_test) {
-            _ = try module.importPkg(module.main_pkg);
+            _ = try module.importPkg(module.main_mod);
         }
-        if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| {
+        if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| {
-            _ = try module.importPkg(compiler_rt_pkg);
+            _ = try module.importPkg(compiler_rt_mod);
         }
         // Put a work item in for every known source file to detect if
 @@ -2185,13 +2183,13 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void
             }
         }
-        try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
+        try comp.work_queue.writeItem(.{ .analyze_mod = std_mod });
         if (comp.bin_file.options.is_test) {
-            try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
+            try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod });
         }
-        if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| {
+        if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| {
-            try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg });
+            try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod });
         }
     }
 @@ -2420,19 +2418,19 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes
     comptime assert(link_hash_implementation_version == 10);
     if (comp.bin_file.options.module) |mod| {
-        const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{
+        const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{
-            mod.main_pkg.root_src_path,
+            mod.main_mod.root_src_path,
         });
         _ = try man.addFile(main_zig_file, null);
         {
-            var seen_table = std.AutoHashMap(*Package, void).init(arena);
+            var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
             // Skip builtin.zig; it is useless as an input, and we don't want to have to
             // write it before checking for a cache hit.
-            const builtin_pkg = mod.main_pkg.table.get("builtin").?;
+            const builtin_mod = mod.main_mod.deps.get("builtin").?;
-            try seen_table.put(builtin_pkg, {});
+            try seen_table.put(builtin_mod, {});
-            try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
+            try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man });
         }
         // Synchronize with other matching comments: ZigOnlyHashStuff
 @@ -3564,8 +3562,8 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
                 decl.analysis = .codegen_failure_retryable;
             };
         },
-        .analyze_pkg => |pkg| {
+        .analyze_mod => |pkg| {
-            const named_frame = tracy.namedFrame("analyze_pkg");
+            const named_frame = tracy.namedFrame("analyze_mod");
             defer named_frame.end();
             const module = comp.bin_file.options.module.?;
 @@ -6379,11 +6377,11 @@ fn buildOutputFromZig(
     std.debug.assert(output_mode != .Exe);
-    var main_pkg: Package = .{
+    var main_mod: Package = .{
         .root_src_directory = comp.zig_lib_directory,
         .root_src_path = src_basename,
     };
-    defer main_pkg.deinitTable(comp.gpa);
+    defer main_mod.deinitTable(comp.gpa);
     const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
     const target = comp.getTarget();
     const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
 @@ -6404,7 +6402,7 @@ fn buildOutputFromZig(
         .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
-        .main_pkg = &main_pkg,
+        .main_mod = &main_mod,
         .output_mode = output_mode,
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,
 @@ -6481,7 +6479,7 @@ pub fn build_crt_file(
         .cache_mode = .whole,
         .target = target,
         .root_name = root_name,
-        .main_pkg = null,
+        .main_mod = null,
         .output_mode = output_mode,
         .thread_pool = comp.thread_pool,
         .libc_installation = comp.bin_file.options.libc_installation,

src/Manifest.zig+8-6

...	@@ -1,6 +1,10 @@	...	@@ -1,6 +1,10 @@
1	pub const max_bytes = 10 * 1024 * 1024;	1	pub const max_bytes = 10 * 1024 * 1024;
2	pub const basename = "build.zig.zon";	2	pub const basename = "build.zig.zon";
3	pub const Hash = std.crypto.hash.sha2.Sha256;	3	pub const Hash = std.crypto.hash.sha2.Sha256;
		4	pub const Digest = [Hash.digest_length]u8;
		5	pub const multihash_len = 1 + 1 + Hash.digest_length;
		6	pub const multihash_hex_digest_len = 2 * multihash_len;
		7	pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
4		8
5	pub const Dependency = struct {	9	pub const Dependency = struct {
6	location: union(enum) {	10	location: union(enum) {
...	@@ -46,7 +50,6 @@ comptime {	...	@@ -46,7 +50,6 @@ comptime {
46	assert(@intFromEnum(multihash_function) < 127);	50	assert(@intFromEnum(multihash_function) < 127);
47	assert(Hash.digest_length < 127);	51	assert(Hash.digest_length < 127);
48	}	52	}
49	pub const multihash_len = 1 + 1 + Hash.digest_length;
50		53
51	name: []const u8,	54	name: []const u8,
52	version: std.SemanticVersion,	55	version: std.SemanticVersion,
...	@@ -122,8 +125,8 @@ test hex64 {	...	@@ -122,8 +125,8 @@ test hex64 {
122	try std.testing.expectEqualStrings("[00efcdab78563412]", s);	125	try std.testing.expectEqualStrings("[00efcdab78563412]", s);
123	}	126	}
124		127
125	pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {	128	pub fn hexDigest(digest: Digest) MultiHashHexDigest {
126	var result: [multihash_len * 2]u8 = undefined;	129	var result: MultiHashHexDigest = undefined;
127		130
128	result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];	131	result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
129	result[1] = hex_charset[@intFromEnum(multihash_function) & 15];	132	result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
...	@@ -339,10 +342,9 @@ const Parse = struct {	...	@@ -339,10 +342,9 @@ const Parse = struct {
339	}	342	}
340	}	343	}
341		344
342	const hex_multihash_len = 2 * Manifest.multihash_len;	345	if (h.len != multihash_hex_digest_len) {
343	if (h.len != hex_multihash_len) {
344	return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{	346	return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
345	hex_multihash_len, h.len,	347	multihash_hex_digest_len, h.len,
346	});	348	});
347	}	349	}
348		350

src/Module.zig+37-44

 @@ -55,10 +55,10 @@ comp: *Compilation,
 /// Where build artifacts and incremental compilation metadata serialization go.
 zig_cache_artifact_directory: Compilation.Directory,
 /// Pointer to externally managed resource.
-root_pkg: *Package,
+root_mod: *Package.Module,
-/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which
+/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which
-/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests.
+/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests.
-main_pkg: *Package,
+main_mod: *Package.Module,
 sema_prog_node: std.Progress.Node = undefined,
 /// Used by AstGen worker to load and store ZIR cache.
 @@ -973,8 +973,8 @@ pub const File = struct {
     tree: Ast,
     /// Whether this is populated or not depends on `zir_loaded`.
     zir: Zir,
-    /// Package that this file is a part of, managed externally.
+    /// Module that this file is a part of, managed externally.
-    pkg: *Package,
+    mod: *Package.Module,
     /// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen.
     multi_pkg: bool = false,
     /// List of references to this file, used for multi-package errors.
 @@ -1058,14 +1058,9 @@ pub const File = struct {
             .stat = file.stat,
         };
-        const root_dir_path = file.pkg.root_src_directory.path orelse ".";
-        log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{
-            root_dir_path, file.sub_file_path,
-        });
         // Keep track of inode, file size, mtime, hash so we can detect which files
         // have been modified when an incremental update is requested.
-        var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{});
+        var f = try file.mod.root.openFile(file.sub_file_path, .{});
         defer f.close();
         const stat = try f.stat();
 @@ -1134,14 +1129,12 @@ pub const File = struct {
         return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start);
     }
-    /// Returns the full path to this file relative to its package.
     pub fn fullPath(file: File, ally: Allocator) ![]u8 {
-        return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path});
+        return file.mod.root.joinString(ally, file.sub_file_path);
     }
-    /// Returns the full path to this file relative to its package.
     pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 {
-        return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path});
+        return file.mod.root.joinStringZ(ally, file.sub_file_path);
     }
     pub fn dumpSrc(file: *File, src: LazySrcLoc) void {
 @@ -2543,25 +2536,25 @@ pub fn deinit(mod: *Module) void {
     mod.deletion_set.deinit(gpa);
-    // The callsite of `Compilation.create` owns the `main_pkg`, however
+    // The callsite of `Compilation.create` owns the `main_mod`, however
     // Module owns the builtin and std packages that it adds.
-    if (mod.main_pkg.table.fetchRemove("builtin")) |kv| {
+    if (mod.main_mod.table.fetchRemove("builtin")) |kv| {
         gpa.free(kv.key);
         kv.value.destroy(gpa);
     }
-    if (mod.main_pkg.table.fetchRemove("std")) |kv| {
+    if (mod.main_mod.table.fetchRemove("std")) |kv| {
         gpa.free(kv.key);
-        // It's possible for main_pkg to be std when running 'zig test'! In this case, we must not
+        // It's possible for main_mod to be std when running 'zig test'! In this case, we must not
         // destroy it, since it would lead to a double-free.
-        if (kv.value != mod.main_pkg) {
+        if (kv.value != mod.main_mod) {
             kv.value.destroy(gpa);
         }
     }
-    if (mod.main_pkg.table.fetchRemove("root")) |kv| {
+    if (mod.main_mod.table.fetchRemove("root")) |kv| {
         gpa.free(kv.key);
     }
-    if (mod.root_pkg != mod.main_pkg) {
+    if (mod.root_mod != mod.main_mod) {
-        mod.root_pkg.destroy(gpa);
+        mod.root_mod.destroy(gpa);
     }
     mod.compile_log_text.deinit(gpa);
 @@ -2715,7 +2708,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
     const stat = try source_file.stat();
-    const want_local_cache = file.pkg == mod.main_pkg;
+    const want_local_cache = file.pkg == mod.main_mod;
     const digest = hash: {
         var path_hash: Cache.HashHelper = .{};
         path_hash.addBytes(build_options.version);
 @@ -3158,23 +3151,23 @@ pub fn populateBuiltinFile(mod: *Module) !void {
         comp.mutex.lock();
         defer comp.mutex.unlock();
-        const builtin_pkg = mod.main_pkg.table.get("builtin").?;
+        const builtin_mod = mod.main_mod.table.get("builtin").?;
-        const result = try mod.importPkg(builtin_pkg);
+        const result = try mod.importPkg(builtin_mod);
         break :blk .{
             .file = result.file,
-            .pkg = builtin_pkg,
+            .pkg = builtin_mod,
         };
     };
     const file = pkg_and_file.file;
-    const builtin_pkg = pkg_and_file.pkg;
+    const builtin_mod = pkg_and_file.pkg;
     const gpa = mod.gpa;
     file.source = try comp.generateBuiltinZigSource(gpa);
     file.source_loaded = true;
-    if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) |stat| {
+    if (builtin_mod.root_src_directory.handle.statFile(builtin_mod.root_src_path)) |stat| {
         if (stat.size != file.source.len) {
-            const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{
+            const full_path = try builtin_mod.root_src_directory.join(gpa, &.{
-                builtin_pkg.root_src_path,
+                builtin_mod.root_src_path,
             });
             defer gpa.free(full_path);
 @@ -3184,7 +3177,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
                 .{ full_path, file.source.len, stat.size },
             );
-            try writeBuiltinFile(file, builtin_pkg);
+            try writeBuiltinFile(file, builtin_mod);
         } else {
             file.stat = .{
                 .size = stat.size,
 @@ -3198,7 +3191,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
         error.PipeBusy => unreachable, // it's not a pipe
         error.WouldBlock => unreachable, // not asking for non-blocking I/O
-        error.FileNotFound => try writeBuiltinFile(file, builtin_pkg),
+        error.FileNotFound => try writeBuiltinFile(file, builtin_mod),
         else => |e| return e,
     }
 @@ -3212,8 +3205,8 @@ pub fn populateBuiltinFile(mod: *Module) !void {
     file.status = .success_zir;
 }
-fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void {
+fn writeBuiltinFile(file: *File, builtin_mod: *Package.Module) !void {
-    var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{});
+    var af = try builtin_mod.root_src_directory.handle.atomicFile(builtin_mod.root_src_path, .{});
     defer af.deinit();
     try af.file.writeAll(file.source);
     try af.finish();
 @@ -3748,7 +3741,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
     // TODO: figure out how this works under incremental changes to builtin.zig!
     const builtin_type_target_index: InternPool.Index = blk: {
-        const std_mod = mod.main_pkg.table.get("std").?;
+        const std_mod = mod.main_mod.table.get("std").?;
         if (decl.getFileScope(mod).pkg != std_mod) break :blk .none;
         // We're in the std module.
         const std_file = (try mod.importPkg(std_mod)).file;
 @@ -4100,13 +4093,13 @@ pub fn importFile(
     import_string: []const u8,
 ) !ImportFileResult {
     if (std.mem.eql(u8, import_string, "std")) {
-        return mod.importPkg(mod.main_pkg.table.get("std").?);
+        return mod.importPkg(mod.main_mod.table.get("std").?);
     }
     if (std.mem.eql(u8, import_string, "builtin")) {
-        return mod.importPkg(mod.main_pkg.table.get("builtin").?);
+        return mod.importPkg(mod.main_mod.table.get("builtin").?);
     }
     if (std.mem.eql(u8, import_string, "root")) {
-        return mod.importPkg(mod.root_pkg);
+        return mod.importPkg(mod.root_mod);
     }
     if (cur_file.pkg.table.get(import_string)) |pkg| {
         return mod.importPkg(pkg);
 @@ -4462,14 +4455,14 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
                 // test decl with no name. Skip the part where we check against
                 // the test name filter.
                 if (!comp.bin_file.options.is_test) break :blk false;
-                if (decl_pkg != mod.main_pkg) break :blk false;
+                if (decl_pkg != mod.main_mod) break :blk false;
                 try mod.test_functions.put(gpa, new_decl_index, {});
                 break :blk true;
             },
             else => blk: {
                 if (!is_named_test) break :blk false;
                 if (!comp.bin_file.options.is_test) break :blk false;
-                if (decl_pkg != mod.main_pkg) break :blk false;
+                if (decl_pkg != mod.main_mod) break :blk false;
                 if (comp.test_filter) |test_filter| {
                     if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) {
                         break :blk false;
 @@ -5596,8 +5589,8 @@ pub fn populateTestFunctions(
 ) !void {
     const gpa = mod.gpa;
     const ip = &mod.intern_pool;
-    const builtin_pkg = mod.main_pkg.table.get("builtin").?;
+    const builtin_mod = mod.main_mod.table.get("builtin").?;
-    const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file;
+    const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file;
     const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?);
     const builtin_namespace = mod.namespacePtr(root_decl.src_namespace);
     const test_functions_str = try ip.getOrPutString(gpa, "test_functions");

src/Package.zig+69-233

...	@@ -1,251 +1,87 @@	...	@@ -1,251 +1,87 @@
1	const Package = @This();	1	pub const Module = @import("Package/Module.zig");
2		2	pub const Fetch = @import("Package/Fetch.zig");
3	const builtin = @import("builtin");
4	const std = @import("std");
5	const fs = std.fs;
6	const mem = std.mem;
7	const Allocator = mem.Allocator;
8	const ascii = std.ascii;
9	const assert = std.debug.assert;
10	const log = std.log.scoped(.package);
11	const main = @import("main.zig");
12	const ThreadPool = std.Thread.Pool;
13
14	const Compilation = @import("Compilation.zig");
15	const Module = @import("Module.zig");
16	const Cache = std.Build.Cache;
17	const build_options = @import("build_options");
18	const Fetch = @import("Package/Fetch.zig");
19
20	pub const build_zig_basename = "build.zig";	3	pub const build_zig_basename = "build.zig";
21	pub const Manifest = @import("Manifest.zig");	4	pub const Manifest = @import("Manifest.zig");
22	pub const Table = std.StringHashMapUnmanaged(*Package);
23
24	root_src_directory: Compilation.Directory,
25	/// Relative to `root_src_directory`. May contain path separators.
26	root_src_path: []const u8,
27	/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root'
28	/// are not specified in every dependency table, but instead only in the table of `main_pkg`.
29	/// `Module.importFile` is responsible for detecting these names and using the correct package.
30	table: Table = .{},
31	/// Whether to free `root_src_directory` on `destroy`.
32	root_src_directory_owned: bool = false,
33
34	/// Allocate a Package. No references to the slices passed are kept.
35	pub fn create(
36	gpa: Allocator,
37	/// Null indicates the current working directory
38	root_src_dir_path: ?[]const u8,
39	/// Relative to root_src_dir_path
40	root_src_path: []const u8,
41	) !*Package {
42	const ptr = try gpa.create(Package);
43	errdefer gpa.destroy(ptr);
44
45	const owned_dir_path = if (root_src_dir_path) \|p\| try gpa.dupe(u8, p) else null;
46	errdefer if (owned_dir_path) \|p\| gpa.free(p);
47
48	const owned_src_path = try gpa.dupe(u8, root_src_path);
49	errdefer gpa.free(owned_src_path);
50		5
51	ptr.* = .{	6	pub const Path = struct {
52	.root_src_directory = .{	7	root_dir: Cache.Directory,
53	.path = owned_dir_path,	8	/// The path, relative to the root dir, that this `Path` represents.
54	.handle = if (owned_dir_path) \|p\| try fs.cwd().openDir(p, .{}) else fs.cwd(),	9	/// Empty string means the root_dir is the path.
55	},	10	sub_path: []const u8 = "",
56	.root_src_path = owned_src_path,
57	.root_src_directory_owned = true,
58	};
59		11
60	return ptr;	12	pub fn cwd() Path {
61	}	13	return .{ .root_dir = Cache.Directory.cwd() };
62
63	pub fn createWithDir(
64	gpa: Allocator,
65	directory: Compilation.Directory,
66	/// Relative to `directory`. If null, means `directory` is the root src dir
67	/// and is owned externally.
68	root_src_dir_path: ?[]const u8,
69	/// Relative to root_src_dir_path
70	root_src_path: []const u8,
71	) !*Package {
72	const ptr = try gpa.create(Package);
73	errdefer gpa.destroy(ptr);
74
75	const owned_src_path = try gpa.dupe(u8, root_src_path);
76	errdefer gpa.free(owned_src_path);
77
78	if (root_src_dir_path) \|p\| {
79	const owned_dir_path = try directory.join(gpa, &[1][]const u8{p});
80	errdefer gpa.free(owned_dir_path);
81
82	ptr.* = .{
83	.root_src_directory = .{
84	.path = owned_dir_path,
85	.handle = try directory.handle.openDir(p, .{}),
86	},
87	.root_src_directory_owned = true,
88	.root_src_path = owned_src_path,
89	};
90	} else {
91	ptr.* = .{
92	.root_src_directory = directory,
93	.root_src_directory_owned = false,
94	.root_src_path = owned_src_path,
95	};
96	}	14	}
97	return ptr;
98	}
99		15
100	/// Free all memory associated with this package. It does not destroy any packages	16	pub fn join(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error!Path {
101	/// inside its table; the caller is responsible for calling destroy() on them.	17	const parts: []const []const u8 =
102	pub fn destroy(pkg: *Package, gpa: Allocator) void {	18	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
103	gpa.free(pkg.root_src_path);	19	return .{
104		20	.root_dir = p.root_dir,
105	if (pkg.root_src_directory_owned) {	21	.sub_path = try fs.path.join(allocator, parts),
106	// If root_src_directory.path is null then the handle is the cwd()	22	};
107	// which shouldn't be closed.
108	if (pkg.root_src_directory.path) \|p\| {
109	gpa.free(p);
110	pkg.root_src_directory.handle.close();
111	}
112	}	23	}
113		24
114	pkg.deinitTable(gpa);	25	pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
115	gpa.destroy(pkg);	26	const parts: []const []const u8 =
116	}	27	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
117		28	return p.root_dir.join(allocator, parts);
118	/// Only frees memory associated with the table.
119	pub fn deinitTable(pkg: *Package, gpa: Allocator) void {
120	pkg.table.deinit(gpa);
121	}
122
123	pub fn add(pkg: Package, gpa: Allocator, name: []const u8, package: Package) !void {
124	try pkg.table.ensureUnusedCapacity(gpa, 1);
125	const name_dupe = try gpa.dupe(u8, name);
126	pkg.table.putAssumeCapacityNoClobber(name_dupe, package);
127	}
128
129	/// Compute a readable name for the package. The returned name should be freed from gpa. This
130	/// function is very slow, as it traverses the whole package hierarchy to find a path to this
131	/// package. It should only be used for error output.
132	pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 {
133	// we'll do a breadth-first search from the root module to try and find a short name for this
134	// module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is
135	// just the first-found shortest path - a module may be children of arbitrarily many other
136	// modules. This path may vary between executions due to hashmap iteration order, but that
137	// doesn't matter too much.
138	var node_arena = std.heap.ArenaAllocator.init(gpa);
139	defer node_arena.deinit();
140	const Parented = struct {
141	parent: ?*const @This(),
142	mod: *const Package,
143	};
144	const Queue = std.DoublyLinkedList(Parented);
145	var to_check: Queue = .{};
146
147	{
148	const new = try node_arena.allocator().create(Queue.Node);
149	new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } };
150	to_check.prepend(new);
151	}	29	}
152		30
153	if (mod.main_pkg != mod.root_pkg) {	31	pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
154	const new = try node_arena.allocator().create(Queue.Node);	32	const parts: []const []const u8 =
155	// TODO: once #12201 is resolved, we may want a way of indicating a different name for this	33	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
156	new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } };	34	return p.root_dir.joinZ(allocator, parts);
157	to_check.prepend(new);
158	}	35	}
159		36
160	// set of modules we've already checked to prevent loops	37	pub fn openFile(
161	var checked = std.AutoHashMap(*const Package, void).init(gpa);	38	p: Path,
162	defer checked.deinit();	39	sub_path: []const u8,
163		40	flags: fs.File.OpenFlags,
164	const linked = while (to_check.pop()) \|node\| {	41	) fs.File.OpenError!fs.File {
165	const check = &node.data;	42	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
166		43	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
167	if (checked.contains(check.mod)) continue;	44	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
168	try checked.put(check.mod, {});	45	p.sub_path, sub_path,
169		46	}) catch return error.NameTooLong;
170	if (check.mod == target) break check;	47	};
171		48	return p.root_dir.handle.openFile(joined_path, flags);
172	var it = check.mod.table.iterator();
173	while (it.next()) \|kv\| {
174	var new = try node_arena.allocator().create(Queue.Node);
175	new.* = .{ .data = .{
176	.parent = check,
177	.mod = kv.value_ptr.*,
178	} };
179	to_check.prepend(new);
180	}
181	} else {
182	// this can happen for e.g. @cImport packages
183	return gpa.dupe(u8, "<unnamed>");
184	};
185
186	// we found a path to the module! unfortunately, we can only traverse up it, so we have to put
187	// all the names into a buffer so we can then print them in order.
188	var names = std.ArrayList([]const u8).init(gpa);
189	defer names.deinit();
190
191	var cur: *const Parented = linked;
192	while (cur.parent) \|parent\| : (cur = parent) {
193	// find cur's name in parent
194	var it = parent.mod.table.iterator();
195	const name = while (it.next()) \|kv\| {
196	if (kv.value_ptr.* == cur.mod) {
197	break kv.key_ptr.*;
198	}
199	} else unreachable;
200	try names.append(name);
201	}	49	}
202		50
203	// finally, print the names into a buffer!	51	pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
204	var buf = std.ArrayList(u8).init(gpa);	52	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
205	defer buf.deinit();	53	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
206	try buf.writer().writeAll("root");	54	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
207	var i: usize = names.items.len;	55	p.sub_path, sub_path,
208	while (i > 0) {	56	}) catch return error.NameTooLong;
209	i -= 1;	57	};
210	try buf.writer().print(".{s}", .{names.items[i]});	58	return p.root_dir.handle.makeOpenPath(joined_path, opts);
211	}	59	}
212		60
213	return buf.toOwnedSlice();	61	pub fn format(
214	}	62	self: Path,
215		63	comptime fmt_string: []const u8,
216	pub fn createFilePkg(	64	options: std.fmt.FormatOptions,
217	gpa: Allocator,	65	writer: anytype,
218	cache_directory: Compilation.Directory,	66	) !void {
219	basename: []const u8,	67	_ = options;
220	contents: []const u8,	68	if (fmt_string.len > 0)
221	) !*Package {	69	std.fmt.invalidFmtError(fmt_string, self);
222	const rand_int = std.crypto.random.int(u64);	70	if (self.root_dir.path) \|p\| {
223	const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int);	71	try writer.writeAll(p);
224	{	72	try writer.writeAll(fs.path.sep_str);
225	var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});	73	}
226	defer tmp_dir.close();	74	if (self.sub_path.len > 0) {
227	try tmp_dir.writeFile(basename, contents);	75	try writer.writeAll(self.sub_path);
		76	try writer.writeAll(fs.path.sep_str);
		77	}
228	}	78	}
229
230	var hh: Cache.HashHelper = .{};
231	hh.addBytes(build_options.version);
232	hh.addBytes(contents);
233	const hex_digest = hh.final();
234
235	const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest;
236	try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path);
237
238	return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
239	}
240
241	const hex_multihash_len = 2 * Manifest.multihash_len;
242	const MultiHashHexDigest = [hex_multihash_len]u8;
243
244	const DependencyModule = union(enum) {
245	zig_pkg: *Package,
246	non_zig_pkg: *Package,
247	};	79	};
248	/// This is to avoid creating multiple modules for the same build.zig file.	80
249	/// If the value is `null`, the package is a known dependency, but has not yet	81	const Package = @This();
250	/// been fetched.	82	const builtin = @import("builtin");
251	pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule);	83	const std = @import("std");
		84	const fs = std.fs;
		85	const Allocator = std.mem.Allocator;
		86	const assert = std.debug.assert;
		87	const Cache = std.Build.Cache;

src/Package/Fetch.zig+446-205

...	@@ -27,59 +27,84 @@	...	@@ -27,59 +27,84 @@
27	//! All of this must be done with only referring to the state inside this struct	27	//! All of this must be done with only referring to the state inside this struct
28	//! because this work will be done in a dedicated thread.	28	//! because this work will be done in a dedicated thread.
29		29
30	/// Try to avoid this as much as possible since arena will have less contention.
31	gpa: Allocator,
32	arena: std.heap.ArenaAllocator,	30	arena: std.heap.ArenaAllocator,
33	location: Location,	31	location: Location,
34	location_tok: std.zig.Ast.TokenIndex,	32	location_tok: std.zig.Ast.TokenIndex,
35	hash_tok: std.zig.Ast.TokenIndex,	33	hash_tok: std.zig.Ast.TokenIndex,
36	global_cache: Cache.Directory,	34	parent_package_root: Package.Path,
37	parent_package_root: Path,
38	parent_manifest_ast: ?*const std.zig.Ast,	35	parent_manifest_ast: ?*const std.zig.Ast,
39	prog_node: *std.Progress.Node,	36	prog_node: *std.Progress.Node,
40	http_client: *std.http.Client,
41	thread_pool: *ThreadPool,
42	job_queue: *JobQueue,	37	job_queue: *JobQueue,
43	wait_group: *WaitGroup,	38	/// If true, don't add an error for a missing hash. This flag is not passed
		39	/// down to recursive dependencies. It's intended to be used only be the CLI.
		40	omit_missing_hash_error: bool,
44		41
45	// Above this are fields provided as inputs to `run`.	42	// Above this are fields provided as inputs to `run`.
46	// Below this are fields populated by `run`.	43	// Below this are fields populated by `run`.
47		44
48	/// This will either be relative to `global_cache`, or to the build root of	45	/// This will either be relative to `global_cache`, or to the build root of
49	/// the root package.	46	/// the root package.
50	package_root: Path,	47	package_root: Package.Path,
51	error_bundle: std.zig.ErrorBundle.Wip,	48	error_bundle: std.zig.ErrorBundle.Wip,
52	manifest: ?Manifest,	49	manifest: ?Manifest,
53	manifest_ast: ?*std.zig.Ast,	50	manifest_ast: std.zig.Ast,
54	actual_hash: Digest,	51	actual_hash: Manifest.Digest,
55	/// Fetch logic notices whether a package has a build.zig file and sets this flag.	52	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
56	has_build_zig: bool,	53	has_build_zig: bool,
57	/// Indicates whether the task aborted due to an out-of-memory condition.	54	/// Indicates whether the task aborted due to an out-of-memory condition.
58	oom_flag: bool,	55	oom_flag: bool,
59		56
		57	/// Contains shared state among all `Fetch` tasks.
60	pub const JobQueue = struct {	58	pub const JobQueue = struct {
61	mutex: std.Thread.Mutex = .{},	59	mutex: std.Thread.Mutex = .{},
62	};	60	/// Protected by `mutex`.
63		61	table: Table = .{},
64	pub const Digest = [Manifest.Hash.digest_length]u8;	62	/// `table` may be missing some tasks such as ones that failed, so this
65	pub const MultiHashHexDigest = [hex_multihash_len]u8;	63	/// field contains references to all of them.
66		64	/// Protected by `mutex`.
67	pub const Path = struct {	65	all_fetches: std.ArrayListUnmanaged(*Fetch) = .{},
68	root_dir: Cache.Directory,	66
69	/// The path, relative to the root dir, that this `Path` represents.	67	http_client: *std.http.Client,
70	/// Empty string means the root_dir is the path.	68	thread_pool: *ThreadPool,
71	sub_path: []const u8 = "",	69	wait_group: WaitGroup = .{},
		70	global_cache: Cache.Directory,
		71	recursive: bool,
		72	work_around_btrfs_bug: bool,
		73
		74	pub const Table = std.AutoHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
		75
		76	pub fn deinit(jq: *JobQueue) void {
		77	if (jq.all_fetches.items.len == 0) return;
		78	const gpa = jq.all_fetches.items[0].arena.child_allocator;
		79	jq.table.deinit(gpa);
		80	// These must be deinitialized in reverse order because subsequent
		81	// `Fetch` instances are allocated in prior ones' arenas.
		82	// Sorry, I know it's a bit weird, but it slightly simplifies the
		83	// critical section.
		84	while (jq.all_fetches.popOrNull()) \|f\| f.deinit();
		85	jq.all_fetches.deinit(gpa);
		86	jq.* = undefined;
		87	}
72	};	88	};
73		89
74	pub const Location = union(enum) {	90	pub const Location = union(enum) {
75	remote: Remote,	91	remote: Remote,
		92	/// A directory found inside the parent package.
76	relative_path: []const u8,	93	relative_path: []const u8,
		94	/// Recursive Fetch tasks will never use this Location, but it may be
		95	/// passed in by the CLI. Indicates the file contents here should be copied
		96	/// into the global package cache. It may be a file relative to the cwd or
		97	/// absolute, in which case it should be treated exactly like a `file://`
		98	/// URL, or a directory, in which case it should be treated as an
		99	/// already-unpacked directory (but still needs to be copied into the
		100	/// global package cache and have inclusion rules applied).
		101	path_or_url: []const u8,
77		102
78	pub const Remote = struct {	103	pub const Remote = struct {
79	url: []const u8,	104	url: []const u8,
80	/// If this is null it means the user omitted the hash field from a dependency.	105	/// If this is null it means the user omitted the hash field from a dependency.
81	/// It will be an error but the logic should still fetch and print the discovered hash.	106	/// It will be an error but the logic should still fetch and print the discovered hash.
82	hash: ?[hex_multihash_len]u8,	107	hash: ?Manifest.MultiHashHexDigest,
83	};	108	};
84	};	109	};
85		110
...	@@ -92,7 +117,11 @@ pub const RunError = error{	...	@@ -92,7 +117,11 @@ pub const RunError = error{
92		117
93	pub fn run(f: *Fetch) RunError!void {	118	pub fn run(f: *Fetch) RunError!void {
94	const eb = &f.error_bundle;	119	const eb = &f.error_bundle;
95	const arena = f.arena_allocator.allocator();	120	const arena = f.arena.allocator();
		121	const gpa = f.arena.child_allocator;
		122	const cache_root = f.job_queue.global_cache;
		123
		124	try eb.init(gpa);
96		125
97	// Check the global zig package cache to see if the hash already exists. If	126	// Check the global zig package cache to see if the hash already exists. If
98	// so, load, parse, and validate the build.zig.zon file therein, and skip	127	// so, load, parse, and validate the build.zig.zon file therein, and skip
...	@@ -111,43 +140,66 @@ pub fn run(f: *Fetch) RunError!void {	...	@@ -111,43 +140,66 @@ pub fn run(f: *Fetch) RunError!void {
111	);	140	);
112	f.package_root = try f.parent_package_root.join(arena, sub_path);	141	f.package_root = try f.parent_package_root.join(arena, sub_path);
113	try loadManifest(f, f.package_root);	142	try loadManifest(f, f.package_root);
		143	if (!f.job_queue.recursive) return;
114	// Package hashes are used as unique identifiers for packages, so	144	// Package hashes are used as unique identifiers for packages, so
115	// we still need one for relative paths.	145	// we still need one for relative paths.
116	const hash = h: {	146	const digest = h: {
117	var hasher = Manifest.Hash.init(.{});	147	var hasher = Manifest.Hash.init(.{});
118	// This hash is a tuple of:	148	// This hash is a tuple of:
119	// * whether it relative to the global cache directory or to the root package	149	// * whether it relative to the global cache directory or to the root package
120	// * the relative file path from there to the build root of the package	150	// * the relative file path from there to the build root of the package
121	hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle)	151	hasher.update(if (f.package_root.root_dir.eql(cache_root))
122	&package_hash_prefix_cached	152	&package_hash_prefix_cached
123	else	153	else
124	&package_hash_prefix_project);	154	&package_hash_prefix_project);
125	hasher.update(f.package_root.sub_path);	155	hasher.update(f.package_root.sub_path);
126	break :h hasher.finalResult();	156	break :h hasher.finalResult();
127	};	157	};
128	return queueJobsForDeps(f, hash);	158	return queueJobsForDeps(f, Manifest.hexDigest(digest));
129	},	159	},
130	.remote => \|remote\| remote,	160	.remote => \|remote\| remote,
		161	.path_or_url => \|path_or_url\| {
		162	if (fs.cwd().openIterableDir(path_or_url, .{})) \|dir\| {
		163	var resource: Resource = .{ .dir = dir };
		164	return runResource(f, path_or_url, &resource, null);
		165	} else \|dir_err\| {
		166	const file_err = if (dir_err == error.NotDir) e: {
		167	if (fs.cwd().openFile(path_or_url, .{})) \|file\| {
		168	var resource: Resource = .{ .file = file };
		169	return runResource(f, path_or_url, &resource, null);
		170	} else \|err\| break :e err;
		171	} else dir_err;
		172
		173	const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
		174	return f.fail(0, try eb.printString(
		175	"'{s}' could not be recognized as a file path ({s}) or an URL ({s})",
		176	.{ path_or_url, @errorName(file_err), @errorName(uri_err) },
		177	));
		178	};
		179	var resource = try f.initResource(uri);
		180	return runResource(f, uri.path, &resource, null);
		181	}
		182	},
131	};	183	};
		184
132	const s = fs.path.sep_str;	185	const s = fs.path.sep_str;
133	if (remote.hash) \|expected_hash\| {	186	if (remote.hash) \|expected_hash\| {
134	const pkg_sub_path = "p" ++ s ++ expected_hash;	187	const pkg_sub_path = "p" ++ s ++ expected_hash;
135	if (f.global_cache.handle.access(pkg_sub_path, .{})) \|_\| {	188	if (cache_root.handle.access(pkg_sub_path, .{})) \|_\| {
136	f.package_root = .{	189	f.package_root = .{
137	.root_dir = f.global_cache,	190	.root_dir = cache_root,
138	.sub_path = pkg_sub_path,	191	.sub_path = pkg_sub_path,
139	};	192	};
140	try loadManifest(f, f.package_root);	193	try loadManifest(f, f.package_root);
		194	if (!f.job_queue.recursive) return;
141	return queueJobsForDeps(f, expected_hash);	195	return queueJobsForDeps(f, expected_hash);
142	} else \|err\| switch (err) {	196	} else \|err\| switch (err) {
143	error.FileNotFound => {},	197	error.FileNotFound => {},
144	else => \|e\| {	198	else => \|e\| {
145	try eb.addRootErrorMessage(.{	199	try eb.addRootErrorMessage(.{
146	.msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{	200	.msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{
147	try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e),	201	try cache_root.join(arena, &.{pkg_sub_path}), @errorName(e),
148	}),	202	}),
149	.src_loc = .none,
150	.notes_len = 0,
151	});	203	});
152	return error.FetchFailed;	204	return error.FetchFailed;
153	},	205	},
...	@@ -158,22 +210,50 @@ pub fn run(f: *Fetch) RunError!void {	...	@@ -158,22 +210,50 @@ pub fn run(f: *Fetch) RunError!void {
158		210
159	const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(	211	const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
160	f.location_tok,	212	f.location_tok,
161	"invalid URI: {s}",	213	try eb.printString("invalid URI: {s}", .{@errorName(err)}),
162	.{@errorName(err)},
163	);	214	);
		215	var resource = try f.initResource(uri);
		216	return runResource(f, uri.path, &resource, remote.hash);
		217	}
		218
		219	pub fn deinit(f: *Fetch) void {
		220	f.error_bundle.deinit();
		221	f.arena.deinit();
		222	}
		223
		224	/// Consumes `resource`, even if an error is returned.
		225	fn runResource(
		226	f: *Fetch,
		227	uri_path: []const u8,
		228	resource: *Resource,
		229	remote_hash: ?Manifest.MultiHashHexDigest,
		230	) RunError!void {
		231	defer resource.deinit();
		232	const arena = f.arena.allocator();
		233	const eb = &f.error_bundle;
		234	const s = fs.path.sep_str;
		235	const cache_root = f.job_queue.global_cache;
164	const rand_int = std.crypto.random.int(u64);	236	const rand_int = std.crypto.random.int(u64);
165	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);	237	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
166		238
		239	const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
167	var tmp_directory: Cache.Directory = .{	240	var tmp_directory: Cache.Directory = .{
168	.path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}),	241	.path = tmp_directory_path,
169	.handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir,	242	.handle = handle: {
		243	const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch \|err\| {
		244	try eb.addRootErrorMessage(.{
		245	.msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{
		246	tmp_directory_path, @errorName(err),
		247	}),
		248	});
		249	return error.FetchFailed;
		250	};
		251	break :handle dir.dir;
		252	},
170	};	253	};
171	defer tmp_directory.handle.close();	254	defer tmp_directory.handle.close();
172		255
173	var resource = try f.initResource(uri);	256	try unpackResource(f, resource, uri_path, tmp_directory);
174	defer resource.deinit(); // releases more than memory
175
176	try f.unpackResource(&resource, uri.path, tmp_directory);
177		257
178	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed	258	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
179	// for the file to be missing, in which case this fetched package is	259	// for the file to be missing, in which case this fetched package is
...	@@ -194,15 +274,15 @@ pub fn run(f: *Fetch) RunError!void {	...	@@ -194,15 +274,15 @@ pub fn run(f: *Fetch) RunError!void {
194	// Compute the package hash based on the remaining files in the temporary	274	// Compute the package hash based on the remaining files in the temporary
195	// directory.	275	// directory.
196		276
197	if (builtin.os.tag == .linux and f.work_around_btrfs_bug) {	277	if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) {
198	// https://github.com/ziglang/zig/issues/17095	278	// https://github.com/ziglang/zig/issues/17095
199	tmp_directory.handle.close();	279	tmp_directory.handle.close();
200	const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch	280	const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch
201	@panic("btrfs workaround failed");	281	@panic("btrfs workaround failed");
202	tmp_directory.handle = iterable_dir.dir;	282	tmp_directory.handle = iterable_dir.dir;
203	}	283	}
204		284
205	f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter);	285	f.actual_hash = try computeHash(f, tmp_directory, filter);
206		286
207	// Rename the temporary directory into the global zig package cache	287	// Rename the temporary directory into the global zig package cache
208	// directory. If the hash already exists, delete the temporary directory	288	// directory. If the hash already exists, delete the temporary directory
...	@@ -211,40 +291,54 @@ pub fn run(f: *Fetch) RunError!void {	...	@@ -211,40 +291,54 @@ pub fn run(f: *Fetch) RunError!void {
211	// package with the different hash is used in the future.	291	// package with the different hash is used in the future.
212		292
213	const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash);	293	const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash);
214	try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path);	294	renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, dest_pkg_sub_path) catch \|err\| {
		295	const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
		296	const dest = try cache_root.join(arena, &.{dest_pkg_sub_path});
		297	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
		298	"unable to rename temporary directory '{s}' into package cache directory '{s}': {s}",
		299	.{ src, dest, @errorName(err) },
		300	) });
		301	return error.FetchFailed;
		302	};
215		303
216	// Validate the computed hash against the expected hash. If invalid, this	304	// Validate the computed hash against the expected hash. If invalid, this
217	// job is done.	305	// job is done.
218		306
219	const actual_hex = Manifest.hexDigest(f.actual_hash);	307	const actual_hex = Manifest.hexDigest(f.actual_hash);
220	if (remote.hash) \|declared_hash\| {	308	if (remote_hash) \|declared_hash\| {
221	if (!std.mem.eql(u8, declared_hash, &actual_hex)) {	309	if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
222	return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{	310	return f.fail(f.hash_tok, try eb.printString(
223	declared_hash, actual_hex,	311	"hash mismatch: manifest declares {s} but the fetched package has {s}",
224	});	312	.{ declared_hash, actual_hex },
		313	));
225	}	314	}
226	} else {	315	} else if (!f.omit_missing_hash_error) {
227	const notes_len = 1;	316	const notes_len = 1;
228	try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field");	317	try eb.addRootErrorMessage(.{
		318	.msg = try eb.addString("dependency is missing hash field"),
		319	.src_loc = try f.srcLoc(f.location_tok),
		320	.notes_len = notes_len,
		321	});
229	const notes_start = try eb.reserveNotes(notes_len);	322	const notes_start = try eb.reserveNotes(notes_len);
230	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{	323	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
231	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),	324	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
232	}));	325	}));
233	return error.PackageFetchFailed;	326	return error.FetchFailed;
234	}	327	}
235		328
236	// Spawn a new fetch job for each dependency in the manifest file. Use	329	// Spawn a new fetch job for each dependency in the manifest file. Use
237	// a mutex and a hash map so that redundant jobs do not get queued up.	330	// a mutex and a hash map so that redundant jobs do not get queued up.
238	return queueJobsForDeps(f, .{ .hash = f.actual_hash });	331	if (!f.job_queue.recursive) return;
		332	return queueJobsForDeps(f, actual_hex);
239	}	333	}
240		334
241	/// This function populates `f.manifest` or leaves it `null`.	335	/// This function populates `f.manifest` or leaves it `null`.
242	fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {	336	fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void {
243	const eb = &f.error_bundle;	337	const eb = &f.error_bundle;
244	const arena = f.arena_allocator.allocator();	338	const arena = f.arena.allocator();
245	const manifest_bytes = pkg_root.readFileAllocOptions(	339	const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
246	arena,	340	arena,
247	Manifest.basename,	341	try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }),
248	Manifest.max_bytes,	342	Manifest.max_bytes,
249	null,	343	null,
250	1,	344	1,
...	@@ -252,39 +346,39 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {	...	@@ -252,39 +346,39 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
252	) catch \|err\| switch (err) {	346	) catch \|err\| switch (err) {
253	error.FileNotFound => return,	347	error.FileNotFound => return,
254	else => \|e\| {	348	else => \|e\| {
255	const file_path = try pkg_root.join(arena, .{Manifest.basename});	349	const file_path = try pkg_root.join(arena, Manifest.basename);
256	try eb.addRootErrorMessage(.{	350	try eb.addRootErrorMessage(.{
257	.msg = try eb.printString("unable to load package manifest '{s}': {s}", .{	351	.msg = try eb.printString("unable to load package manifest '{}': {s}", .{
258	file_path, @errorName(e),	352	file_path, @errorName(e),
259	}),	353	}),
260	.src_loc = .none,
261	.notes_len = 0,
262	});	354	});
		355	return error.FetchFailed;
263	},	356	},
264	};	357	};
265		358
266	var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon);	359	const ast = &f.manifest_ast;
267	f.manifest_ast = ast;	360	ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
268		361
269	if (ast.errors.len > 0) {	362	if (ast.errors.len > 0) {
270	const file_path = try pkg_root.join(arena, .{Manifest.basename});	363	const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root});
271	try main.putAstErrorsIntoBundle(arena, ast, file_path, eb);	364	try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb);
272	return error.PackageFetchFailed;	365	return error.FetchFailed;
273	}	366	}
274		367
275	f.manifest = try Manifest.parse(arena, ast);	368	f.manifest = try Manifest.parse(arena, ast.*);
		369	const manifest = &f.manifest.?;
276		370
277	if (f.manifest.errors.len > 0) {	371	if (manifest.errors.len > 0) {
278	const file_path = try pkg_root.join(arena, .{Manifest.basename});	372	const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });
279	const token_starts = ast.tokens.items(.start);	373	const token_starts = ast.tokens.items(.start);
280		374
281	for (f.manifest.errors) \|msg\| {	375	for (manifest.errors) \|msg\| {
282	const start_loc = ast.tokenLocation(0, msg.tok);	376	const start_loc = ast.tokenLocation(0, msg.tok);
283		377
284	try eb.addRootErrorMessage(.{	378	try eb.addRootErrorMessage(.{
285	.msg = try eb.addString(msg.msg),	379	.msg = try eb.addString(msg.msg),
286	.src_loc = try eb.addSourceLocation(.{	380	.src_loc = try eb.addSourceLocation(.{
287	.src_path = try eb.addString(file_path),	381	.src_path = src_path,
288	.span_start = token_starts[msg.tok],	382	.span_start = token_starts[msg.tok],
289	.span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len),	383	.span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len),
290	.span_main = token_starts[msg.tok] + msg.off,	384	.span_main = token_starts[msg.tok] + msg.off,
...	@@ -292,71 +386,80 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {	...	@@ -292,71 +386,80 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
292	.column = @intCast(start_loc.column),	386	.column = @intCast(start_loc.column),
293	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),	387	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
294	}),	388	}),
295	.notes_len = 0,
296	});	389	});
297	}	390	}
298	return error.PackageFetchFailed;	391	return error.FetchFailed;
299	}	392	}
300	}	393	}
301		394
302	fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {	395	fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void {
		396	assert(f.job_queue.recursive);
		397
303	// If the package does not have a build.zig.zon file then there are no dependencies.	398	// If the package does not have a build.zig.zon file then there are no dependencies.
304	const manifest = f.manifest orelse return;	399	const manifest = f.manifest orelse return;
305		400
306	const new_fetches = nf: {	401	const new_fetches = nf: {
		402	const deps = manifest.dependencies.values();
		403	const gpa = f.arena.child_allocator;
307	// Grab the new tasks into a temporary buffer so we can unlock that mutex	404	// Grab the new tasks into a temporary buffer so we can unlock that mutex
308	// as fast as possible.	405	// as fast as possible.
309	// This overallocates any fetches that get skipped by the `continue` in the	406	// This overallocates any fetches that get skipped by the `continue` in the
310	// loop below.	407	// loop below.
311	const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count());	408	const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len);
312	var new_fetch_index: usize = 0;	409	var new_fetch_index: usize = 0;
313		410
314	f.job_queue.lock();	411	f.job_queue.mutex.lock();
315	defer f.job_queue.unlock();	412	defer f.job_queue.mutex.unlock();
		413
		414	try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
		415	try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1));
316		416
317	// It is impossible for there to be a collision here. Consider all three cases:	417	// It is impossible for there to be a collision here. Consider all three cases:
318	// * Correct hash is provided by manifest.	418	// * Correct hash is provided by manifest.
319	// - Redundant jobs are skipped in the loop below.	419	// - Redundant jobs are skipped in the loop below.
320	// * Incorrect has is provided by manifest.	420	// * Incorrect hash is provided by manifest.
321	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.	421	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
322	// * Hash is not provided by manifest.	422	// * Hash is not provided by manifest.
323	// - Hash missing error emitted; `queueJobsForDeps` is not called.	423	// - Hash missing error emitted; `queueJobsForDeps` is not called.
324	try f.job_queue.finish(hash, f, new_fetches.len);	424	f.job_queue.table.putAssumeCapacityNoClobber(hash, f);
325		425
326	for (manifest.dependencies.values()) \|dep\| {	426	for (deps) \|dep\| {
		427	const new_fetch = &new_fetches[new_fetch_index];
327	const location: Location = switch (dep.location) {	428	const location: Location = switch (dep.location) {
328	.url => \|url\| .{ .remote = .{	429	.url => \|url\| .{ .remote = .{
329	.url = url,	430	.url = url,
330	.hash = if (dep.hash) \|h\| h[0..hex_multihash_len].* else null,	431	.hash = h: {
		432	const h = dep.hash orelse break :h null;
		433	const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len;
		434	const multihash_digest = h[0..digest_len].*;
		435	const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
		436	if (gop.found_existing) continue;
		437	gop.value_ptr.* = new_fetch;
		438	break :h multihash_digest;
		439	},
331	} },	440	} },
332	.path => \|path\| .{ .relative_path = path },	441	.path => \|path\| .{ .relative_path = path },
333	};	442	};
334	const new_fetch = &new_fetches[new_fetch_index];
335	const already_done = f.job_queue.add(location, new_fetch);
336	if (already_done) continue;
337	new_fetch_index += 1;	443	new_fetch_index += 1;
338		444	f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
339	new_fetch.* = .{	445	new_fetch.* = .{
340	.gpa = f.gpa,	446	.arena = std.heap.ArenaAllocator.init(gpa),
341	.arena = std.heap.ArenaAllocator.init(f.gpa),
342	.location = location,	447	.location = location,
343	.location_tok = dep.location_tok,	448	.location_tok = dep.location_tok,
344	.hash_tok = dep.hash_tok,	449	.hash_tok = dep.hash_tok,
345	.global_cache = f.global_cache,
346	.parent_package_root = f.package_root,	450	.parent_package_root = f.package_root,
347	.parent_manifest_ast = f.manifest_ast.?,	451	.parent_manifest_ast = &f.manifest_ast,
348	.prog_node = f.prog_node,	452	.prog_node = f.prog_node,
349	.http_client = f.http_client,
350	.thread_pool = f.thread_pool,
351	.job_queue = f.job_queue,	453	.job_queue = f.job_queue,
352	.wait_group = f.wait_group,	454	.omit_missing_hash_error = false,
353		455
354	.package_root = undefined,	456	.package_root = undefined,
355	.error_bundle = .{},	457	.error_bundle = undefined,
356	.manifest = null,	458	.manifest = null,
357	.manifest_ast = null,	459	.manifest_ast = undefined,
358	.actual_hash = undefined,	460	.actual_hash = undefined,
359	.has_build_zig = false,	461	.has_build_zig = false,
		462	.oom_flag = false,
360	};	463	};
361	}	464	}
362		465
...	@@ -364,12 +467,14 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {	...	@@ -364,12 +467,14 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
364	};	467	};
365		468
366	// Now it's time to give tasks to the thread pool.	469	// Now it's time to give tasks to the thread pool.
367	for (new_fetches) \|new_fetch\| {	470	const thread_pool = f.job_queue.thread_pool;
368	f.wait_group.start();	471
369	f.thread_pool.spawn(workerRun, .{f}) catch \|err\| switch (err) {	472	for (new_fetches) \|*new_fetch\| {
		473	f.job_queue.wait_group.start();
		474	thread_pool.spawn(workerRun, .{new_fetch}) catch \|err\| switch (err) {
370	error.OutOfMemory => {	475	error.OutOfMemory => {
371	new_fetch.oom_flag = true;	476	new_fetch.oom_flag = true;
372	f.wait_group.finish();	477	f.job_queue.wait_group.finish();
373	continue;	478	continue;
374	},	479	},
375	};	480	};
...	@@ -377,43 +482,83 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {	...	@@ -377,43 +482,83 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
377	}	482	}
378		483
379	fn workerRun(f: *Fetch) void {	484	fn workerRun(f: *Fetch) void {
380	defer f.wait_group.finish();	485	defer f.job_queue.wait_group.finish();
381	run(f) catch \|err\| switch (err) {	486	run(f) catch \|err\| switch (err) {
382	error.OutOfMemory => f.oom_flag = true,	487	error.OutOfMemory => f.oom_flag = true,
383	error.FetchFailed => {}, // See `error_bundle`.	488	error.FetchFailed => {
		489	// Nothing to do because the errors are already reported in `error_bundle`,
		490	// and a reference is kept to the `Fetch` task inside `all_fetches`.
		491	},
384	};	492	};
385	}	493	}
386		494
387	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void {	495	fn srcLoc(
388	const ast = f.parent_manifest_ast;	496	f: *Fetch,
389	const token_starts = ast.tokens.items(.start);	497	tok: std.zig.Ast.TokenIndex,
390	const start_loc = ast.tokenLocation(0, msg_tok);	498	) Allocator.Error!std.zig.ErrorBundle.SourceLocationIndex {
		499	const ast = f.parent_manifest_ast orelse return .none;
391	const eb = &f.error_bundle;	500	const eb = &f.error_bundle;
392	const file_path = try f.parent_package_root.join(f.arena, Manifest.basename);	501	const token_starts = ast.tokens.items(.start);
		502	const start_loc = ast.tokenLocation(0, tok);
		503	const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root});
393	const msg_off = 0;	504	const msg_off = 0;
		505	return eb.addSourceLocation(.{
		506	.src_path = src_path,
		507	.span_start = token_starts[tok],
		508	.span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len),
		509	.span_main = token_starts[tok] + msg_off,
		510	.line = @intCast(start_loc.line),
		511	.column = @intCast(start_loc.column),
		512	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
		513	});
		514	}
394		515
		516	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
		517	const eb = &f.error_bundle;
395	try eb.addRootErrorMessage(.{	518	try eb.addRootErrorMessage(.{
396	.msg = msg_str,	519	.msg = msg_str,
397	.src_loc = try eb.addSourceLocation(.{	520	.src_loc = try f.srcLoc(msg_tok),
398	.src_path = try eb.addString(file_path),
399	.span_start = token_starts[msg_tok],
400	.span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len),
401	.span_main = token_starts[msg_tok] + msg_off,
402	.line = @intCast(start_loc.line),
403	.column = @intCast(start_loc.column),
404	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
405	}),
406	.notes_len = 0,
407	});	521	});
408
409	return error.FetchFailed;	522	return error.FetchFailed;
410	}	523	}
411		524
412	const Resource = union(enum) {	525	const Resource = union(enum) {
413	file: fs.File,	526	file: fs.File,
414	http_request: std.http.Client.Request,	527	http_request: std.http.Client.Request,
415	git_fetch_stream: git.Session.FetchStream,	528	git: Git,
416	dir: fs.IterableDir,	529	dir: fs.IterableDir,
		530
		531	const Git = struct {
		532	fetch_stream: git.Session.FetchStream,
		533	want_oid: [git.oid_length]u8,
		534	};
		535
		536	fn deinit(resource: *Resource) void {
		537	switch (resource.*) {
		538	.file => \|*file\| file.close(),
		539	.http_request => \|*req\| req.deinit(),
		540	.git => \|*git_resource\| git_resource.fetch_stream.deinit(),
		541	.dir => \|*dir\| dir.close(),
		542	}
		543	resource.* = undefined;
		544	}
		545
		546	fn reader(resource: *Resource) std.io.AnyReader {
		547	return .{
		548	.context = resource,
		549	.readFn = read,
		550	};
		551	}
		552
		553	fn read(context: *const anyopaque, buffer: []u8) anyerror!usize {
		554	const resource: *Resource = @constCast(@ptrCast(@alignCast(context)));
		555	switch (resource.*) {
		556	.file => \|*f\| return f.read(buffer),
		557	.http_request => \|*r\| return r.read(buffer),
		558	.git => \|*g\| return g.fetch_stream.read(buffer),
		559	.dir => unreachable,
		560	}
		561	}
417	};	562	};
418		563
419	const FileType = enum {	564	const FileType = enum {
...	@@ -468,30 +613,52 @@ const FileType = enum {	...	@@ -468,30 +613,52 @@ const FileType = enum {
468	};	613	};
469		614
470	fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {	615	fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
471	const gpa = f.gpa;	616	const gpa = f.arena.child_allocator;
472	const arena = f.arena_allocator.allocator();	617	const arena = f.arena.allocator();
473	const eb = &f.error_bundle;	618	const eb = &f.error_bundle;
474		619
475	if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{	620	if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{
476	.file = try f.parent_package_root.openFile(uri.path, .{}),	621	.file = f.parent_package_root.openFile(uri.path, .{}) catch \|err\| {
		622	return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{
		623	f.parent_package_root, uri.path, @errorName(err),
		624	}));
		625	},
477	};	626	};
478		627
		628	const http_client = f.job_queue.http_client;
		629
479	if (ascii.eqlIgnoreCase(uri.scheme, "http") or	630	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
480	ascii.eqlIgnoreCase(uri.scheme, "https"))	631	ascii.eqlIgnoreCase(uri.scheme, "https"))
481	{	632	{
482	var h = std.http.Headers{ .allocator = gpa };	633	var h = std.http.Headers{ .allocator = gpa };
483	defer h.deinit();	634	defer h.deinit();
484		635
485	var req = try f.http_client.request(.GET, uri, h, .{});	636	var req = http_client.request(.GET, uri, h, .{}) catch \|err\| {
		637	return f.fail(f.location_tok, try eb.printString(
		638	"unable to connect to server: {s}",
		639	.{@errorName(err)},
		640	));
		641	};
486	errdefer req.deinit(); // releases more than memory	642	errdefer req.deinit(); // releases more than memory
487		643
488	try req.start(.{});	644	req.start(.{}) catch \|err\| {
489	try req.wait();	645	return f.fail(f.location_tok, try eb.printString(
		646	"HTTP request failed: {s}",
		647	.{@errorName(err)},
		648	));
		649	};
		650	req.wait() catch \|err\| {
		651	return f.fail(f.location_tok, try eb.printString(
		652	"invalid HTTP response: {s}",
		653	.{@errorName(err)},
		654	));
		655	};
490		656
491	if (req.response.status != .ok) {	657	if (req.response.status != .ok) {
492	return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{	658	return f.fail(f.location_tok, try eb.printString(
493	@intFromEnum(req.response.status), req.response.status.phrase() orelse "",	659	"bad HTTP response code: '{d} {s}'",
494	});	660	.{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" },
		661	));
495	}	662	}
496		663
497	return .{ .http_request = req };	664	return .{ .http_request = req };
...	@@ -503,13 +670,21 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {	...	@@ -503,13 +670,21 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
503	var transport_uri = uri;	670	var transport_uri = uri;
504	transport_uri.scheme = uri.scheme["git+".len..];	671	transport_uri.scheme = uri.scheme["git+".len..];
505	var redirect_uri: []u8 = undefined;	672	var redirect_uri: []u8 = undefined;
506	var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri };	673	var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
507	session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {	674	session.discoverCapabilities(gpa, &redirect_uri) catch \|err\| switch (err) {
508	error.Redirected => {	675	error.Redirected => {
509	defer gpa.free(redirect_uri);	676	defer gpa.free(redirect_uri);
510	return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri});	677	return f.fail(f.location_tok, try eb.printString(
		678	"repository moved to {s}",
		679	.{redirect_uri},
		680	));
		681	},
		682	else => \|e\| {
		683	return f.fail(f.location_tok, try eb.printString(
		684	"unable to discover remote git server capabilities: {s}",
		685	.{@errorName(e)},
		686	));
511	},	687	},
512	else => \|other\| return other,
513	};	688	};
514		689
515	const want_oid = want_oid: {	690	const want_oid = want_oid: {
...	@@ -519,12 +694,22 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {	...	@@ -519,12 +694,22 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
519	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});	694	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
520	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});	695	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
521		696
522	var ref_iterator = try session.listRefs(gpa, .{	697	var ref_iterator = session.listRefs(gpa, .{
523	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },	698	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
524	.include_peeled = true,	699	.include_peeled = true,
525	});	700	}) catch \|err\| {
		701	return f.fail(f.location_tok, try eb.printString(
		702	"unable to list refs: {s}",
		703	.{@errorName(err)},
		704	));
		705	};
526	defer ref_iterator.deinit();	706	defer ref_iterator.deinit();
527	while (try ref_iterator.next()) \|ref\| {	707	while (ref_iterator.next() catch \|err\| {
		708	return f.fail(f.location_tok, try eb.printString(
		709	"unable to iterate refs: {s}",
		710	.{@errorName(err)},
		711	));
		712	}) \|ref\| {
528	if (std.mem.eql(u8, ref.name, want_ref) or	713	if (std.mem.eql(u8, ref.name, want_ref) or
529	std.mem.eql(u8, ref.name, want_ref_head) or	714	std.mem.eql(u8, ref.name, want_ref_head) or
530	std.mem.eql(u8, ref.name, want_ref_tag))	715	std.mem.eql(u8, ref.name, want_ref_tag))
...	@@ -532,31 +717,46 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {	...	@@ -532,31 +717,46 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
532	break :want_oid ref.peeled orelse ref.oid;	717	break :want_oid ref.peeled orelse ref.oid;
533	}	718	}
534	}	719	}
535	return f.fail(f.location_tok, "ref not found: {s}", .{want_ref});	720	return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
536	};	721	};
537	if (uri.fragment == null) {	722	if (uri.fragment == null) {
538	const notes_len = 1;	723	const notes_len = 1;
539	try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref");	724	try eb.addRootErrorMessage(.{
		725	.msg = try eb.addString("url field is missing an explicit ref"),
		726	.src_loc = try f.srcLoc(f.location_tok),
		727	.notes_len = notes_len,
		728	});
540	const notes_start = try eb.reserveNotes(notes_len);	729	const notes_start = try eb.reserveNotes(notes_len);
541	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{	730	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
542	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{	731	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{
543	uri, std.fmt.fmtSliceHexLower(&want_oid),	732	uri, std.fmt.fmtSliceHexLower(&want_oid),
544	}),	733	}),
545	}));	734	}));
546	return error.PackageFetchFailed;	735	return error.FetchFailed;
547	}	736	}
548		737
549	var want_oid_buf: [git.fmt_oid_length]u8 = undefined;	738	var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
550	_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{	739	_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{
551	std.fmt.fmtSliceHexLower(&want_oid),	740	std.fmt.fmtSliceHexLower(&want_oid),
552	}) catch unreachable;	741	}) catch unreachable;
553	var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf});	742	var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch \|err\| {
		743	return f.fail(f.location_tok, try eb.printString(
		744	"unable to create fetch stream: {s}",
		745	.{@errorName(err)},
		746	));
		747	};
554	errdefer fetch_stream.deinit();	748	errdefer fetch_stream.deinit();
555		749
556	return .{ .git_fetch_stream = fetch_stream };	750	return .{ .git = .{
		751	.fetch_stream = fetch_stream,
		752	.want_oid = want_oid,
		753	} };
557	}	754	}
558		755
559	return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme});	756	return f.fail(f.location_tok, try eb.printString(
		757	"unsupported URL scheme: {s}",
		758	.{uri.scheme},
		759	));
560	}	760	}
561		761
562	fn unpackResource(	762	fn unpackResource(
...	@@ -565,52 +765,62 @@ fn unpackResource(	...	@@ -565,52 +765,62 @@ fn unpackResource(
565	uri_path: []const u8,	765	uri_path: []const u8,
566	tmp_directory: Cache.Directory,	766	tmp_directory: Cache.Directory,
567	) RunError!void {	767	) RunError!void {
		768	const eb = &f.error_bundle;
568	const file_type = switch (resource.*) {	769	const file_type = switch (resource.*) {
569	.file => FileType.fromPath(uri_path) orelse	770	.file => FileType.fromPath(uri_path) orelse
570	return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}),	771	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
571		772
572	.http_request => \|req\| ft: {	773	.http_request => \|req\| ft: {
573	// Content-Type takes first precedence.	774	// Content-Type takes first precedence.
574	const content_type = req.response.headers.getFirstValue("Content-Type") orelse	775	const content_type = req.response.headers.getFirstValue("Content-Type") orelse
575	return f.fail(f.location_tok, "missing 'Content-Type' header", .{});	776	return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
576		777
577	if (ascii.eqlIgnoreCase(content_type, "application/x-tar"))	778	if (ascii.eqlIgnoreCase(content_type, "application/x-tar"))
578	return .tar;	779	break :ft .tar;
579		780
580	if (ascii.eqlIgnoreCase(content_type, "application/gzip") or	781	if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
581	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or	782	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
582	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))	783	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
583	{	784	{
584	return .@"tar.gz";	785	break :ft .@"tar.gz";
585	}	786	}
586		787
587	if (ascii.eqlIgnoreCase(content_type, "application/x-xz"))	788	if (ascii.eqlIgnoreCase(content_type, "application/x-xz"))
588	return .@"tar.xz";	789	break :ft .@"tar.xz";
589		790
590	if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {	791	if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
591	return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{	792	return f.fail(f.location_tok, try eb.printString(
592	content_type,	793	"unrecognized 'Content-Type' header: '{s}'",
593	});	794	.{content_type},
		795	));
594	}	796	}
595		797
596	// Next, the filename from 'content-disposition: attachment' takes precedence.	798	// Next, the filename from 'content-disposition: attachment' takes precedence.
597	if (req.response.headers.getFirstValue("Content-Disposition")) \|cd_header\| {	799	if (req.response.headers.getFirstValue("Content-Disposition")) \|cd_header\| {
598	break :ft FileType.fromContentDisposition(cd_header) orelse	800	break :ft FileType.fromContentDisposition(cd_header) orelse {
599	return f.fail(	801	return f.fail(f.location_tok, try eb.printString(
600	f.location_tok,	802	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
601	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",	803	.{cd_header},
602	.{cd_header},	804	));
603	);	805	};
604	}	806	}
605		807
606	// Finally, the path from the URI is used.	808	// Finally, the path from the URI is used.
607	break :ft FileType.fromPath(uri_path) orelse	809	break :ft FileType.fromPath(uri_path) orelse {
608	return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path});	810	return f.fail(f.location_tok, try eb.printString(
		811	"unknown file type: '{s}'",
		812	.{uri_path},
		813	));
		814	};
609	},	815	},
610	.git_fetch_stream => return .git_pack,	816
611	.dir => \|dir\| {	817	.git => .git_pack,
612	try f.recursiveDirectoryCopy(dir, tmp_directory.handle);	818
613	return;	819	.dir => \|dir\| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
		820	return f.fail(f.location_tok, try eb.printString(
		821	"unable to copy directory '{s}': {s}",
		822	.{ uri_path, @errorName(err) },
		823	));
614	},	824	},
615	};	825	};
616		826
...	@@ -618,7 +828,14 @@ fn unpackResource(	...	@@ -618,7 +828,14 @@ fn unpackResource(
618	.tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),	828	.tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
619	.@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),	829	.@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),
620	.@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),	830	.@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
621	.git_pack => try unpackGitPack(f, tmp_directory.handle, resource),	831	.git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch \|err\| switch (err) {
		832	error.FetchFailed => return error.FetchFailed,
		833	error.OutOfMemory => return error.OutOfMemory,
		834	else => \|e\| return f.fail(f.location_tok, try eb.printString(
		835	"unable to unpack git files: {s}",
		836	.{@errorName(e)},
		837	)),
		838	},
622	}	839	}
623	}	840	}
624		841
...	@@ -628,11 +845,17 @@ fn unpackTarballCompressed(	...	@@ -628,11 +845,17 @@ fn unpackTarballCompressed(
628	resource: *Resource,	845	resource: *Resource,
629	comptime Compression: type,	846	comptime Compression: type,
630	) RunError!void {	847	) RunError!void {
631	const gpa = f.gpa;	848	const gpa = f.arena.child_allocator;
		849	const eb = &f.error_bundle;
632	const reader = resource.reader();	850	const reader = resource.reader();
633	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);	851	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
634		852
635	var decompress = try Compression.decompress(gpa, br.reader());	853	var decompress = Compression.decompress(gpa, br.reader()) catch \|err\| {
		854	return f.fail(f.location_tok, try eb.printString(
		855	"unable to decompress tarball: {s}",
		856	.{@errorName(err)},
		857	));
		858	};
636	defer decompress.deinit();	859	defer decompress.deinit();
637		860
638	return unpackTarball(f, out_dir, decompress.reader());	861	return unpackTarball(f, out_dir, decompress.reader());
...	@@ -640,11 +863,12 @@ fn unpackTarballCompressed(	...	@@ -640,11 +863,12 @@ fn unpackTarballCompressed(
640		863
641	fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {	864	fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
642	const eb = &f.error_bundle;	865	const eb = &f.error_bundle;
		866	const gpa = f.arena.child_allocator;
643		867
644	var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa };	868	var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
645	defer diagnostics.deinit();	869	defer diagnostics.deinit();
646		870
647	try std.tar.pipeToFileSystem(out_dir, reader, .{	871	std.tar.pipeToFileSystem(out_dir, reader, .{
648	.diagnostics = &diagnostics,	872	.diagnostics = &diagnostics,
649	.strip_components = 1,	873	.strip_components = 1,
650	// TODO: we would like to set this to executable_bit_only, but two	874	// TODO: we would like to set this to executable_bit_only, but two
...	@@ -653,12 +877,19 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {	...	@@ -653,12 +877,19 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
653	// 2. the hashing algorithm here needs to support detecting the is_executable	877	// 2. the hashing algorithm here needs to support detecting the is_executable
654	// bit on Windows from the ACLs (see the isExecutable function).	878	// bit on Windows from the ACLs (see the isExecutable function).
655	.mode_mode = .ignore,	879	.mode_mode = .ignore,
656	.filter = .{ .exclude_empty_directories = true },	880	.exclude_empty_directories = true,
657	});	881	}) catch \|err\| return f.fail(f.location_tok, try eb.printString(
		882	"unable to unpack tarball to temporary directory: {s}",
		883	.{@errorName(err)},
		884	));
658		885
659	if (diagnostics.errors.items.len > 0) {	886	if (diagnostics.errors.items.len > 0) {
660	const notes_len: u32 = @intCast(diagnostics.errors.items.len);	887	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
661	try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball");	888	try eb.addRootErrorMessage(.{
		889	.msg = try eb.addString("unable to unpack tarball"),
		890	.src_loc = try f.srcLoc(f.location_tok),
		891	.notes_len = notes_len,
		892	});
662	const notes_start = try eb.reserveNotes(notes_len);	893	const notes_start = try eb.reserveNotes(notes_len);
663	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {	894	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
664	switch (item) {	895	switch (item) {
...	@@ -678,19 +909,15 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {	...	@@ -678,19 +909,15 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
678	},	909	},
679	}	910	}
680	}	911	}
681	return error.InvalidTarball;	912	return error.FetchFailed;
682	}	913	}
683	}	914	}
684		915
685	fn unpackGitPack(	916	fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource) anyerror!void {
686	f: *Fetch,
687	out_dir: fs.Dir,
688	resource: *Resource,
689	want_oid: git.Oid,
690	) !void {
691	const eb = &f.error_bundle;	917	const eb = &f.error_bundle;
692	const gpa = f.gpa;	918	const gpa = f.arena.child_allocator;
693	const reader = resource.reader();	919	const want_oid = resource.git.want_oid;
		920	const reader = resource.git.fetch_stream.reader();
694	// The .git directory is used to store the packfile and associated index, but	921	// The .git directory is used to store the packfile and associated index, but
695	// we do not attempt to replicate the exact structure of a real .git	922	// we do not attempt to replicate the exact structure of a real .git
696	// directory, since that isn't relevant for fetching a package.	923	// directory, since that isn't relevant for fetching a package.
...	@@ -700,13 +927,13 @@ fn unpackGitPack(	...	@@ -700,13 +927,13 @@ fn unpackGitPack(
700	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });	927	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
701	defer pack_file.close();	928	defer pack_file.close();
702	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();	929	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
703	try fifo.pump(reader.reader(), pack_file.writer());	930	try fifo.pump(reader, pack_file.writer());
704	try pack_file.sync();	931	try pack_file.sync();
705		932
706	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });	933	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
707	defer index_file.close();	934	defer index_file.close();
708	{	935	{
709	var index_prog_node = reader.prog_node.start("Index pack", 0);	936	var index_prog_node = f.prog_node.start("Index pack", 0);
710	defer index_prog_node.end();	937	defer index_prog_node.end();
711	index_prog_node.activate();	938	index_prog_node.activate();
712	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());	939	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
...	@@ -716,7 +943,7 @@ fn unpackGitPack(	...	@@ -716,7 +943,7 @@ fn unpackGitPack(
716	}	943	}
717		944
718	{	945	{
719	var checkout_prog_node = reader.prog_node.start("Checkout", 0);	946	var checkout_prog_node = f.prog_node.start("Checkout", 0);
720	defer checkout_prog_node.end();	947	defer checkout_prog_node.end();
721	checkout_prog_node.activate();	948	checkout_prog_node.activate();
722	var repository = try git.Repository.init(gpa, pack_file, index_file);	949	var repository = try git.Repository.init(gpa, pack_file, index_file);
...	@@ -727,7 +954,11 @@ fn unpackGitPack(	...	@@ -727,7 +954,11 @@ fn unpackGitPack(
727		954
728	if (diagnostics.errors.items.len > 0) {	955	if (diagnostics.errors.items.len > 0) {
729	const notes_len: u32 = @intCast(diagnostics.errors.items.len);	956	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
730	try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile");	957	try eb.addRootErrorMessage(.{
		958	.msg = try eb.addString("unable to unpack packfile"),
		959	.src_loc = try f.srcLoc(f.location_tok),
		960	.notes_len = notes_len,
		961	});
731	const notes_start = try eb.reserveNotes(notes_len);	962	const notes_start = try eb.reserveNotes(notes_len);
732	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {	963	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
733	switch (item) {	964	switch (item) {
...	@@ -748,9 +979,10 @@ fn unpackGitPack(	...	@@ -748,9 +979,10 @@ fn unpackGitPack(
748	try out_dir.deleteTree(".git");	979	try out_dir.deleteTree(".git");
749	}	980	}
750		981
751	fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void {	982	fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void {
		983	const gpa = f.arena.child_allocator;
752	// Recursive directory copy.	984	// Recursive directory copy.
753	var it = try dir.walk(f.gpa);	985	var it = try dir.walk(gpa);
754	defer it.deinit();	986	defer it.deinit();
755	while (try it.next()) \|entry\| {	987	while (try it.next()) \|entry\| {
756	switch (entry.kind) {	988	switch (entry.kind) {
...	@@ -816,16 +1048,22 @@ pub fn renameTmpIntoCache(	...	@@ -816,16 +1048,22 @@ pub fn renameTmpIntoCache(
816	/// the hash are not present on the file system. Empty directories are *not	1048	/// the hash are not present on the file system. Empty directories are *not
817	/// hashed* and must not be present on the file system when calling this	1049	/// hashed* and must not be present on the file system when calling this
818	/// function.	1050	/// function.
819	fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest {	1051	fn computeHash(
		1052	f: *Fetch,
		1053	tmp_directory: Cache.Directory,
		1054	filter: Filter,
		1055	) RunError!Manifest.Digest {
820	// All the path name strings need to be in memory for sorting.	1056	// All the path name strings need to be in memory for sorting.
821	const arena = f.arena_allocator.allocator();	1057	const arena = f.arena.allocator();
822	const gpa = f.gpa;	1058	const gpa = f.arena.child_allocator;
		1059	const eb = &f.error_bundle;
		1060	const thread_pool = f.job_queue.thread_pool;
823		1061
824	// Collect all files, recursively, then sort.	1062	// Collect all files, recursively, then sort.
825	var all_files = std.ArrayList(*HashedFile).init(gpa);	1063	var all_files = std.ArrayList(*HashedFile).init(gpa);
826	defer all_files.deinit();	1064	defer all_files.deinit();
827		1065
828	var walker = try pkg_dir.walk(gpa);	1066	var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa);
829	defer walker.deinit();	1067	defer walker.deinit();
830		1068
831	{	1069	{
...	@@ -834,19 +1072,28 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige	...	@@ -834,19 +1072,28 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
834	var wait_group: WaitGroup = .{};	1072	var wait_group: WaitGroup = .{};
835	// `computeHash` is called from a worker thread so there must not be	1073	// `computeHash` is called from a worker thread so there must not be
836	// any waiting without working or a deadlock could occur.	1074	// any waiting without working or a deadlock could occur.
837	defer wait_group.waitAndWork();	1075	defer thread_pool.waitAndWork(&wait_group);
838		1076
839	while (try walker.next()) \|entry\| {	1077	while (walker.next() catch \|err\| {
		1078	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
		1079	"unable to walk temporary directory '{}': {s}",
		1080	.{ tmp_directory, @errorName(err) },
		1081	) });
		1082	return error.FetchFailed;
		1083	}) \|entry\| {
840	_ = filter; // TODO: apply filter rules here	1084	_ = filter; // TODO: apply filter rules here
841		1085
842	const kind: HashedFile.Kind = switch (entry.kind) {	1086	const kind: HashedFile.Kind = switch (entry.kind) {
843	.directory => continue,	1087	.directory => continue,
844	.file => .file,	1088	.file => .file,
845	.sym_link => .sym_link,	1089	.sym_link => .sym_link,
846	else => return error.IllegalFileTypeInPackage,	1090	else => return f.fail(f.location_tok, try eb.printString(
		1091	"package contains '{s}' which has illegal file type '{s}'",
		1092	.{ entry.path, @tagName(entry.kind) },
		1093	)),
847	};	1094	};
848		1095
849	if (std.mem.eql(u8, entry.path, build_zig_basename))	1096	if (std.mem.eql(u8, entry.path, Package.build_zig_basename))
850	f.has_build_zig = true;	1097	f.has_build_zig = true;
851		1098
852	const hashed_file = try arena.create(HashedFile);	1099	const hashed_file = try arena.create(HashedFile);
...	@@ -859,7 +1106,9 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige	...	@@ -859,7 +1106,9 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
859	.failure = undefined, // to be populated by the worker	1106	.failure = undefined, // to be populated by the worker
860	};	1107	};
861	wait_group.start();	1108	wait_group.start();
862	try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });	1109	try thread_pool.spawn(workerHashFile, .{
		1110	tmp_directory.handle, hashed_file, &wait_group,
		1111	});
863		1112
864	try all_files.append(hashed_file);	1113	try all_files.append(hashed_file);
865	}	1114	}
...	@@ -869,19 +1118,13 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige	...	@@ -869,19 +1118,13 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
869		1118
870	var hasher = Manifest.Hash.init(.{});	1119	var hasher = Manifest.Hash.init(.{});
871	var any_failures = false;	1120	var any_failures = false;
872	const eb = &f.error_bundle;
873	for (all_files.items) \|hashed_file\| {	1121	for (all_files.items) \|hashed_file\| {
874	hashed_file.failure catch \|err\| {	1122	hashed_file.failure catch \|err\| {
875	any_failures = true;	1123	any_failures = true;
876	try eb.addRootErrorMessage(.{	1124	try eb.addRootErrorMessage(.{
877	.msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}),	1125	.msg = try eb.printString("unable to hash '{s}': {s}", .{
878	.src_loc = try eb.addSourceLocation(.{	1126	hashed_file.fs_path, @errorName(err),
879	.src_path = try eb.addString(hashed_file.fs_path),
880	.span_start = 0,
881	.span_end = 0,
882	.span_main = 0,
883	}),	1127	}),
884	.notes_len = 0,
885	});	1128	});
886	};	1129	};
887	hasher.update(&hashed_file.hash);	1130	hasher.update(&hashed_file.hash);
...	@@ -934,7 +1177,7 @@ fn isExecutable(file: fs.File) !bool {	...	@@ -934,7 +1177,7 @@ fn isExecutable(file: fs.File) !bool {
934	const HashedFile = struct {	1177	const HashedFile = struct {
935	fs_path: []const u8,	1178	fs_path: []const u8,
936	normalized_path: []const u8,	1179	normalized_path: []const u8,
937	hash: Digest,	1180	hash: Manifest.Digest,
938	failure: Error!void,	1181	failure: Error!void,
939	kind: Kind,	1182	kind: Kind,
940		1183
...	@@ -970,7 +1213,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {	...	@@ -970,7 +1213,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
970	return normalized;	1213	return normalized;
971	}	1214	}
972		1215
973	pub const Filter = struct {	1216	const Filter = struct {
974	include_paths: std.StringArrayHashMapUnmanaged(void) = .{},	1217	include_paths: std.StringArrayHashMapUnmanaged(void) = .{},
975		1218
976	/// sub_path is relative to the tarball root.	1219	/// sub_path is relative to the tarball root.
...	@@ -990,12 +1233,9 @@ pub const Filter = struct {	...	@@ -990,12 +1233,9 @@ pub const Filter = struct {
990	}	1233	}
991	};	1234	};
992		1235
993	const build_zig_basename = @import("../Package.zig").build_zig_basename;
994	const hex_multihash_len = 2 * Manifest.multihash_len;
995
996	// These are random bytes.	1236	// These are random bytes.
997	const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };	1237	const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
998	const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };	1238	const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
999		1239
1000	const builtin = @import("builtin");	1240	const builtin = @import("builtin");
1001	const std = @import("std");	1241	const std = @import("std");
...	@@ -1010,3 +1250,4 @@ const Manifest = @import("../Manifest.zig");	...	@@ -1010,3 +1250,4 @@ const Manifest = @import("../Manifest.zig");
1010	const Fetch = @This();	1250	const Fetch = @This();
1011	const main = @import("../main.zig");	1251	const main = @import("../main.zig");
1012	const git = @import("../git.zig");	1252	const git = @import("../git.zig");
		1253	const Package = @import("../Package.zig");

src/Package/Module.zig created+32

...	@@ -0,0 +1,32 @@
1	//! Corresponds to something that Zig source code can `@import`.
2	//! Not to be confused with src/Module.zig which should be renamed
3	//! to something else. https://github.com/ziglang/zig/issues/14307
4
5	/// Only files inside this directory can be imported.
6	root: Package.Path,
7	/// Relative to `root`. May contain path separators.
8	root_src_path: []const u8,
9	/// The dependency table of this module. Shared dependencies such as 'std',
10	/// 'builtin', and 'root' are not specified in every dependency table, but
11	/// instead only in the table of `main_pkg`. `Module.importFile` is
12	/// responsible for detecting these names and using the correct package.
13	deps: Deps = .{},
14
15	pub const Deps = std.StringHashMapUnmanaged(*Module);
16
17	pub const Tree = struct {
18	/// Each `Package` exposes a `Module` with build.zig as its root source file.
19	build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module),
20	};
21
22	pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module {
23	const new = try allocator.create(Module);
24	new.* = m;
25	return new;
26	}
27
28	const Module = @This();
29	const Package = @import("../Package.zig");
30	const std = @import("std");
31	const Allocator = std.mem.Allocator;
32	const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest;

src/crash_report.zig+13-9

...	@@ -139,18 +139,22 @@ fn dumpStatusReport() !void {	...	@@ -139,18 +139,22 @@ fn dumpStatusReport() !void {
139		139
140	var crash_heap: [16 * 4096]u8 = undefined;	140	var crash_heap: [16 * 4096]u8 = undefined;
141		141
142	fn writeFilePath(file: *Module.File, stream: anytype) !void {	142	fn writeFilePath(file: *Module.File, writer: anytype) !void {
143	if (file.pkg.root_src_directory.path) \|path\| {	143	if (file.mod.root.root_dir.path) \|path\| {
144	try stream.writeAll(path);	144	try writer.writeAll(path);
145	try stream.writeAll(std.fs.path.sep_str);	145	try writer.writeAll(std.fs.path.sep_str);
146	}	146	}
147	try stream.writeAll(file.sub_file_path);	147	if (file.mod.root.sub_path.len > 0) {
		148	try writer.writeAll(file.mod.root.sub_path);
		149	try writer.writeAll(std.fs.path.sep_str);
		150	}
		151	try writer.writeAll(file.sub_file_path);
148	}	152	}
149		153
150	fn writeFullyQualifiedDeclWithFile(mod: Module, decl: Decl, stream: anytype) !void {	154	fn writeFullyQualifiedDeclWithFile(mod: Module, decl: Decl, writer: anytype) !void {
151	try writeFilePath(decl.getFileScope(mod), stream);	155	try writeFilePath(decl.getFileScope(mod), writer);
152	try stream.writeAll(": ");	156	try writer.writeAll(": ");
153	try decl.renderFullyQualifiedDebugName(mod, stream);	157	try decl.renderFullyQualifiedDebugName(mod, writer);
154	}	158	}
155		159
156	pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn {	160	pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn {

src/main.zig+153-155

...	@@ -416,7 +416,7 @@ const usage_build_generic =	...	@@ -416,7 +416,7 @@ const usage_build_generic =
416	\\ dep: [[import=]name]	416	\\ dep: [[import=]name]
417	\\ --deps [dep],[dep],... Set dependency names for the root package	417	\\ --deps [dep],[dep],... Set dependency names for the root package
418	\\ dep: [[import=]name]	418	\\ dep: [[import=]name]
419	\\ --main-pkg-path Set the directory of the root package	419	\\ --main-mod-path Set the directory of the root module
420	\\ -fPIC Force-enable Position Independent Code	420	\\ -fPIC Force-enable Position Independent Code
421	\\ -fno-PIC Force-disable Position Independent Code	421	\\ -fno-PIC Force-disable Position Independent Code
422	\\ -fPIE Force-enable Position Independent Executable	422	\\ -fPIE Force-enable Position Independent Executable
...	@@ -765,17 +765,11 @@ const Framework = struct {	...	@@ -765,17 +765,11 @@ const Framework = struct {
765	};	765	};
766		766
767	const CliModule = struct {	767	const CliModule = struct {
768	mod: *Package,	768	mod: *Package.Module,
769	/// still in CLI arg format	769	/// still in CLI arg format
770	deps_str: []const u8,	770	deps_str: []const u8,
771	};	771	};
772		772
773	fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void {
774	var it = modules.iterator();
775	while (it.next()) \|kv\| kv.value_ptr.mod.destroy(modules.allocator);
776	modules.deinit();
777	}
778
779	fn buildOutputType(	773	fn buildOutputType(
780	gpa: Allocator,	774	gpa: Allocator,
781	arena: Allocator,	775	arena: Allocator,
...	@@ -950,8 +944,7 @@ fn buildOutputType(	...	@@ -950,8 +944,7 @@ fn buildOutputType(
950	// Contains every module specified via --mod. The dependencies are added	944	// Contains every module specified via --mod. The dependencies are added
951	// after argument parsing is completed. We use a StringArrayHashMap to make	945	// after argument parsing is completed. We use a StringArrayHashMap to make
952	// error output consistent.	946	// error output consistent.
953	var modules = std.StringArrayHashMap(CliModule).init(gpa);	947	var modules = std.StringArrayHashMap(CliModule).init(arena);
954	defer cleanupModules(&modules);
955		948
956	// The dependency string for the root package	949	// The dependency string for the root package
957	var root_deps_str: ?[]const u8 = null;	950	var root_deps_str: ?[]const u8 = null;
...	@@ -1023,32 +1016,37 @@ fn buildOutputType(	...	@@ -1023,32 +1016,37 @@ fn buildOutputType(
1023		1016
1024	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {	1017	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {
1025	if (mem.eql(u8, mod_name, name)) {	1018	if (mem.eql(u8, mod_name, name)) {
1026	fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src });	1019	fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{
		1020	mod_name, root_src,
		1021	});
1027	}	1022	}
1028	}	1023	}
1029		1024
1030	var mod_it = modules.iterator();	1025	var mod_it = modules.iterator();
1031	while (mod_it.next()) \|kv\| {	1026	while (mod_it.next()) \|kv\| {
1032	if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) {	1027	if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) {
1033	fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path });	1028	fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{
		1029	mod_name, root_src, kv.value_ptr.mod.root_src_path,
		1030	});
1034	}	1031	}
1035	}	1032	}
1036		1033
1037	try modules.ensureUnusedCapacity(1);	1034	try modules.put(mod_name, .{
1038	modules.put(mod_name, .{	1035	.mod = try Package.Module.create(arena, .{
1039	.mod = try Package.create(	1036	.root = .{
1040	gpa,	1037	.root_dir = Cache.Directory.cwd(),
1041	fs.path.dirname(root_src),	1038	.sub_path = fs.path.dirname(root_src) orelse "",
1042	fs.path.basename(root_src),	1039	},
1043	),	1040	.root_src_path = fs.path.basename(root_src),
		1041	}),
1044	.deps_str = deps_str,	1042	.deps_str = deps_str,
1045	}) catch unreachable;	1043	});
1046	} else if (mem.eql(u8, arg, "--deps")) {	1044	} else if (mem.eql(u8, arg, "--deps")) {
1047	if (root_deps_str != null) {	1045	if (root_deps_str != null) {
1048	fatal("only one --deps argument is allowed", .{});	1046	fatal("only one --deps argument is allowed", .{});
1049	}	1047	}
1050	root_deps_str = args_iter.nextOrFatal();	1048	root_deps_str = args_iter.nextOrFatal();
1051	} else if (mem.eql(u8, arg, "--main-pkg-path")) {	1049	} else if (mem.eql(u8, arg, "--main-mod-path")) {
1052	main_pkg_path = args_iter.nextOrFatal();	1050	main_pkg_path = args_iter.nextOrFatal();
1053	} else if (mem.eql(u8, arg, "-cflags")) {	1051	} else if (mem.eql(u8, arg, "-cflags")) {
1054	extra_cflags.shrinkRetainingCapacity(0);	1052	extra_cflags.shrinkRetainingCapacity(0);
...	@@ -2461,19 +2459,26 @@ fn buildOutputType(	...	@@ -2461,19 +2459,26 @@ fn buildOutputType(
2461	var deps_it = ModuleDepIterator.init(deps_str);	2459	var deps_it = ModuleDepIterator.init(deps_str);
2462	while (deps_it.next()) \|dep\| {	2460	while (deps_it.next()) \|dep\| {
2463	if (dep.expose.len == 0) {	2461	if (dep.expose.len == 0) {
2464	fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name });	2462	fatal("module '{s}' depends on '{s}' with a blank name", .{
		2463	kv.key_ptr.*, dep.name,
		2464	});
2465	}	2465	}
2466		2466
2467	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {	2467	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {
2468	if (mem.eql(u8, dep.expose, name)) {	2468	if (mem.eql(u8, dep.expose, name)) {
2469	fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose });	2469	fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{
		2470	dep.name, dep.expose,
		2471	});
2470	}	2472	}
2471	}	2473	}
2472		2474
2473	const dep_mod = modules.get(dep.name) orelse	2475	const dep_mod = modules.get(dep.name) orelse {
2474	fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name });	2476	fatal("module '{s}' depends on module '{s}' which does not exist", .{
		2477	kv.key_ptr.*, dep.name,
		2478	});
		2479	};
2475		2480
2476	try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod);	2481	try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod);
2477	}	2482	}
2478	}	2483	}
2479	}	2484	}
...	@@ -3229,31 +3234,33 @@ fn buildOutputType(	...	@@ -3229,31 +3234,33 @@ fn buildOutputType(
3229	};	3234	};
3230	defer emit_implib_resolved.deinit();	3235	defer emit_implib_resolved.deinit();
3231		3236
3232	const main_pkg: ?*Package = if (root_src_file) \|unresolved_src_path\| blk: {	3237	const main_mod: ?*Package.Module = if (root_src_file) \|unresolved_src_path\| blk: {
3233	const src_path = try introspect.resolvePath(arena, unresolved_src_path);	3238	const src_path = try introspect.resolvePath(arena, unresolved_src_path);
3234	if (main_pkg_path) \|unresolved_main_pkg_path\| {	3239	if (main_pkg_path) \|unresolved_main_pkg_path\| {
3235	const p = try introspect.resolvePath(arena, unresolved_main_pkg_path);	3240	const p = try introspect.resolvePath(arena, unresolved_main_pkg_path);
3236	if (p.len == 0) {	3241	break :blk try Package.Module.create(arena, .{
3237	break :blk try Package.create(gpa, null, src_path);	3242	.root = .{
3238	} else {	3243	.root_dir = Cache.Directory.cwd(),
3239	const rel_src_path = try fs.path.relative(arena, p, src_path);	3244	.sub_path = p,
3240	break :blk try Package.create(gpa, p, rel_src_path);	3245	},
3241	}	3246	.root_src_path = if (p.len == 0)
		3247	src_path
		3248	else
		3249	try fs.path.relative(arena, p, src_path),
		3250	});
3242	} else {	3251	} else {
3243	const root_src_dir_path = fs.path.dirname(src_path);	3252	break :blk try Package.Module.create(arena, .{
3244	break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch \|err\| {	3253	.root = .{
3245	if (root_src_dir_path) \|p\| {	3254	.root_dir = Cache.Directory.cwd(),
3246	fatal("unable to open '{s}': {s}", .{ p, @errorName(err) });	3255	.sub_path = fs.path.dirname(src_path) orelse "",
3247	} else {	3256	},
3248	return err;	3257	.root_src_path = fs.path.basename(src_path),
3249	}	3258	});
3250	};
3251	}	3259	}
3252	} else null;	3260	} else null;
3253	defer if (main_pkg) \|p\| p.destroy(gpa);
3254		3261
3255	// Transfer packages added with --deps to the root package	3262	// Transfer packages added with --deps to the root package
3256	if (main_pkg) \|mod\| {	3263	if (main_mod) \|mod\| {
3257	var it = ModuleDepIterator.init(root_deps_str orelse "");	3264	var it = ModuleDepIterator.init(root_deps_str orelse "");
3258	while (it.next()) \|dep\| {	3265	while (it.next()) \|dep\| {
3259	if (dep.expose.len == 0) {	3266	if (dep.expose.len == 0) {
...	@@ -3269,7 +3276,7 @@ fn buildOutputType(	...	@@ -3269,7 +3276,7 @@ fn buildOutputType(
3269	const dep_mod = modules.get(dep.name) orelse	3276	const dep_mod = modules.get(dep.name) orelse
3270	fatal("root module depends on module '{s}' which does not exist", .{dep.name});	3277	fatal("root module depends on module '{s}' which does not exist", .{dep.name});
3271		3278
3272	try mod.add(gpa, dep.expose, dep_mod.mod);	3279	try mod.deps.put(arena, dep.expose, dep_mod.mod);
3273	}	3280	}
3274	}	3281	}
3275		3282
...	@@ -3310,17 +3317,18 @@ fn buildOutputType(	...	@@ -3310,17 +3317,18 @@ fn buildOutputType(
3310	if (arg_mode == .run) {	3317	if (arg_mode == .run) {
3311	break :l global_cache_directory;	3318	break :l global_cache_directory;
3312	}	3319	}
3313	if (main_pkg) \|pkg\| {	3320	if (main_mod != null) {
3314	// search upwards from cwd until we find directory with build.zig	3321	// search upwards from cwd until we find directory with build.zig
3315	const cwd_path = try process.getCwdAlloc(arena);	3322	const cwd_path = try process.getCwdAlloc(arena);
3316	const build_zig = "build.zig";
3317	const zig_cache = "zig-cache";	3323	const zig_cache = "zig-cache";
3318	var dirname: []const u8 = cwd_path;	3324	var dirname: []const u8 = cwd_path;
3319	while (true) {	3325	while (true) {
3320	const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig });	3326	const joined_path = try fs.path.join(arena, &.{
		3327	dirname, Package.build_zig_basename,
		3328	});
3321	if (fs.cwd().access(joined_path, .{})) \|_\| {	3329	if (fs.cwd().access(joined_path, .{})) \|_\| {
3322	const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache });	3330	const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache });
3323	const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{});	3331	const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{});
3324	cleanup_local_cache_dir = dir;	3332	cleanup_local_cache_dir = dir;
3325	break :l .{ .handle = dir, .path = cache_dir_path };	3333	break :l .{ .handle = dir, .path = cache_dir_path };
3326	} else \|err\| switch (err) {	3334	} else \|err\| switch (err) {
...	@@ -3378,6 +3386,8 @@ fn buildOutputType(	...	@@ -3378,6 +3386,8 @@ fn buildOutputType(
3378		3386
3379	gimmeMoreOfThoseSweetSweetFileDescriptors();	3387	gimmeMoreOfThoseSweetSweetFileDescriptors();
3380		3388
		3389	if (true) @panic("TODO restore Compilation logic");
		3390
3381	const comp = Compilation.create(gpa, .{	3391	const comp = Compilation.create(gpa, .{
3382	.zig_lib_directory = zig_lib_directory,	3392	.zig_lib_directory = zig_lib_directory,
3383	.local_cache_directory = local_cache_directory,	3393	.local_cache_directory = local_cache_directory,
...	@@ -3389,7 +3399,7 @@ fn buildOutputType(	...	@@ -3389,7 +3399,7 @@ fn buildOutputType(
3389	.dynamic_linker = target_info.dynamic_linker.get(),	3399	.dynamic_linker = target_info.dynamic_linker.get(),
3390	.sysroot = sysroot,	3400	.sysroot = sysroot,
3391	.output_mode = output_mode,	3401	.output_mode = output_mode,
3392	.main_pkg = main_pkg,	3402	.main_mod = main_mod,
3393	.emit_bin = emit_bin_loc,	3403	.emit_bin = emit_bin_loc,
3394	.emit_h = emit_h_resolved.data,	3404	.emit_h = emit_h_resolved.data,
3395	.emit_asm = emit_asm_resolved.data,	3405	.emit_asm = emit_asm_resolved.data,
...	@@ -4799,32 +4809,22 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -4799,32 +4809,22 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4799	try thread_pool.init(.{ .allocator = gpa });	4809	try thread_pool.init(.{ .allocator = gpa });
4800	defer thread_pool.deinit();	4810	defer thread_pool.deinit();
4801		4811
4802	var cleanup_build_runner_dir: ?fs.Dir = null;	4812	var main_mod: Package.Module = if (override_build_runner) \|build_runner_path\|
4803	defer if (cleanup_build_runner_dir) \|*dir\| dir.close();
4804
4805	var main_pkg: Package = if (override_build_runner) \|build_runner_path\|
4806	.{	4813	.{
4807	.root_src_directory = blk: {	4814	.root = .{
4808	if (std.fs.path.dirname(build_runner_path)) \|dirname\| {	4815	.root_dir = Cache.Directory.cwd(),
4809	const dir = fs.cwd().openDir(dirname, .{}) catch \|err\| {	4816	.sub_path = fs.path.dirname(build_runner_path) orelse "",
4810	fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) });
4811	};
4812	cleanup_build_runner_dir = dir;
4813	break :blk .{ .path = dirname, .handle = dir };
4814	}
4815
4816	break :blk .{ .path = null, .handle = fs.cwd() };
4817	},	4817	},
4818	.root_src_path = std.fs.path.basename(build_runner_path),	4818	.root_src_path = fs.path.basename(build_runner_path),
4819	}	4819	}
4820	else	4820	else
4821	.{	4821	.{
4822	.root_src_directory = zig_lib_directory,	4822	.root = .{ .root_dir = zig_lib_directory },
4823	.root_src_path = "build_runner.zig",	4823	.root_src_path = "build_runner.zig",
4824	};	4824	};
4825		4825
4826	var build_pkg: Package = .{	4826	var build_mod: Package.Module = .{
4827	.root_src_directory = build_directory,	4827	.root = .{ .root_dir = build_directory },
4828	.root_src_path = build_zig_basename,	4828	.root_src_path = build_zig_basename,
4829	};	4829	};
4830	if (build_options.only_core_functionality) {	4830	if (build_options.only_core_functionality) {
...	@@ -4833,11 +4833,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -4833,11 +4833,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4833	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};	4833	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
4834	\\	4834	\\
4835	);	4835	);
4836	try main_pkg.add(gpa, "@dependencies", deps_pkg);	4836	try main_mod.deps.put(arena, "@dependencies", deps_pkg);
4837	} else {	4837	} else {
4838	var http_client: std.http.Client = .{ .allocator = gpa };	4838	var http_client: std.http.Client = .{ .allocator = gpa };
4839	defer http_client.deinit();	4839	defer http_client.deinit();
4840		4840
		4841	if (true) @panic("TODO restore package fetching logic");
		4842
4841	// Here we provide an import to the build runner that allows using reflection to find	4843	// Here we provide an import to the build runner that allows using reflection to find
4842	// all of the dependencies. Without this, there would be no way to use `@import` to	4844	// all of the dependencies. Without this, there would be no way to use `@import` to
4843	// access dependencies by name, since `@import` requires string literals.	4845	// access dependencies by name, since `@import` requires string literals.
...	@@ -4857,8 +4859,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -4857,8 +4859,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4857		4859
4858	// Here we borrow main package's table and will replace it with a fresh	4860	// Here we borrow main package's table and will replace it with a fresh
4859	// one after this process completes.	4861	// one after this process completes.
4860	const fetch_result = build_pkg.fetchAndAddDependencies(	4862	const fetch_result = build_mod.fetchAndAddDependencies(
4861	&main_pkg,	4863	&main_mod,
4862	arena,	4864	arena,
4863	&thread_pool,	4865	&thread_pool,
4864	&http_client,	4866	&http_client,
...	@@ -4886,10 +4888,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -4886,10 +4888,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4886	dependencies_source.items,	4888	dependencies_source.items,
4887	);	4889	);
4888		4890
4889	mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table);	4891	mem.swap(Package.Table, &main_mod.table, &deps_pkg.table);
4890	try main_pkg.add(gpa, "@dependencies", deps_pkg);	4892	try main_mod.add(gpa, "@dependencies", deps_pkg);
4891	}	4893	}
4892	try main_pkg.add(gpa, "@build", &build_pkg);	4894	try main_mod.add(gpa, "@build", &build_mod);
4893		4895
4894	const comp = Compilation.create(gpa, .{	4896	const comp = Compilation.create(gpa, .{
4895	.zig_lib_directory = zig_lib_directory,	4897	.zig_lib_directory = zig_lib_directory,
...	@@ -4901,7 +4903,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi	...	@@ -4901,7 +4903,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4901	.is_native_abi = cross_target.isNativeAbi(),	4903	.is_native_abi = cross_target.isNativeAbi(),
4902	.dynamic_linker = target_info.dynamic_linker.get(),	4904	.dynamic_linker = target_info.dynamic_linker.get(),
4903	.output_mode = .Exe,	4905	.output_mode = .Exe,
4904	.main_pkg = &main_pkg,	4906	.main_mod = &main_mod,
4905	.emit_bin = emit_bin,	4907	.emit_bin = emit_bin,
4906	.emit_h = null,	4908	.emit_h = null,
4907	.optimize_mode = .Debug,	4909	.optimize_mode = .Debug,
...	@@ -5115,12 +5117,14 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void	...	@@ -5115,12 +5117,14 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
5115	.tree = tree,	5117	.tree = tree,
5116	.tree_loaded = true,	5118	.tree_loaded = true,
5117	.zir = undefined,	5119	.zir = undefined,
5118	.pkg = undefined,	5120	.mod = undefined,
5119	.root_decl = .none,	5121	.root_decl = .none,
5120	};	5122	};
5121		5123
5122	file.pkg = try Package.create(gpa, null, file.sub_file_path);	5124	file.mod = try Package.Module.create(arena, .{
5123	defer file.pkg.destroy(gpa);	5125	.root = Package.Path.cwd(),
		5126	.root_src_path = file.sub_file_path,
		5127	});
5124		5128
5125	file.zir = try AstGen.generate(gpa, file.tree);	5129	file.zir = try AstGen.generate(gpa, file.tree);
5126	file.zir_loaded = true;	5130	file.zir_loaded = true;
...	@@ -5321,12 +5325,14 @@ fn fmtPathFile(	...	@@ -5321,12 +5325,14 @@ fn fmtPathFile(
5321	.tree = tree,	5325	.tree = tree,
5322	.tree_loaded = true,	5326	.tree_loaded = true,
5323	.zir = undefined,	5327	.zir = undefined,
5324	.pkg = undefined,	5328	.mod = undefined,
5325	.root_decl = .none,	5329	.root_decl = .none,
5326	};	5330	};
5327		5331
5328	file.pkg = try Package.create(gpa, null, file.sub_file_path);	5332	file.mod = try Package.Module.create(fmt.arena, .{
5329	defer file.pkg.destroy(gpa);	5333	.root = Package.Path.cwd(),
		5334	.root_src_path = file.sub_file_path,
		5335	});
5330		5336
5331	if (stat.size > max_src_size)	5337	if (stat.size > max_src_size)
5332	return error.FileTooBig;	5338	return error.FileTooBig;
...	@@ -5387,7 +5393,7 @@ pub fn putAstErrorsIntoBundle(	...	@@ -5387,7 +5393,7 @@ pub fn putAstErrorsIntoBundle(
5387	tree: Ast,	5393	tree: Ast,
5388	path: []const u8,	5394	path: []const u8,
5389	wip_errors: *std.zig.ErrorBundle.Wip,	5395	wip_errors: *std.zig.ErrorBundle.Wip,
5390	) !void {	5396	) Allocator.Error!void {
5391	var file: Module.File = .{	5397	var file: Module.File = .{
5392	.status = .never_loaded,	5398	.status = .never_loaded,
5393	.source_loaded = true,	5399	.source_loaded = true,
...	@@ -5402,12 +5408,15 @@ pub fn putAstErrorsIntoBundle(	...	@@ -5402,12 +5408,15 @@ pub fn putAstErrorsIntoBundle(
5402	.tree = tree,	5408	.tree = tree,
5403	.tree_loaded = true,	5409	.tree_loaded = true,
5404	.zir = undefined,	5410	.zir = undefined,
5405	.pkg = undefined,	5411	.mod = undefined,
5406	.root_decl = .none,	5412	.root_decl = .none,
5407	};	5413	};
5408		5414
5409	file.pkg = try Package.create(gpa, null, path);	5415	file.mod = try Package.Module.create(gpa, .{
5410	defer file.pkg.destroy(gpa);	5416	.root = Package.Path.cwd(),
		5417	.root_src_path = file.sub_file_path,
		5418	});
		5419	defer gpa.destroy(file.mod);
5411		5420
5412	file.zir = try AstGen.generate(gpa, file.tree);	5421	file.zir = try AstGen.generate(gpa, file.tree);
5413	file.zir_loaded = true;	5422	file.zir_loaded = true;
...	@@ -5933,7 +5942,7 @@ pub fn cmdAstCheck(	...	@@ -5933,7 +5942,7 @@ pub fn cmdAstCheck(
5933	.stat = undefined,	5942	.stat = undefined,
5934	.tree = undefined,	5943	.tree = undefined,
5935	.zir = undefined,	5944	.zir = undefined,
5936	.pkg = undefined,	5945	.mod = undefined,
5937	.root_decl = .none,	5946	.root_decl = .none,
5938	};	5947	};
5939	if (zig_source_file) \|file_name\| {	5948	if (zig_source_file) \|file_name\| {
...	@@ -5971,8 +5980,10 @@ pub fn cmdAstCheck(	...	@@ -5971,8 +5980,10 @@ pub fn cmdAstCheck(
5971	file.stat.size = source.len;	5980	file.stat.size = source.len;
5972	}	5981	}
5973		5982
5974	file.pkg = try Package.create(gpa, null, file.sub_file_path);	5983	file.mod = try Package.Module.create(arena, .{
5975	defer file.pkg.destroy(gpa);	5984	.root = Package.Path.cwd(),
		5985	.root_src_path = file.sub_file_path,
		5986	});
5976		5987
5977	file.tree = try Ast.parse(gpa, file.source, .zig);	5988	file.tree = try Ast.parse(gpa, file.source, .zig);
5978	file.tree_loaded = true;	5989	file.tree_loaded = true;
...	@@ -6067,7 +6078,7 @@ pub fn cmdDumpZir(	...	@@ -6067,7 +6078,7 @@ pub fn cmdDumpZir(
6067	.stat = undefined,	6078	.stat = undefined,
6068	.tree = undefined,	6079	.tree = undefined,
6069	.zir = try Module.loadZirCache(gpa, f),	6080	.zir = try Module.loadZirCache(gpa, f),
6070	.pkg = undefined,	6081	.mod = undefined,
6071	.root_decl = .none,	6082	.root_decl = .none,
6072	};	6083	};
6073		6084
...	@@ -6136,12 +6147,14 @@ pub fn cmdChangelist(	...	@@ -6136,12 +6147,14 @@ pub fn cmdChangelist(
6136	},	6147	},
6137	.tree = undefined,	6148	.tree = undefined,
6138	.zir = undefined,	6149	.zir = undefined,
6139	.pkg = undefined,	6150	.mod = undefined,
6140	.root_decl = .none,	6151	.root_decl = .none,
6141	};	6152	};
6142		6153
6143	file.pkg = try Package.create(gpa, null, file.sub_file_path);	6154	file.mod = try Package.Module.create(arena, .{
6144	defer file.pkg.destroy(gpa);	6155	.root = Package.Path.cwd(),
		6156	.root_src_path = file.sub_file_path,
		6157	});
6145		6158
6146	const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);	6159	const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
6147	const amt = try f.readAll(source);	6160	const amt = try f.readAll(source);
...	@@ -6623,8 +6636,11 @@ fn cmdFetch(	...	@@ -6623,8 +6636,11 @@ fn cmdFetch(
6623	args: []const []const u8,	6636	args: []const []const u8,
6624	) !void {	6637	) !void {
6625	const color: Color = .auto;	6638	const color: Color = .auto;
6626	var opt_url: ?[]const u8 = null;	6639	const work_around_btrfs_bug = builtin.os.tag == .linux and
		6640	std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
		6641	var opt_path_or_url: ?[]const u8 = null;
6627	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");	6642	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
		6643	var recursive = false;
6628		6644
6629	{	6645	{
6630	var i: usize = 0;	6646	var i: usize = 0;
...	@@ -6640,18 +6656,21 @@ fn cmdFetch(	...	@@ -6640,18 +6656,21 @@ fn cmdFetch(
6640	i += 1;	6656	i += 1;
6641	override_global_cache_dir = args[i];	6657	override_global_cache_dir = args[i];
6642	continue;	6658	continue;
		6659	} else if (mem.eql(u8, arg, "--recursive")) {
		6660	recursive = true;
		6661	continue;
6643	} else {	6662	} else {
6644	fatal("unrecognized parameter: '{s}'", .{arg});	6663	fatal("unrecognized parameter: '{s}'", .{arg});
6645	}	6664	}
6646	} else if (opt_url != null) {	6665	} else if (opt_path_or_url != null) {
6647	fatal("unexpected extra parameter: '{s}'", .{arg});	6666	fatal("unexpected extra parameter: '{s}'", .{arg});
6648	} else {	6667	} else {
6649	opt_url = arg;	6668	opt_path_or_url = arg;
6650	}	6669	}
6651	}	6670	}
6652	}	6671	}
6653		6672
6654	const url = opt_url orelse fatal("missing url or path parameter", .{});	6673	const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
6655		6674
6656	var thread_pool: ThreadPool = undefined;	6675	var thread_pool: ThreadPool = undefined;
6657	try thread_pool.init(.{ .allocator = gpa });	6676	try thread_pool.init(.{ .allocator = gpa });
...	@@ -6664,19 +6683,6 @@ fn cmdFetch(	...	@@ -6664,19 +6683,6 @@ fn cmdFetch(
6664	const root_prog_node = progress.start("Fetch", 0);	6683	const root_prog_node = progress.start("Fetch", 0);
6665	defer root_prog_node.end();	6684	defer root_prog_node.end();
6666		6685
6667	var wip_errors: std.zig.ErrorBundle.Wip = undefined;
6668	try wip_errors.init(gpa);
6669	defer wip_errors.deinit();
6670
6671	var report: Package.Report = .{
6672	.ast = null,
6673	.directory = .{
6674	.handle = fs.cwd(),
6675	.path = null,
6676	},
6677	.error_bundle = &wip_errors,
6678	};
6679
6680	var global_cache_directory: Compilation.Directory = l: {	6686	var global_cache_directory: Compilation.Directory = l: {
6681	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);	6687	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
6682	break :l .{	6688	break :l .{
...	@@ -6686,56 +6692,48 @@ fn cmdFetch(	...	@@ -6686,56 +6692,48 @@ fn cmdFetch(
6686	};	6692	};
6687	defer global_cache_directory.handle.close();	6693	defer global_cache_directory.handle.close();
6688		6694
6689	var readable_resource: Package.ReadableResource = rr: {	6695	var job_queue: Package.Fetch.JobQueue = .{
6690	if (fs.cwd().openIterableDir(url, .{})) \|dir\| {	6696	.http_client = &http_client,
6691	break :rr .{	6697	.thread_pool = &thread_pool,
6692	.path = try gpa.dupe(u8, url),	6698	.global_cache = global_cache_directory,
6693	.resource = .{ .dir = dir },	6699	.recursive = recursive,
6694	};	6700	.work_around_btrfs_bug = work_around_btrfs_bug,
6695	} else \|dir_err\| {	6701	};
6696	const file_err = if (dir_err == error.NotDir) e: {	6702	defer job_queue.deinit();
6697	if (fs.cwd().openFile(url, .{})) \|f\| {	6703
6698	break :rr .{	6704	var fetch: Package.Fetch = .{
6699	.path = try gpa.dupe(u8, url),	6705	.arena = std.heap.ArenaAllocator.init(gpa),
6700	.resource = .{ .file = f },	6706	.location = .{ .path_or_url = path_or_url },
6701	};	6707	.location_tok = 0,
6702	} else \|err\| break :e err;	6708	.hash_tok = 0,
6703	} else dir_err;	6709	.parent_package_root = undefined,
6704		6710	.parent_manifest_ast = null,
6705	const uri = std.Uri.parse(url) catch \|uri_err\| {	6711	.prog_node = root_prog_node,
6706	fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{	6712	.job_queue = &job_queue,
6707	url, @errorName(file_err), @errorName(uri_err),	6713	.omit_missing_hash_error = true,
6708	});	6714
6709	};	6715	.package_root = undefined,
6710	const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);	6716	.error_bundle = undefined,
6711	const cwd: Cache.Directory = .{	6717	.manifest = null,
6712	.handle = fs.cwd(),	6718	.manifest_ast = undefined,
6713	.path = null,	6719	.actual_hash = undefined,
6714	};	6720	.has_build_zig = false,
6715	break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);	6721	.oom_flag = false,
6716	}
6717	};	6722	};
6718	defer readable_resource.deinit(gpa);	6723	defer fetch.deinit();
6719		6724
6720	var package_location = readable_resource.unpack(	6725	fetch.run() catch \|err\| switch (err) {
6721	gpa,	6726	error.OutOfMemory => fatal("out of memory", .{}),
6722	&thread_pool,	6727	error.FetchFailed => {}, // error bundle checked below
6723	global_cache_directory,
6724	0,
6725	report,
6726	root_prog_node,
6727	) catch \|err\| {
6728	if (wip_errors.root_list.items.len > 0) {
6729	var errors = try wip_errors.toOwnedBundle("");
6730	defer errors.deinit(gpa);
6731	errors.renderToStdErr(renderOptions(color));
6732	process.exit(1);
6733	}
6734	fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) });
6735	};	6728	};
6736	defer package_location.deinit(gpa);
6737		6729
6738	const hex_digest = Package.Manifest.hexDigest(package_location.hash);	6730	if (fetch.error_bundle.root_list.items.len > 0) {
		6731	var errors = try fetch.error_bundle.toOwnedBundle("");
		6732	errors.renderToStdErr(renderOptions(color));
		6733	process.exit(1);
		6734	}
		6735
		6736	const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
6739		6737
6740	progress.done = true;	6738	progress.done = true;
6741	progress.refresh();	6739	progress.refresh();