Commit d0bcc390e8f61ada470b524e3fd203c1af521a99 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-10-06 17:05:58-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-10-08 16:54:31-07:00
log	d0bcc390e8f61ada470b524e3fd203c1af521a99
tree	14503dd0b9cb5b3666b6c397f3449c7931bbcb08
parent	88bbec8f9b2f8f023a0177c204f51b8ac0aee83a

get `zig fetch` working with the new system

* start renaming "package" to "module" (see #14307)
  - build system gains `main_mod_path` and `main_pkg_path` is still
    there but it is deprecated.
* eliminate the object-oriented memory management style of what was
  previously `*Package`. Now it is `*Package.Module` and all pointers
  point to externally managed memory.
* fixes to get the new Fetch.zig code working. The previous commit was
  work-in-progress. There are still two commented out code paths, the
  one that leads to `Compilation.create` and the one for `zig build`
  that fetches the entire dependency tree and creates the required
  modules for the build runner.

12 files changed, 883 insertions(+), 750 deletions(-)

build.zig
lib
- std
  - Build.zig
  - Build
    Cache.zig
    Step
    Compile.zig
src

build.zig+1-1

...	...	@@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void {
88	88	.name = "check-case",
89	89	.root_source_file = .{ .path = "test/src/Cases.zig" },
90	90	.optimize = optimize,
91		.main_pkg_path = .{ .path = "." },
	91	.main_mod_path = .{ .path = "." },
92	92	});
93	93	check_case_exe.stack_size = stack_size;
94	94	check_case_exe.single_threaded = single_threaded;

lib/std/Build.zig+20-5

...	...	@@ -634,6 +634,9 @@ pub const ExecutableOptions = struct {
634	634	use_llvm: ?bool = null,
635	635	use_lld: ?bool = null,
636	636	zig_lib_dir: ?LazyPath = null,
	637	main_mod_path: ?LazyPath = null,
	638
	639	/// Deprecated; use `main_mod_path`.
637	640	main_pkg_path: ?LazyPath = null,
638	641	};
639	642
...	...	@@ -652,7 +655,7 @@ pub fn addExecutable(b: Build, options: ExecutableOptions) Step.Compile {
652	655	.use_llvm = options.use_llvm,
653	656	.use_lld = options.use_lld,
654	657	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
655		.main_pkg_path = options.main_pkg_path,
	658	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
656	659	});
657	660	}
658	661
...	...	@@ -667,6 +670,9 @@ pub const ObjectOptions = struct {
667	670	use_llvm: ?bool = null,
668	671	use_lld: ?bool = null,
669	672	zig_lib_dir: ?LazyPath = null,
	673	main_mod_path: ?LazyPath = null,
	674
	675	/// Deprecated; use `main_mod_path`.
670	676	main_pkg_path: ?LazyPath = null,
671	677	};
672	678
...	...	@@ -683,7 +689,7 @@ pub fn addObject(b: Build, options: ObjectOptions) Step.Compile {
683	689	.use_llvm = options.use_llvm,
684	690	.use_lld = options.use_lld,
685	691	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
686		.main_pkg_path = options.main_pkg_path,
	692	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
687	693	});
688	694	}
689	695
...	...	@@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct {
699	705	use_llvm: ?bool = null,
700	706	use_lld: ?bool = null,
701	707	zig_lib_dir: ?LazyPath = null,
	708	main_mod_path: ?LazyPath = null,
	709
	710	/// Deprecated; use `main_mod_path`.
702	711	main_pkg_path: ?LazyPath = null,
703	712	};
704	713
...	...	@@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: Build, options: SharedLibraryOptions) Step.Compile
717	726	.use_llvm = options.use_llvm,
718	727	.use_lld = options.use_lld,
719	728	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
720		.main_pkg_path = options.main_pkg_path,
	729	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
721	730	});
722	731	}
723	732
...	...	@@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct {
733	742	use_llvm: ?bool = null,
734	743	use_lld: ?bool = null,
735	744	zig_lib_dir: ?LazyPath = null,
	745	main_mod_path: ?LazyPath = null,
	746
	747	/// Deprecated; use `main_mod_path`.
736	748	main_pkg_path: ?LazyPath = null,
737	749	};
738	750
...	...	@@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: Build, options: StaticLibraryOptions) Step.Compile
751	763	.use_llvm = options.use_llvm,
752	764	.use_lld = options.use_lld,
753	765	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
754		.main_pkg_path = options.main_pkg_path,
	766	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
755	767	});
756	768	}
757	769
...	...	@@ -769,6 +781,9 @@ pub const TestOptions = struct {
769	781	use_llvm: ?bool = null,
770	782	use_lld: ?bool = null,
771	783	zig_lib_dir: ?LazyPath = null,
	784	main_mod_path: ?LazyPath = null,
	785
	786	/// Deprecated; use `main_mod_path`.
772	787	main_pkg_path: ?LazyPath = null,
773	788	};
774	789
...	...	@@ -787,7 +802,7 @@ pub fn addTest(b: Build, options: TestOptions) Step.Compile {
787	802	.use_llvm = options.use_llvm,
788	803	.use_lld = options.use_lld,
789	804	.zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
790		.main_pkg_path = options.main_pkg_path,
	805	.main_mod_path = options.main_mod_path orelse options.main_pkg_path,
791	806	});
792	807	}
793	808

lib/std/Build/Cache.zig+11

...	...	@@ -9,6 +9,13 @@ pub const Directory = struct {
9	9	path: ?[]const u8,
10	10	handle: fs.Dir,
11	11
	12	pub fn cwd() Directory {
	13	return .{
	14	.path = null,
	15	.handle = fs.cwd(),
	16	};
	17	}
	18
12	19	pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
13	20	if (self.path) \|p\| {
14	21	// TODO clean way to do this with only 1 allocation
...	...	@@ -53,6 +60,10 @@ pub const Directory = struct {
53	60	try writer.writeAll(fs.path.sep_str);
54	61	}
55	62	}
	63
	64	pub fn eql(self: Directory, other: Directory) bool {
	65	return self.handle.fd == other.handle.fd;
	66	}
56	67	};
57	68
58	69	gpa: Allocator,

lib/std/Build/Step/Compile.zig+9-6

...	...	@@ -68,7 +68,7 @@ c_std: std.Build.CStd,
68	68	/// Set via options; intended to be read-only after that.
69	69	zig_lib_dir: ?LazyPath,
70	70	/// Set via options; intended to be read-only after that.
71		main_pkg_path: ?LazyPath,
	71	main_mod_path: ?LazyPath,
72	72	exec_cmd_args: ?[]const ?[]const u8,
73	73	filter: ?[]const u8,
74	74	test_evented_io: bool = false,
...	...	@@ -316,6 +316,9 @@ pub const Options = struct {
316	316	use_llvm: ?bool = null,
317	317	use_lld: ?bool = null,
318	318	zig_lib_dir: ?LazyPath = null,
	319	main_mod_path: ?LazyPath = null,
	320
	321	/// deprecated; use `main_mod_path`.
319	322	main_pkg_path: ?LazyPath = null,
320	323	};
321	324
...	...	@@ -480,7 +483,7 @@ pub fn create(owner: std.Build, options: Options) Compile {
480	483	.installed_headers = ArrayList(*Step).init(owner.allocator),
481	484	.c_std = std.Build.CStd.C99,
482	485	.zig_lib_dir = null,
483		.main_pkg_path = null,
	486	.main_mod_path = null,
484	487	.exec_cmd_args = null,
485	488	.filter = options.filter,
486	489	.test_runner = options.test_runner,
...	...	@@ -515,8 +518,8 @@ pub fn create(owner: std.Build, options: Options) Compile {
515	518	lp.addStepDependencies(&self.step);
516	519	}
517	520
518		if (options.main_pkg_path) \|lp\| {
519		self.main_pkg_path = lp.dupe(self.step.owner);
	521	if (options.main_mod_path orelse options.main_pkg_path) \|lp\| {
	522	self.main_mod_path = lp.dupe(self.step.owner);
520	523	lp.addStepDependencies(&self.step);
521	524	}
522	525
...	...	@@ -1998,8 +2001,8 @@ fn make(step: Step, prog_node: std.Progress.Node) !void {
1998	2001	try zig_args.append(dir.getPath(b));
1999	2002	}
2000	2003
2001		if (self.main_pkg_path) \|dir\| {
2002		try zig_args.append("--main-pkg-path");
	2004	if (self.main_mod_path) \|dir\| {
	2005	try zig_args.append("--main-mod-path");
2003	2006	try zig_args.append(dir.getPath(b));
2004	2007	}
2005	2008

src/Compilation.zig+84-86

...	...	@@ -273,8 +273,8 @@ const Job = union(enum) {
273	273	/// The source file containing the Decl has been updated, and so the
274	274	/// Decl may need its line number information updated in the debug info.
275	275	update_line_number: Module.Decl.Index,
276		/// The main source file for the package needs to be analyzed.
277		analyze_pkg: *Package,
	276	/// The main source file for the module needs to be analyzed.
	277	analyze_mod: *Package.Module,
278	278
279	279	/// one of the glibc static objects
280	280	glibc_crt_file: glibc.CRTFile,
...	...	@@ -414,7 +414,7 @@ pub const MiscTask = enum {
414	414	compiler_rt,
415	415	libssp,
416	416	zig_libc,
417		analyze_pkg,
	417	analyze_mod,
418	418
419	419	@"musl crti.o",
420	420	@"musl crtn.o",
...	...	@@ -544,7 +544,7 @@ pub const InitOptions = struct {
544	544	global_cache_directory: Directory,
545	545	target: Target,
546	546	root_name: []const u8,
547		main_pkg: ?*Package,
	547	main_mod: ?*Package.Module,
548	548	output_mode: std.builtin.OutputMode,
549	549	thread_pool: *ThreadPool,
550	550	dynamic_linker: ?[]const u8 = null,
...	...	@@ -736,53 +736,53 @@ pub const InitOptions = struct {
736	736	pdb_out_path: ?[]const u8 = null,
737	737	};
738	738
739		fn addPackageTableToCacheHash(
	739	fn addModuleTableToCacheHash(
740	740	hash: *Cache.HashHelper,
741	741	arena: *std.heap.ArenaAllocator,
742		pkg_table: Package.Table,
743		seen_table: std.AutoHashMap(Package, void),
	742	mod_table: Package.Module.Deps,
	743	seen_table: std.AutoHashMap(Package.Module, void),
744	744	hash_type: union(enum) { path_bytes, files: *Cache.Manifest },
745	745	) (error{OutOfMemory} \|\| std.os.GetCwdError)!void {
746	746	const allocator = arena.allocator();
747	747
748		const packages = try allocator.alloc(Package.Table.KV, pkg_table.count());
	748	const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count());
749	749	{
750	750	// Copy over the hashmap entries to our slice
751		var table_it = pkg_table.iterator();
	751	var table_it = mod_table.iterator();
752	752	var idx: usize = 0;
753	753	while (table_it.next()) \|entry\| : (idx += 1) {
754		packages[idx] = .{
	754	modules[idx] = .{
755	755	.key = entry.key_ptr.*,
756	756	.value = entry.value_ptr.*,
757	757	};
758	758	}
759	759	}
760	760	// Sort the slice by package name
761		mem.sort(Package.Table.KV, packages, {}, struct {
762		fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool {
	761	mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct {
	762	fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool {
763	763	return std.mem.lessThan(u8, lhs.key, rhs.key);
764	764	}
765	765	}.lessThan);
766	766
767		for (packages) \|pkg\| {
768		if ((try seen_table.getOrPut(pkg.value)).found_existing) continue;
	767	for (modules) \|mod\| {
	768	if ((try seen_table.getOrPut(mod.value)).found_existing) continue;
769	769
770	770	// Finally insert the package name and path to the cache hash.
771		hash.addBytes(pkg.key);
	771	hash.addBytes(mod.key);
772	772	switch (hash_type) {
773	773	.path_bytes => {
774		hash.addBytes(pkg.value.root_src_path);
775		hash.addOptionalBytes(pkg.value.root_src_directory.path);
	774	hash.addBytes(mod.value.root_src_path);
	775	hash.addOptionalBytes(mod.value.root_src_directory.path);
776	776	},
777	777	.files => \|man\| {
778		const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{
779		pkg.value.root_src_path,
	778	const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{
	779	mod.value.root_src_path,
780	780	});
781	781	_ = try man.addFile(pkg_zig_file, null);
782	782	},
783	783	}
784		// Recurse to handle the package's dependencies
785		try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type);
	784	// Recurse to handle the module's dependencies
	785	try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type);
786	786	}
787	787	}
788	788
...	...	@@ -839,7 +839,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
839	839	break :blk true;
840	840
841	841	// If we have no zig code to compile, no need for LLVM.
842		if (options.main_pkg == null)
	842	if (options.main_mod == null)
843	843	break :blk false;
844	844
845	845	// If LLVM does not support the target, then we can't use it.
...	...	@@ -869,7 +869,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
869	869	// compiler state, the second clause here can be removed so that incremental
870	870	// cache mode is used for LLVM backend too. We need some fuzz testing before
871	871	// that can be enabled.
872		const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching)
	872	const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching)
873	873	CacheMode.whole
874	874	else
875	875	options.cache_mode;
...	...	@@ -925,7 +925,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
925	925	if (use_llvm) {
926	926	// If stage1 generates an object file, self-hosted linker is not
927	927	// yet sophisticated enough to handle that.
928		break :blk options.main_pkg != null;
	928	break :blk options.main_mod != null;
929	929	}
930	930
931	931	break :blk false;
...	...	@@ -1210,7 +1210,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1210	1210	if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
1211	1211	// TODO audit this and make sure everything is in it
1212	1212
1213		const module: ?*Module = if (options.main_pkg) \|main_pkg\| blk: {
	1213	const module: ?*Module = if (options.main_mod) \|main_mod\| blk: {
1214	1214	// Options that are specific to zig source files, that cannot be
1215	1215	// modified between incremental updates.
1216	1216	var hash = cache.hash;
...	...	@@ -1223,11 +1223,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1223	1223	// do want to namespace different source file names because they are
1224	1224	// likely different compilations and therefore this would be likely to
1225	1225	// cause cache hits.
1226		hash.addBytes(main_pkg.root_src_path);
1227		hash.addOptionalBytes(main_pkg.root_src_directory.path);
	1226	hash.addBytes(main_mod.root_src_path);
	1227	hash.addOptionalBytes(main_mod.root.root_dir.path);
	1228	hash.addBytes(main_mod.root.sub_path);
1228	1229	{
1229		var seen_table = std.AutoHashMap(*Package, void).init(arena);
1230		try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes);
	1230	var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
	1231	try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes);
1231	1232	}
1232	1233	},
1233	1234	.whole => {
...	...	@@ -1283,34 +1284,31 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1283	1284	.path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}),
1284	1285	};
1285	1286
1286		const builtin_pkg = try Package.createWithDir(
1287		gpa,
1288		zig_cache_artifact_directory,
1289		null,
1290		"builtin.zig",
1291		);
1292		errdefer builtin_pkg.destroy(gpa);
	1287	const builtin_mod = try Package.Module.create(arena, .{
	1288	.root = .{ .root_dir = zig_cache_artifact_directory },
	1289	.root_src_path = "builtin.zig",
	1290	});
1293	1291
1294		// When you're testing std, the main module is std. In that case, we'll just set the std
1295		// module to the main one, since avoiding the errors caused by duplicating it is more
1296		// effort than it's worth.
1297		const main_pkg_is_std = m: {
	1292	// When you're testing std, the main module is std. In that case,
	1293	// we'll just set the std module to the main one, since avoiding
	1294	// the errors caused by duplicating it is more effort than it's
	1295	// worth.
	1296	const main_mod_is_std = m: {
1298	1297	const std_path = try std.fs.path.resolve(arena, &[_][]const u8{
1299	1298	options.zig_lib_directory.path orelse ".",
1300	1299	"std",
1301	1300	"std.zig",
1302	1301	});
1303		defer arena.free(std_path);
1304	1302	const main_path = try std.fs.path.resolve(arena, &[_][]const u8{
1305		main_pkg.root_src_directory.path orelse ".",
1306		main_pkg.root_src_path,
	1303	main_mod.root.root_dir.path orelse ".",
	1304	main_mod.root.sub_path,
	1305	main_mod.root_src_path,
1307	1306	});
1308		defer arena.free(main_path);
1309	1307	break :m mem.eql(u8, main_path, std_path);
1310	1308	};
1311	1309
1312		const std_pkg = if (main_pkg_is_std)
1313		main_pkg
	1310	const std_mod = if (main_mod_is_std)
	1311	main_mod
1314	1312	else
1315	1313	try Package.createWithDir(
1316	1314	gpa,
...	...	@@ -1319,16 +1317,16 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1319	1317	"std.zig",
1320	1318	);
1321	1319
1322		errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa);
	1320	errdefer if (!main_mod_is_std) std_mod.destroy(gpa);
1323	1321
1324		const root_pkg = if (options.is_test) root_pkg: {
	1322	const root_mod = if (options.is_test) root_mod: {
1325	1323	const test_pkg = if (options.test_runner_path) \|test_runner\| test_pkg: {
1326	1324	const test_dir = std.fs.path.dirname(test_runner);
1327	1325	const basename = std.fs.path.basename(test_runner);
1328	1326	const pkg = try Package.create(gpa, test_dir, basename);
1329	1327
1330		// copy package table from main_pkg to root_pkg
1331		pkg.table = try main_pkg.table.clone(gpa);
	1328	// copy module table from main_mod to root_mod
	1329	pkg.deps = try main_mod.deps.clone(gpa);
1332	1330	break :test_pkg pkg;
1333	1331	} else try Package.createWithDir(
1334	1332	gpa,
...	...	@@ -1338,26 +1336,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1338	1336	);
1339	1337	errdefer test_pkg.destroy(gpa);
1340	1338
1341		break :root_pkg test_pkg;
1342		} else main_pkg;
1343		errdefer if (options.is_test) root_pkg.destroy(gpa);
	1339	break :root_mod test_pkg;
	1340	} else main_mod;
	1341	errdefer if (options.is_test) root_mod.destroy(gpa);
1344	1342
1345		const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: {
1346		break :compiler_rt_pkg try Package.createWithDir(
	1343	const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: {
	1344	break :compiler_rt_mod try Package.createWithDir(
1347	1345	gpa,
1348	1346	options.zig_lib_directory,
1349	1347	null,
1350	1348	"compiler_rt.zig",
1351	1349	);
1352	1350	} else null;
1353		errdefer if (compiler_rt_pkg) \|p\| p.destroy(gpa);
	1351	errdefer if (compiler_rt_mod) \|p\| p.destroy(gpa);
1354	1352
1355		try main_pkg.add(gpa, "builtin", builtin_pkg);
1356		try main_pkg.add(gpa, "root", root_pkg);
1357		try main_pkg.add(gpa, "std", std_pkg);
	1353	try main_mod.add(gpa, "builtin", builtin_mod);
	1354	try main_mod.add(gpa, "root", root_mod);
	1355	try main_mod.add(gpa, "std", std_mod);
1358	1356
1359		if (compiler_rt_pkg) \|p\| {
1360		try main_pkg.add(gpa, "compiler_rt", p);
	1357	if (compiler_rt_mod) \|p\| {
	1358	try main_mod.add(gpa, "compiler_rt", p);
1361	1359	}
1362	1360
1363	1361	// Pre-open the directory handles for cached ZIR code so that it does not need
...	...	@@ -1395,8 +1393,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1395	1393	module.* = .{
1396	1394	.gpa = gpa,
1397	1395	.comp = comp,
1398		.main_pkg = main_pkg,
1399		.root_pkg = root_pkg,
	1396	.main_mod = main_mod,
	1397	.root_mod = root_mod,
1400	1398	.zig_cache_artifact_directory = zig_cache_artifact_directory,
1401	1399	.global_zir_cache = global_zir_cache,
1402	1400	.local_zir_cache = local_zir_cache,
...	...	@@ -2005,8 +2003,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: Compilation, directory: Director
2005	2003	// This is only for cleanup purposes; Module.deinit calls close
2006	2004	// on the handle of zig_cache_artifact_directory.
2007	2005	if (comp.bin_file.options.module) \|module\| {
2008		const builtin_pkg = module.main_pkg.table.get("builtin").?;
2009		module.zig_cache_artifact_directory = builtin_pkg.root_src_directory;
	2006	const builtin_mod = module.main_mod.deps.get("builtin").?;
	2007	module.zig_cache_artifact_directory = builtin_mod.root_src_directory;
2010	2008	}
2011	2009	}
2012	2010
...	...	@@ -2148,8 +2146,8 @@ pub fn update(comp: Compilation, main_progress_node: std.Progress.Node) !void
2148	2146
2149	2147	// Make sure std.zig is inside the import_table. We unconditionally need
2150	2148	// it for start.zig.
2151		const std_pkg = module.main_pkg.table.get("std").?;
2152		_ = try module.importPkg(std_pkg);
	2149	const std_mod = module.main_mod.deps.get("std").?;
	2150	_ = try module.importPkg(std_mod);
2153	2151
2154	2152	// Normally we rely on importing std to in turn import the root source file
2155	2153	// in the start code, but when using the stage1 backend that won't happen,
...	...	@@ -2158,11 +2156,11 @@ pub fn update(comp: Compilation, main_progress_node: std.Progress.Node) !void
2158	2156	// Likewise, in the case of `zig test`, the test runner is the root source file,
2159	2157	// and so there is nothing to import the main file.
2160	2158	if (comp.bin_file.options.is_test) {
2161		_ = try module.importPkg(module.main_pkg);
	2159	_ = try module.importPkg(module.main_mod);
2162	2160	}
2163	2161
2164		if (module.main_pkg.table.get("compiler_rt")) \|compiler_rt_pkg\| {
2165		_ = try module.importPkg(compiler_rt_pkg);
	2162	if (module.main_mod.deps.get("compiler_rt")) \|compiler_rt_mod\| {
	2163	_ = try module.importPkg(compiler_rt_mod);
2166	2164	}
2167	2165
2168	2166	// Put a work item in for every known source file to detect if
...	...	@@ -2185,13 +2183,13 @@ pub fn update(comp: Compilation, main_progress_node: std.Progress.Node) !void
2185	2183	}
2186	2184	}
2187	2185
2188		try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg });
	2186	try comp.work_queue.writeItem(.{ .analyze_mod = std_mod });
2189	2187	if (comp.bin_file.options.is_test) {
2190		try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg });
	2188	try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod });
2191	2189	}
2192	2190
2193		if (module.main_pkg.table.get("compiler_rt")) \|compiler_rt_pkg\| {
2194		try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg });
	2191	if (module.main_mod.deps.get("compiler_rt")) \|compiler_rt_mod\| {
	2192	try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod });
2195	2193	}
2196	2194	}
2197	2195
...	...	@@ -2420,19 +2418,19 @@ fn addNonIncrementalStuffToCacheManifest(comp: Compilation, man: Cache.Manifes
2420	2418	comptime assert(link_hash_implementation_version == 10);
2421	2419
2422	2420	if (comp.bin_file.options.module) \|mod\| {
2423		const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{
2424		mod.main_pkg.root_src_path,
	2421	const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{
	2422	mod.main_mod.root_src_path,
2425	2423	});
2426	2424	_ = try man.addFile(main_zig_file, null);
2427	2425	{
2428		var seen_table = std.AutoHashMap(*Package, void).init(arena);
	2426	var seen_table = std.AutoHashMap(*Package.Module, void).init(arena);
2429	2427
2430	2428	// Skip builtin.zig; it is useless as an input, and we don't want to have to
2431	2429	// write it before checking for a cache hit.
2432		const builtin_pkg = mod.main_pkg.table.get("builtin").?;
2433		try seen_table.put(builtin_pkg, {});
	2430	const builtin_mod = mod.main_mod.deps.get("builtin").?;
	2431	try seen_table.put(builtin_mod, {});
2434	2432
2435		try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man });
	2433	try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man });
2436	2434	}
2437	2435
2438	2436	// Synchronize with other matching comments: ZigOnlyHashStuff
...	...	@@ -3564,8 +3562,8 @@ fn processOneJob(comp: Compilation, job: Job, prog_node: std.Progress.Node) !v
3564	3562	decl.analysis = .codegen_failure_retryable;
3565	3563	};
3566	3564	},
3567		.analyze_pkg => \|pkg\| {
3568		const named_frame = tracy.namedFrame("analyze_pkg");
	3565	.analyze_mod => \|pkg\| {
	3566	const named_frame = tracy.namedFrame("analyze_mod");
3569	3567	defer named_frame.end();
3570	3568
3571	3569	const module = comp.bin_file.options.module.?;
...	...	@@ -6379,11 +6377,11 @@ fn buildOutputFromZig(
6379	6377
6380	6378	std.debug.assert(output_mode != .Exe);
6381	6379
6382		var main_pkg: Package = .{
	6380	var main_mod: Package = .{
6383	6381	.root_src_directory = comp.zig_lib_directory,
6384	6382	.root_src_path = src_basename,
6385	6383	};
6386		defer main_pkg.deinitTable(comp.gpa);
	6384	defer main_mod.deinitTable(comp.gpa);
6387	6385	const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len];
6388	6386	const target = comp.getTarget();
6389	6387	const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{
...	...	@@ -6404,7 +6402,7 @@ fn buildOutputFromZig(
6404	6402	.cache_mode = .whole,
6405	6403	.target = target,
6406	6404	.root_name = root_name,
6407		.main_pkg = &main_pkg,
	6405	.main_mod = &main_mod,
6408	6406	.output_mode = output_mode,
6409	6407	.thread_pool = comp.thread_pool,
6410	6408	.libc_installation = comp.bin_file.options.libc_installation,
...	...	@@ -6481,7 +6479,7 @@ pub fn build_crt_file(
6481	6479	.cache_mode = .whole,
6482	6480	.target = target,
6483	6481	.root_name = root_name,
6484		.main_pkg = null,
	6482	.main_mod = null,
6485	6483	.output_mode = output_mode,
6486	6484	.thread_pool = comp.thread_pool,
6487	6485	.libc_installation = comp.bin_file.options.libc_installation,

src/Manifest.zig+8-6

...	...	@@ -1,6 +1,10 @@
1	1	pub const max_bytes = 10 * 1024 * 1024;
2	2	pub const basename = "build.zig.zon";
3	3	pub const Hash = std.crypto.hash.sha2.Sha256;
	4	pub const Digest = [Hash.digest_length]u8;
	5	pub const multihash_len = 1 + 1 + Hash.digest_length;
	6	pub const multihash_hex_digest_len = 2 * multihash_len;
	7	pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
4	8
5	9	pub const Dependency = struct {
6	10	location: union(enum) {
...	...	@@ -46,7 +50,6 @@ comptime {
46	50	assert(@intFromEnum(multihash_function) < 127);
47	51	assert(Hash.digest_length < 127);
48	52	}
49		pub const multihash_len = 1 + 1 + Hash.digest_length;
50	53
51	54	name: []const u8,
52	55	version: std.SemanticVersion,
...	...	@@ -122,8 +125,8 @@ test hex64 {
122	125	try std.testing.expectEqualStrings("[00efcdab78563412]", s);
123	126	}
124	127
125		pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 {
126		var result: [multihash_len * 2]u8 = undefined;
	128	pub fn hexDigest(digest: Digest) MultiHashHexDigest {
	129	var result: MultiHashHexDigest = undefined;
127	130
128	131	result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
129	132	result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
...	...	@@ -339,10 +342,9 @@ const Parse = struct {
339	342	}
340	343	}
341	344
342		const hex_multihash_len = 2 * Manifest.multihash_len;
343		if (h.len != hex_multihash_len) {
	345	if (h.len != multihash_hex_digest_len) {
344	346	return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
345		hex_multihash_len, h.len,
	347	multihash_hex_digest_len, h.len,
346	348	});
347	349	}
348	350

src/Module.zig+37-44

...	...	@@ -55,10 +55,10 @@ comp: *Compilation,
55	55	/// Where build artifacts and incremental compilation metadata serialization go.
56	56	zig_cache_artifact_directory: Compilation.Directory,
57	57	/// Pointer to externally managed resource.
58		root_pkg: *Package,
59		/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which
60		/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests.
61		main_pkg: *Package,
	58	root_mod: *Package.Module,
	59	/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which
	60	/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests.
	61	main_mod: *Package.Module,
62	62	sema_prog_node: std.Progress.Node = undefined,
63	63
64	64	/// Used by AstGen worker to load and store ZIR cache.
...	...	@@ -973,8 +973,8 @@ pub const File = struct {
973	973	tree: Ast,
974	974	/// Whether this is populated or not depends on `zir_loaded`.
975	975	zir: Zir,
976		/// Package that this file is a part of, managed externally.
977		pkg: *Package,
	976	/// Module that this file is a part of, managed externally.
	977	mod: *Package.Module,
978	978	/// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen.
979	979	multi_pkg: bool = false,
980	980	/// List of references to this file, used for multi-package errors.
...	...	@@ -1058,14 +1058,9 @@ pub const File = struct {
1058	1058	.stat = file.stat,
1059	1059	};
1060	1060
1061		const root_dir_path = file.pkg.root_src_directory.path orelse ".";
1062		log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{
1063		root_dir_path, file.sub_file_path,
1064		});
1065
1066	1061	// Keep track of inode, file size, mtime, hash so we can detect which files
1067	1062	// have been modified when an incremental update is requested.
1068		var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{});
	1063	var f = try file.mod.root.openFile(file.sub_file_path, .{});
1069	1064	defer f.close();
1070	1065
1071	1066	const stat = try f.stat();
...	...	@@ -1134,14 +1129,12 @@ pub const File = struct {
1134	1129	return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start);
1135	1130	}
1136	1131
1137		/// Returns the full path to this file relative to its package.
1138	1132	pub fn fullPath(file: File, ally: Allocator) ![]u8 {
1139		return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path});
	1133	return file.mod.root.joinString(ally, file.sub_file_path);
1140	1134	}
1141	1135
1142		/// Returns the full path to this file relative to its package.
1143	1136	pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 {
1144		return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path});
	1137	return file.mod.root.joinStringZ(ally, file.sub_file_path);
1145	1138	}
1146	1139
1147	1140	pub fn dumpSrc(file: *File, src: LazySrcLoc) void {
...	...	@@ -2543,25 +2536,25 @@ pub fn deinit(mod: *Module) void {
2543	2536
2544	2537	mod.deletion_set.deinit(gpa);
2545	2538
2546		// The callsite of `Compilation.create` owns the `main_pkg`, however
	2539	// The callsite of `Compilation.create` owns the `main_mod`, however
2547	2540	// Module owns the builtin and std packages that it adds.
2548		if (mod.main_pkg.table.fetchRemove("builtin")) \|kv\| {
	2541	if (mod.main_mod.table.fetchRemove("builtin")) \|kv\| {
2549	2542	gpa.free(kv.key);
2550	2543	kv.value.destroy(gpa);
2551	2544	}
2552		if (mod.main_pkg.table.fetchRemove("std")) \|kv\| {
	2545	if (mod.main_mod.table.fetchRemove("std")) \|kv\| {
2553	2546	gpa.free(kv.key);
2554		// It's possible for main_pkg to be std when running 'zig test'! In this case, we must not
	2547	// It's possible for main_mod to be std when running 'zig test'! In this case, we must not
2555	2548	// destroy it, since it would lead to a double-free.
2556		if (kv.value != mod.main_pkg) {
	2549	if (kv.value != mod.main_mod) {
2557	2550	kv.value.destroy(gpa);
2558	2551	}
2559	2552	}
2560		if (mod.main_pkg.table.fetchRemove("root")) \|kv\| {
	2553	if (mod.main_mod.table.fetchRemove("root")) \|kv\| {
2561	2554	gpa.free(kv.key);
2562	2555	}
2563		if (mod.root_pkg != mod.main_pkg) {
2564		mod.root_pkg.destroy(gpa);
	2556	if (mod.root_mod != mod.main_mod) {
	2557	mod.root_mod.destroy(gpa);
2565	2558	}
2566	2559
2567	2560	mod.compile_log_text.deinit(gpa);
...	...	@@ -2715,7 +2708,7 @@ pub fn astGenFile(mod: Module, file: File) !void {
2715	2708
2716	2709	const stat = try source_file.stat();
2717	2710
2718		const want_local_cache = file.pkg == mod.main_pkg;
	2711	const want_local_cache = file.pkg == mod.main_mod;
2719	2712	const digest = hash: {
2720	2713	var path_hash: Cache.HashHelper = .{};
2721	2714	path_hash.addBytes(build_options.version);
...	...	@@ -3158,23 +3151,23 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3158	3151	comp.mutex.lock();
3159	3152	defer comp.mutex.unlock();
3160	3153
3161		const builtin_pkg = mod.main_pkg.table.get("builtin").?;
3162		const result = try mod.importPkg(builtin_pkg);
	3154	const builtin_mod = mod.main_mod.table.get("builtin").?;
	3155	const result = try mod.importPkg(builtin_mod);
3163	3156	break :blk .{
3164	3157	.file = result.file,
3165		.pkg = builtin_pkg,
	3158	.pkg = builtin_mod,
3166	3159	};
3167	3160	};
3168	3161	const file = pkg_and_file.file;
3169		const builtin_pkg = pkg_and_file.pkg;
	3162	const builtin_mod = pkg_and_file.pkg;
3170	3163	const gpa = mod.gpa;
3171	3164	file.source = try comp.generateBuiltinZigSource(gpa);
3172	3165	file.source_loaded = true;
3173	3166
3174		if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) \|stat\| {
	3167	if (builtin_mod.root_src_directory.handle.statFile(builtin_mod.root_src_path)) \|stat\| {
3175	3168	if (stat.size != file.source.len) {
3176		const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{
3177		builtin_pkg.root_src_path,
	3169	const full_path = try builtin_mod.root_src_directory.join(gpa, &.{
	3170	builtin_mod.root_src_path,
3178	3171	});
3179	3172	defer gpa.free(full_path);
3180	3173
...	...	@@ -3184,7 +3177,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3184	3177	.{ full_path, file.source.len, stat.size },
3185	3178	);
3186	3179
3187		try writeBuiltinFile(file, builtin_pkg);
	3180	try writeBuiltinFile(file, builtin_mod);
3188	3181	} else {
3189	3182	file.stat = .{
3190	3183	.size = stat.size,
...	...	@@ -3198,7 +3191,7 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3198	3191	error.PipeBusy => unreachable, // it's not a pipe
3199	3192	error.WouldBlock => unreachable, // not asking for non-blocking I/O
3200	3193
3201		error.FileNotFound => try writeBuiltinFile(file, builtin_pkg),
	3194	error.FileNotFound => try writeBuiltinFile(file, builtin_mod),
3202	3195
3203	3196	else => \|e\| return e,
3204	3197	}
...	...	@@ -3212,8 +3205,8 @@ pub fn populateBuiltinFile(mod: *Module) !void {
3212	3205	file.status = .success_zir;
3213	3206	}
3214	3207
3215		fn writeBuiltinFile(file: File, builtin_pkg: Package) !void {
3216		var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{});
	3208	fn writeBuiltinFile(file: File, builtin_mod: Package.Module) !void {
	3209	var af = try builtin_mod.root_src_directory.handle.atomicFile(builtin_mod.root_src_path, .{});
3217	3210	defer af.deinit();
3218	3211	try af.file.writeAll(file.source);
3219	3212	try af.finish();
...	...	@@ -3748,7 +3741,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool {
3748	3741
3749	3742	// TODO: figure out how this works under incremental changes to builtin.zig!
3750	3743	const builtin_type_target_index: InternPool.Index = blk: {
3751		const std_mod = mod.main_pkg.table.get("std").?;
	3744	const std_mod = mod.main_mod.table.get("std").?;
3752	3745	if (decl.getFileScope(mod).pkg != std_mod) break :blk .none;
3753	3746	// We're in the std module.
3754	3747	const std_file = (try mod.importPkg(std_mod)).file;
...	...	@@ -4100,13 +4093,13 @@ pub fn importFile(
4100	4093	import_string: []const u8,
4101	4094	) !ImportFileResult {
4102	4095	if (std.mem.eql(u8, import_string, "std")) {
4103		return mod.importPkg(mod.main_pkg.table.get("std").?);
	4096	return mod.importPkg(mod.main_mod.table.get("std").?);
4104	4097	}
4105	4098	if (std.mem.eql(u8, import_string, "builtin")) {
4106		return mod.importPkg(mod.main_pkg.table.get("builtin").?);
	4099	return mod.importPkg(mod.main_mod.table.get("builtin").?);
4107	4100	}
4108	4101	if (std.mem.eql(u8, import_string, "root")) {
4109		return mod.importPkg(mod.root_pkg);
	4102	return mod.importPkg(mod.root_mod);
4110	4103	}
4111	4104	if (cur_file.pkg.table.get(import_string)) \|pkg\| {
4112	4105	return mod.importPkg(pkg);
...	...	@@ -4462,14 +4455,14 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err
4462	4455	// test decl with no name. Skip the part where we check against
4463	4456	// the test name filter.
4464	4457	if (!comp.bin_file.options.is_test) break :blk false;
4465		if (decl_pkg != mod.main_pkg) break :blk false;
	4458	if (decl_pkg != mod.main_mod) break :blk false;
4466	4459	try mod.test_functions.put(gpa, new_decl_index, {});
4467	4460	break :blk true;
4468	4461	},
4469	4462	else => blk: {
4470	4463	if (!is_named_test) break :blk false;
4471	4464	if (!comp.bin_file.options.is_test) break :blk false;
4472		if (decl_pkg != mod.main_pkg) break :blk false;
	4465	if (decl_pkg != mod.main_mod) break :blk false;
4473	4466	if (comp.test_filter) \|test_filter\| {
4474	4467	if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) {
4475	4468	break :blk false;
...	...	@@ -5596,8 +5589,8 @@ pub fn populateTestFunctions(
5596	5589	) !void {
5597	5590	const gpa = mod.gpa;
5598	5591	const ip = &mod.intern_pool;
5599		const builtin_pkg = mod.main_pkg.table.get("builtin").?;
5600		const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file;
	5592	const builtin_mod = mod.main_mod.table.get("builtin").?;
	5593	const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file;
5601	5594	const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?);
5602	5595	const builtin_namespace = mod.namespacePtr(root_decl.src_namespace);
5603	5596	const test_functions_str = try ip.getOrPutString(gpa, "test_functions");

src/Package.zig+69-233

...	...	@@ -1,251 +1,87 @@
1		const Package = @This();
2
3		const builtin = @import("builtin");
4		const std = @import("std");
5		const fs = std.fs;
6		const mem = std.mem;
7		const Allocator = mem.Allocator;
8		const ascii = std.ascii;
9		const assert = std.debug.assert;
10		const log = std.log.scoped(.package);
11		const main = @import("main.zig");
12		const ThreadPool = std.Thread.Pool;
13
14		const Compilation = @import("Compilation.zig");
15		const Module = @import("Module.zig");
16		const Cache = std.Build.Cache;
17		const build_options = @import("build_options");
18		const Fetch = @import("Package/Fetch.zig");
19
	1	pub const Module = @import("Package/Module.zig");
	2	pub const Fetch = @import("Package/Fetch.zig");
20	3	pub const build_zig_basename = "build.zig";
21	4	pub const Manifest = @import("Manifest.zig");
22		pub const Table = std.StringHashMapUnmanaged(*Package);
23
24		root_src_directory: Compilation.Directory,
25		/// Relative to `root_src_directory`. May contain path separators.
26		root_src_path: []const u8,
27		/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root'
28		/// are not specified in every dependency table, but instead only in the table of `main_pkg`.
29		/// `Module.importFile` is responsible for detecting these names and using the correct package.
30		table: Table = .{},
31		/// Whether to free `root_src_directory` on `destroy`.
32		root_src_directory_owned: bool = false,
33
34		/// Allocate a Package. No references to the slices passed are kept.
35		pub fn create(
36		gpa: Allocator,
37		/// Null indicates the current working directory
38		root_src_dir_path: ?[]const u8,
39		/// Relative to root_src_dir_path
40		root_src_path: []const u8,
41		) !*Package {
42		const ptr = try gpa.create(Package);
43		errdefer gpa.destroy(ptr);
44
45		const owned_dir_path = if (root_src_dir_path) \|p\| try gpa.dupe(u8, p) else null;
46		errdefer if (owned_dir_path) \|p\| gpa.free(p);
47
48		const owned_src_path = try gpa.dupe(u8, root_src_path);
49		errdefer gpa.free(owned_src_path);
50	5
51		ptr.* = .{
52		.root_src_directory = .{
53		.path = owned_dir_path,
54		.handle = if (owned_dir_path) \|p\| try fs.cwd().openDir(p, .{}) else fs.cwd(),
55		},
56		.root_src_path = owned_src_path,
57		.root_src_directory_owned = true,
58		};
	6	pub const Path = struct {
	7	root_dir: Cache.Directory,
	8	/// The path, relative to the root dir, that this `Path` represents.
	9	/// Empty string means the root_dir is the path.
	10	sub_path: []const u8 = "",
59	11
60		return ptr;
61		}
62
63		pub fn createWithDir(
64		gpa: Allocator,
65		directory: Compilation.Directory,
66		/// Relative to `directory`. If null, means `directory` is the root src dir
67		/// and is owned externally.
68		root_src_dir_path: ?[]const u8,
69		/// Relative to root_src_dir_path
70		root_src_path: []const u8,
71		) !*Package {
72		const ptr = try gpa.create(Package);
73		errdefer gpa.destroy(ptr);
74
75		const owned_src_path = try gpa.dupe(u8, root_src_path);
76		errdefer gpa.free(owned_src_path);
77
78		if (root_src_dir_path) \|p\| {
79		const owned_dir_path = try directory.join(gpa, &[1][]const u8{p});
80		errdefer gpa.free(owned_dir_path);
81
82		ptr.* = .{
83		.root_src_directory = .{
84		.path = owned_dir_path,
85		.handle = try directory.handle.openDir(p, .{}),
86		},
87		.root_src_directory_owned = true,
88		.root_src_path = owned_src_path,
89		};
90		} else {
91		ptr.* = .{
92		.root_src_directory = directory,
93		.root_src_directory_owned = false,
94		.root_src_path = owned_src_path,
95		};
	12	pub fn cwd() Path {
	13	return .{ .root_dir = Cache.Directory.cwd() };
96	14	}
97		return ptr;
98		}
99	15
100		/// Free all memory associated with this package. It does not destroy any packages
101		/// inside its table; the caller is responsible for calling destroy() on them.
102		pub fn destroy(pkg: *Package, gpa: Allocator) void {
103		gpa.free(pkg.root_src_path);
104
105		if (pkg.root_src_directory_owned) {
106		// If root_src_directory.path is null then the handle is the cwd()
107		// which shouldn't be closed.
108		if (pkg.root_src_directory.path) \|p\| {
109		gpa.free(p);
110		pkg.root_src_directory.handle.close();
111		}
	16	pub fn join(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error!Path {
	17	const parts: []const []const u8 =
	18	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
	19	return .{
	20	.root_dir = p.root_dir,
	21	.sub_path = try fs.path.join(allocator, parts),
	22	};
112	23	}
113	24
114		pkg.deinitTable(gpa);
115		gpa.destroy(pkg);
116		}
117
118		/// Only frees memory associated with the table.
119		pub fn deinitTable(pkg: *Package, gpa: Allocator) void {
120		pkg.table.deinit(gpa);
121		}
122
123		pub fn add(pkg: Package, gpa: Allocator, name: []const u8, package: Package) !void {
124		try pkg.table.ensureUnusedCapacity(gpa, 1);
125		const name_dupe = try gpa.dupe(u8, name);
126		pkg.table.putAssumeCapacityNoClobber(name_dupe, package);
127		}
128
129		/// Compute a readable name for the package. The returned name should be freed from gpa. This
130		/// function is very slow, as it traverses the whole package hierarchy to find a path to this
131		/// package. It should only be used for error output.
132		pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 {
133		// we'll do a breadth-first search from the root module to try and find a short name for this
134		// module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is
135		// just the first-found shortest path - a module may be children of arbitrarily many other
136		// modules. This path may vary between executions due to hashmap iteration order, but that
137		// doesn't matter too much.
138		var node_arena = std.heap.ArenaAllocator.init(gpa);
139		defer node_arena.deinit();
140		const Parented = struct {
141		parent: ?*const @This(),
142		mod: *const Package,
143		};
144		const Queue = std.DoublyLinkedList(Parented);
145		var to_check: Queue = .{};
146
147		{
148		const new = try node_arena.allocator().create(Queue.Node);
149		new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } };
150		to_check.prepend(new);
	25	pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
	26	const parts: []const []const u8 =
	27	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
	28	return p.root_dir.join(allocator, parts);
151	29	}
152	30
153		if (mod.main_pkg != mod.root_pkg) {
154		const new = try node_arena.allocator().create(Queue.Node);
155		// TODO: once #12201 is resolved, we may want a way of indicating a different name for this
156		new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } };
157		to_check.prepend(new);
	31	pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 {
	32	const parts: []const []const u8 =
	33	if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path };
	34	return p.root_dir.joinZ(allocator, parts);
158	35	}
159	36
160		// set of modules we've already checked to prevent loops
161		var checked = std.AutoHashMap(*const Package, void).init(gpa);
162		defer checked.deinit();
163
164		const linked = while (to_check.pop()) \|node\| {
165		const check = &node.data;
166
167		if (checked.contains(check.mod)) continue;
168		try checked.put(check.mod, {});
169
170		if (check.mod == target) break check;
171
172		var it = check.mod.table.iterator();
173		while (it.next()) \|kv\| {
174		var new = try node_arena.allocator().create(Queue.Node);
175		new.* = .{ .data = .{
176		.parent = check,
177		.mod = kv.value_ptr.*,
178		} };
179		to_check.prepend(new);
180		}
181		} else {
182		// this can happen for e.g. @cImport packages
183		return gpa.dupe(u8, "<unnamed>");
184		};
185
186		// we found a path to the module! unfortunately, we can only traverse up it, so we have to put
187		// all the names into a buffer so we can then print them in order.
188		var names = std.ArrayList([]const u8).init(gpa);
189		defer names.deinit();
190
191		var cur: *const Parented = linked;
192		while (cur.parent) \|parent\| : (cur = parent) {
193		// find cur's name in parent
194		var it = parent.mod.table.iterator();
195		const name = while (it.next()) \|kv\| {
196		if (kv.value_ptr.* == cur.mod) {
197		break kv.key_ptr.*;
198		}
199		} else unreachable;
200		try names.append(name);
	37	pub fn openFile(
	38	p: Path,
	39	sub_path: []const u8,
	40	flags: fs.File.OpenFlags,
	41	) fs.File.OpenError!fs.File {
	42	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	43	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	44	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	45	p.sub_path, sub_path,
	46	}) catch return error.NameTooLong;
	47	};
	48	return p.root_dir.handle.openFile(joined_path, flags);
201	49	}
202	50
203		// finally, print the names into a buffer!
204		var buf = std.ArrayList(u8).init(gpa);
205		defer buf.deinit();
206		try buf.writer().writeAll("root");
207		var i: usize = names.items.len;
208		while (i > 0) {
209		i -= 1;
210		try buf.writer().print(".{s}", .{names.items[i]});
	51	pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir {
	52	var buf: [fs.MAX_PATH_BYTES]u8 = undefined;
	53	const joined_path = if (p.sub_path.len == 0) sub_path else p: {
	54	break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{
	55	p.sub_path, sub_path,
	56	}) catch return error.NameTooLong;
	57	};
	58	return p.root_dir.handle.makeOpenPath(joined_path, opts);
211	59	}
212	60
213		return buf.toOwnedSlice();
214		}
215
216		pub fn createFilePkg(
217		gpa: Allocator,
218		cache_directory: Compilation.Directory,
219		basename: []const u8,
220		contents: []const u8,
221		) !*Package {
222		const rand_int = std.crypto.random.int(u64);
223		const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int);
224		{
225		var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{});
226		defer tmp_dir.close();
227		try tmp_dir.writeFile(basename, contents);
	61	pub fn format(
	62	self: Path,
	63	comptime fmt_string: []const u8,
	64	options: std.fmt.FormatOptions,
	65	writer: anytype,
	66	) !void {
	67	_ = options;
	68	if (fmt_string.len > 0)
	69	std.fmt.invalidFmtError(fmt_string, self);
	70	if (self.root_dir.path) \|p\| {
	71	try writer.writeAll(p);
	72	try writer.writeAll(fs.path.sep_str);
	73	}
	74	if (self.sub_path.len > 0) {
	75	try writer.writeAll(self.sub_path);
	76	try writer.writeAll(fs.path.sep_str);
	77	}
228	78	}
229
230		var hh: Cache.HashHelper = .{};
231		hh.addBytes(build_options.version);
232		hh.addBytes(contents);
233		const hex_digest = hh.final();
234
235		const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest;
236		try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path);
237
238		return createWithDir(gpa, cache_directory, o_dir_sub_path, basename);
239		}
240
241		const hex_multihash_len = 2 * Manifest.multihash_len;
242		const MultiHashHexDigest = [hex_multihash_len]u8;
243
244		const DependencyModule = union(enum) {
245		zig_pkg: *Package,
246		non_zig_pkg: *Package,
247	79	};
248		/// This is to avoid creating multiple modules for the same build.zig file.
249		/// If the value is `null`, the package is a known dependency, but has not yet
250		/// been fetched.
251		pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule);
	80
	81	const Package = @This();
	82	const builtin = @import("builtin");
	83	const std = @import("std");
	84	const fs = std.fs;
	85	const Allocator = std.mem.Allocator;
	86	const assert = std.debug.assert;
	87	const Cache = std.Build.Cache;

src/Package/Fetch.zig+446-205

...	...	@@ -27,59 +27,84 @@
27	27	//! All of this must be done with only referring to the state inside this struct
28	28	//! because this work will be done in a dedicated thread.
29	29
30		/// Try to avoid this as much as possible since arena will have less contention.
31		gpa: Allocator,
32	30	arena: std.heap.ArenaAllocator,
33	31	location: Location,
34	32	location_tok: std.zig.Ast.TokenIndex,
35	33	hash_tok: std.zig.Ast.TokenIndex,
36		global_cache: Cache.Directory,
37		parent_package_root: Path,
	34	parent_package_root: Package.Path,
38	35	parent_manifest_ast: ?*const std.zig.Ast,
39	36	prog_node: *std.Progress.Node,
40		http_client: *std.http.Client,
41		thread_pool: *ThreadPool,
42	37	job_queue: *JobQueue,
43		wait_group: *WaitGroup,
	38	/// If true, don't add an error for a missing hash. This flag is not passed
	39	/// down to recursive dependencies. It's intended to be used only be the CLI.
	40	omit_missing_hash_error: bool,
44	41
45	42	// Above this are fields provided as inputs to `run`.
46	43	// Below this are fields populated by `run`.
47	44
48	45	/// This will either be relative to `global_cache`, or to the build root of
49	46	/// the root package.
50		package_root: Path,
	47	package_root: Package.Path,
51	48	error_bundle: std.zig.ErrorBundle.Wip,
52	49	manifest: ?Manifest,
53		manifest_ast: ?*std.zig.Ast,
54		actual_hash: Digest,
	50	manifest_ast: std.zig.Ast,
	51	actual_hash: Manifest.Digest,
55	52	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
56	53	has_build_zig: bool,
57	54	/// Indicates whether the task aborted due to an out-of-memory condition.
58	55	oom_flag: bool,
59	56
	57	/// Contains shared state among all `Fetch` tasks.
60	58	pub const JobQueue = struct {
61	59	mutex: std.Thread.Mutex = .{},
62		};
63
64		pub const Digest = [Manifest.Hash.digest_length]u8;
65		pub const MultiHashHexDigest = [hex_multihash_len]u8;
66
67		pub const Path = struct {
68		root_dir: Cache.Directory,
69		/// The path, relative to the root dir, that this `Path` represents.
70		/// Empty string means the root_dir is the path.
71		sub_path: []const u8 = "",
	60	/// Protected by `mutex`.
	61	table: Table = .{},
	62	/// `table` may be missing some tasks such as ones that failed, so this
	63	/// field contains references to all of them.
	64	/// Protected by `mutex`.
	65	all_fetches: std.ArrayListUnmanaged(*Fetch) = .{},
	66
	67	http_client: *std.http.Client,
	68	thread_pool: *ThreadPool,
	69	wait_group: WaitGroup = .{},
	70	global_cache: Cache.Directory,
	71	recursive: bool,
	72	work_around_btrfs_bug: bool,
	73
	74	pub const Table = std.AutoHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
	75
	76	pub fn deinit(jq: *JobQueue) void {
	77	if (jq.all_fetches.items.len == 0) return;
	78	const gpa = jq.all_fetches.items[0].arena.child_allocator;
	79	jq.table.deinit(gpa);
	80	// These must be deinitialized in reverse order because subsequent
	81	// `Fetch` instances are allocated in prior ones' arenas.
	82	// Sorry, I know it's a bit weird, but it slightly simplifies the
	83	// critical section.
	84	while (jq.all_fetches.popOrNull()) \|f\| f.deinit();
	85	jq.all_fetches.deinit(gpa);
	86	jq.* = undefined;
	87	}
72	88	};
73	89
74	90	pub const Location = union(enum) {
75	91	remote: Remote,
	92	/// A directory found inside the parent package.
76	93	relative_path: []const u8,
	94	/// Recursive Fetch tasks will never use this Location, but it may be
	95	/// passed in by the CLI. Indicates the file contents here should be copied
	96	/// into the global package cache. It may be a file relative to the cwd or
	97	/// absolute, in which case it should be treated exactly like a `file://`
	98	/// URL, or a directory, in which case it should be treated as an
	99	/// already-unpacked directory (but still needs to be copied into the
	100	/// global package cache and have inclusion rules applied).
	101	path_or_url: []const u8,
77	102
78	103	pub const Remote = struct {
79	104	url: []const u8,
80	105	/// If this is null it means the user omitted the hash field from a dependency.
81	106	/// It will be an error but the logic should still fetch and print the discovered hash.
82		hash: ?[hex_multihash_len]u8,
	107	hash: ?Manifest.MultiHashHexDigest,
83	108	};
84	109	};
85	110
...	...	@@ -92,7 +117,11 @@ pub const RunError = error{
92	117
93	118	pub fn run(f: *Fetch) RunError!void {
94	119	const eb = &f.error_bundle;
95		const arena = f.arena_allocator.allocator();
	120	const arena = f.arena.allocator();
	121	const gpa = f.arena.child_allocator;
	122	const cache_root = f.job_queue.global_cache;
	123
	124	try eb.init(gpa);
96	125
97	126	// Check the global zig package cache to see if the hash already exists. If
98	127	// so, load, parse, and validate the build.zig.zon file therein, and skip
...	...	@@ -111,43 +140,66 @@ pub fn run(f: *Fetch) RunError!void {
111	140	);
112	141	f.package_root = try f.parent_package_root.join(arena, sub_path);
113	142	try loadManifest(f, f.package_root);
	143	if (!f.job_queue.recursive) return;
114	144	// Package hashes are used as unique identifiers for packages, so
115	145	// we still need one for relative paths.
116		const hash = h: {
	146	const digest = h: {
117	147	var hasher = Manifest.Hash.init(.{});
118	148	// This hash is a tuple of:
119	149	// * whether it relative to the global cache directory or to the root package
120	150	// * the relative file path from there to the build root of the package
121		hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle)
	151	hasher.update(if (f.package_root.root_dir.eql(cache_root))
122	152	&package_hash_prefix_cached
123	153	else
124	154	&package_hash_prefix_project);
125	155	hasher.update(f.package_root.sub_path);
126	156	break :h hasher.finalResult();
127	157	};
128		return queueJobsForDeps(f, hash);
	158	return queueJobsForDeps(f, Manifest.hexDigest(digest));
129	159	},
130	160	.remote => \|remote\| remote,
	161	.path_or_url => \|path_or_url\| {
	162	if (fs.cwd().openIterableDir(path_or_url, .{})) \|dir\| {
	163	var resource: Resource = .{ .dir = dir };
	164	return runResource(f, path_or_url, &resource, null);
	165	} else \|dir_err\| {
	166	const file_err = if (dir_err == error.NotDir) e: {
	167	if (fs.cwd().openFile(path_or_url, .{})) \|file\| {
	168	var resource: Resource = .{ .file = file };
	169	return runResource(f, path_or_url, &resource, null);
	170	} else \|err\| break :e err;
	171	} else dir_err;
	172
	173	const uri = std.Uri.parse(path_or_url) catch \|uri_err\| {
	174	return f.fail(0, try eb.printString(
	175	"'{s}' could not be recognized as a file path ({s}) or an URL ({s})",
	176	.{ path_or_url, @errorName(file_err), @errorName(uri_err) },
	177	));
	178	};
	179	var resource = try f.initResource(uri);
	180	return runResource(f, uri.path, &resource, null);
	181	}
	182	},
131	183	};
	184
132	185	const s = fs.path.sep_str;
133	186	if (remote.hash) \|expected_hash\| {
134	187	const pkg_sub_path = "p" ++ s ++ expected_hash;
135		if (f.global_cache.handle.access(pkg_sub_path, .{})) \|_\| {
	188	if (cache_root.handle.access(pkg_sub_path, .{})) \|_\| {
136	189	f.package_root = .{
137		.root_dir = f.global_cache,
	190	.root_dir = cache_root,
138	191	.sub_path = pkg_sub_path,
139	192	};
140	193	try loadManifest(f, f.package_root);
	194	if (!f.job_queue.recursive) return;
141	195	return queueJobsForDeps(f, expected_hash);
142	196	} else \|err\| switch (err) {
143	197	error.FileNotFound => {},
144	198	else => \|e\| {
145	199	try eb.addRootErrorMessage(.{
146	200	.msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{
147		try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e),
	201	try cache_root.join(arena, &.{pkg_sub_path}), @errorName(e),
148	202	}),
149		.src_loc = .none,
150		.notes_len = 0,
151	203	});
152	204	return error.FetchFailed;
153	205	},
...	...	@@ -158,22 +210,50 @@ pub fn run(f: *Fetch) RunError!void {
158	210
159	211	const uri = std.Uri.parse(remote.url) catch \|err\| return f.fail(
160	212	f.location_tok,
161		"invalid URI: {s}",
162		.{@errorName(err)},
	213	try eb.printString("invalid URI: {s}", .{@errorName(err)}),
163	214	);
	215	var resource = try f.initResource(uri);
	216	return runResource(f, uri.path, &resource, remote.hash);
	217	}
	218
	219	pub fn deinit(f: *Fetch) void {
	220	f.error_bundle.deinit();
	221	f.arena.deinit();
	222	}
	223
	224	/// Consumes `resource`, even if an error is returned.
	225	fn runResource(
	226	f: *Fetch,
	227	uri_path: []const u8,
	228	resource: *Resource,
	229	remote_hash: ?Manifest.MultiHashHexDigest,
	230	) RunError!void {
	231	defer resource.deinit();
	232	const arena = f.arena.allocator();
	233	const eb = &f.error_bundle;
	234	const s = fs.path.sep_str;
	235	const cache_root = f.job_queue.global_cache;
164	236	const rand_int = std.crypto.random.int(u64);
165	237	const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int);
166	238
	239	const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path});
167	240	var tmp_directory: Cache.Directory = .{
168		.path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}),
169		.handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir,
	241	.path = tmp_directory_path,
	242	.handle = handle: {
	243	const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch \|err\| {
	244	try eb.addRootErrorMessage(.{
	245	.msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{
	246	tmp_directory_path, @errorName(err),
	247	}),
	248	});
	249	return error.FetchFailed;
	250	};
	251	break :handle dir.dir;
	252	},
170	253	};
171	254	defer tmp_directory.handle.close();
172	255
173		var resource = try f.initResource(uri);
174		defer resource.deinit(); // releases more than memory
175
176		try f.unpackResource(&resource, uri.path, tmp_directory);
	256	try unpackResource(f, resource, uri_path, tmp_directory);
177	257
178	258	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
179	259	// for the file to be missing, in which case this fetched package is
...	...	@@ -194,15 +274,15 @@ pub fn run(f: *Fetch) RunError!void {
194	274	// Compute the package hash based on the remaining files in the temporary
195	275	// directory.
196	276
197		if (builtin.os.tag == .linux and f.work_around_btrfs_bug) {
	277	if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) {
198	278	// https://github.com/ziglang/zig/issues/17095
199	279	tmp_directory.handle.close();
200		const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch
	280	const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch
201	281	@panic("btrfs workaround failed");
202	282	tmp_directory.handle = iterable_dir.dir;
203	283	}
204	284
205		f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter);
	285	f.actual_hash = try computeHash(f, tmp_directory, filter);
206	286
207	287	// Rename the temporary directory into the global zig package cache
208	288	// directory. If the hash already exists, delete the temporary directory
...	...	@@ -211,40 +291,54 @@ pub fn run(f: *Fetch) RunError!void {
211	291	// package with the different hash is used in the future.
212	292
213	293	const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash);
214		try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path);
	294	renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, dest_pkg_sub_path) catch \|err\| {
	295	const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
	296	const dest = try cache_root.join(arena, &.{dest_pkg_sub_path});
	297	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	298	"unable to rename temporary directory '{s}' into package cache directory '{s}': {s}",
	299	.{ src, dest, @errorName(err) },
	300	) });
	301	return error.FetchFailed;
	302	};
215	303
216	304	// Validate the computed hash against the expected hash. If invalid, this
217	305	// job is done.
218	306
219	307	const actual_hex = Manifest.hexDigest(f.actual_hash);
220		if (remote.hash) \|declared_hash\| {
221		if (!std.mem.eql(u8, declared_hash, &actual_hex)) {
222		return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{
223		declared_hash, actual_hex,
224		});
	308	if (remote_hash) \|declared_hash\| {
	309	if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
	310	return f.fail(f.hash_tok, try eb.printString(
	311	"hash mismatch: manifest declares {s} but the fetched package has {s}",
	312	.{ declared_hash, actual_hex },
	313	));
225	314	}
226		} else {
	315	} else if (!f.omit_missing_hash_error) {
227	316	const notes_len = 1;
228		try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field");
	317	try eb.addRootErrorMessage(.{
	318	.msg = try eb.addString("dependency is missing hash field"),
	319	.src_loc = try f.srcLoc(f.location_tok),
	320	.notes_len = notes_len,
	321	});
229	322	const notes_start = try eb.reserveNotes(notes_len);
230	323	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
231	324	.msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
232	325	}));
233		return error.PackageFetchFailed;
	326	return error.FetchFailed;
234	327	}
235	328
236	329	// Spawn a new fetch job for each dependency in the manifest file. Use
237	330	// a mutex and a hash map so that redundant jobs do not get queued up.
238		return queueJobsForDeps(f, .{ .hash = f.actual_hash });
	331	if (!f.job_queue.recursive) return;
	332	return queueJobsForDeps(f, actual_hex);
239	333	}
240	334
241	335	/// This function populates `f.manifest` or leaves it `null`.
242		fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
	336	fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void {
243	337	const eb = &f.error_bundle;
244		const arena = f.arena_allocator.allocator();
245		const manifest_bytes = pkg_root.readFileAllocOptions(
	338	const arena = f.arena.allocator();
	339	const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions(
246	340	arena,
247		Manifest.basename,
	341	try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }),
248	342	Manifest.max_bytes,
249	343	null,
250	344	1,
...	...	@@ -252,39 +346,39 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
252	346	) catch \|err\| switch (err) {
253	347	error.FileNotFound => return,
254	348	else => \|e\| {
255		const file_path = try pkg_root.join(arena, .{Manifest.basename});
	349	const file_path = try pkg_root.join(arena, Manifest.basename);
256	350	try eb.addRootErrorMessage(.{
257		.msg = try eb.printString("unable to load package manifest '{s}': {s}", .{
	351	.msg = try eb.printString("unable to load package manifest '{}': {s}", .{
258	352	file_path, @errorName(e),
259	353	}),
260		.src_loc = .none,
261		.notes_len = 0,
262	354	});
	355	return error.FetchFailed;
263	356	},
264	357	};
265	358
266		var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
267		f.manifest_ast = ast;
	359	const ast = &f.manifest_ast;
	360	ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon);
268	361
269	362	if (ast.errors.len > 0) {
270		const file_path = try pkg_root.join(arena, .{Manifest.basename});
271		try main.putAstErrorsIntoBundle(arena, ast, file_path, eb);
272		return error.PackageFetchFailed;
	363	const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root});
	364	try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb);
	365	return error.FetchFailed;
273	366	}
274	367
275		f.manifest = try Manifest.parse(arena, ast);
	368	f.manifest = try Manifest.parse(arena, ast.*);
	369	const manifest = &f.manifest.?;
276	370
277		if (f.manifest.errors.len > 0) {
278		const file_path = try pkg_root.join(arena, .{Manifest.basename});
	371	if (manifest.errors.len > 0) {
	372	const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename });
279	373	const token_starts = ast.tokens.items(.start);
280	374
281		for (f.manifest.errors) \|msg\| {
	375	for (manifest.errors) \|msg\| {
282	376	const start_loc = ast.tokenLocation(0, msg.tok);
283	377
284	378	try eb.addRootErrorMessage(.{
285	379	.msg = try eb.addString(msg.msg),
286	380	.src_loc = try eb.addSourceLocation(.{
287		.src_path = try eb.addString(file_path),
	381	.src_path = src_path,
288	382	.span_start = token_starts[msg.tok],
289	383	.span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len),
290	384	.span_main = token_starts[msg.tok] + msg.off,
...	...	@@ -292,71 +386,80 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
292	386	.column = @intCast(start_loc.column),
293	387	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
294	388	}),
295		.notes_len = 0,
296	389	});
297	390	}
298		return error.PackageFetchFailed;
	391	return error.FetchFailed;
299	392	}
300	393	}
301	394
302		fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
	395	fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void {
	396	assert(f.job_queue.recursive);
	397
303	398	// If the package does not have a build.zig.zon file then there are no dependencies.
304	399	const manifest = f.manifest orelse return;
305	400
306	401	const new_fetches = nf: {
	402	const deps = manifest.dependencies.values();
	403	const gpa = f.arena.child_allocator;
307	404	// Grab the new tasks into a temporary buffer so we can unlock that mutex
308	405	// as fast as possible.
309	406	// This overallocates any fetches that get skipped by the `continue` in the
310	407	// loop below.
311		const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count());
	408	const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len);
312	409	var new_fetch_index: usize = 0;
313	410
314		f.job_queue.lock();
315		defer f.job_queue.unlock();
	411	f.job_queue.mutex.lock();
	412	defer f.job_queue.mutex.unlock();
	413
	414	try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
	415	try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1));
316	416
317	417	// It is impossible for there to be a collision here. Consider all three cases:
318	418	// * Correct hash is provided by manifest.
319	419	// - Redundant jobs are skipped in the loop below.
320		// * Incorrect has is provided by manifest.
	420	// * Incorrect hash is provided by manifest.
321	421	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
322	422	// * Hash is not provided by manifest.
323	423	// - Hash missing error emitted; `queueJobsForDeps` is not called.
324		try f.job_queue.finish(hash, f, new_fetches.len);
	424	f.job_queue.table.putAssumeCapacityNoClobber(hash, f);
325	425
326		for (manifest.dependencies.values()) \|dep\| {
	426	for (deps) \|dep\| {
	427	const new_fetch = &new_fetches[new_fetch_index];
327	428	const location: Location = switch (dep.location) {
328	429	.url => \|url\| .{ .remote = .{
329	430	.url = url,
330		.hash = if (dep.hash) \|h\| h[0..hex_multihash_len].* else null,
	431	.hash = h: {
	432	const h = dep.hash orelse break :h null;
	433	const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len;
	434	const multihash_digest = h[0..digest_len].*;
	435	const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
	436	if (gop.found_existing) continue;
	437	gop.value_ptr.* = new_fetch;
	438	break :h multihash_digest;
	439	},
331	440	} },
332	441	.path => \|path\| .{ .relative_path = path },
333	442	};
334		const new_fetch = &new_fetches[new_fetch_index];
335		const already_done = f.job_queue.add(location, new_fetch);
336		if (already_done) continue;
337	443	new_fetch_index += 1;
338
	444	f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
339	445	new_fetch.* = .{
340		.gpa = f.gpa,
341		.arena = std.heap.ArenaAllocator.init(f.gpa),
	446	.arena = std.heap.ArenaAllocator.init(gpa),
342	447	.location = location,
343	448	.location_tok = dep.location_tok,
344	449	.hash_tok = dep.hash_tok,
345		.global_cache = f.global_cache,
346	450	.parent_package_root = f.package_root,
347		.parent_manifest_ast = f.manifest_ast.?,
	451	.parent_manifest_ast = &f.manifest_ast,
348	452	.prog_node = f.prog_node,
349		.http_client = f.http_client,
350		.thread_pool = f.thread_pool,
351	453	.job_queue = f.job_queue,
352		.wait_group = f.wait_group,
	454	.omit_missing_hash_error = false,
353	455
354	456	.package_root = undefined,
355		.error_bundle = .{},
	457	.error_bundle = undefined,
356	458	.manifest = null,
357		.manifest_ast = null,
	459	.manifest_ast = undefined,
358	460	.actual_hash = undefined,
359	461	.has_build_zig = false,
	462	.oom_flag = false,
360	463	};
361	464	}
362	465
...	...	@@ -364,12 +467,14 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
364	467	};
365	468
366	469	// Now it's time to give tasks to the thread pool.
367		for (new_fetches) \|new_fetch\| {
368		f.wait_group.start();
369		f.thread_pool.spawn(workerRun, .{f}) catch \|err\| switch (err) {
	470	const thread_pool = f.job_queue.thread_pool;
	471
	472	for (new_fetches) \|*new_fetch\| {
	473	f.job_queue.wait_group.start();
	474	thread_pool.spawn(workerRun, .{new_fetch}) catch \|err\| switch (err) {
370	475	error.OutOfMemory => {
371	476	new_fetch.oom_flag = true;
372		f.wait_group.finish();
	477	f.job_queue.wait_group.finish();
373	478	continue;
374	479	},
375	480	};
...	...	@@ -377,43 +482,83 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void {
377	482	}
378	483
379	484	fn workerRun(f: *Fetch) void {
380		defer f.wait_group.finish();
	485	defer f.job_queue.wait_group.finish();
381	486	run(f) catch \|err\| switch (err) {
382	487	error.OutOfMemory => f.oom_flag = true,
383		error.FetchFailed => {}, // See `error_bundle`.
	488	error.FetchFailed => {
	489	// Nothing to do because the errors are already reported in `error_bundle`,
	490	// and a reference is kept to the `Fetch` task inside `all_fetches`.
	491	},
384	492	};
385	493	}
386	494
387		fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void {
388		const ast = f.parent_manifest_ast;
389		const token_starts = ast.tokens.items(.start);
390		const start_loc = ast.tokenLocation(0, msg_tok);
	495	fn srcLoc(
	496	f: *Fetch,
	497	tok: std.zig.Ast.TokenIndex,
	498	) Allocator.Error!std.zig.ErrorBundle.SourceLocationIndex {
	499	const ast = f.parent_manifest_ast orelse return .none;
391	500	const eb = &f.error_bundle;
392		const file_path = try f.parent_package_root.join(f.arena, Manifest.basename);
	501	const token_starts = ast.tokens.items(.start);
	502	const start_loc = ast.tokenLocation(0, tok);
	503	const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root});
393	504	const msg_off = 0;
	505	return eb.addSourceLocation(.{
	506	.src_path = src_path,
	507	.span_start = token_starts[tok],
	508	.span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len),
	509	.span_main = token_starts[tok] + msg_off,
	510	.line = @intCast(start_loc.line),
	511	.column = @intCast(start_loc.column),
	512	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
	513	});
	514	}
394	515
	516	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
	517	const eb = &f.error_bundle;
395	518	try eb.addRootErrorMessage(.{
396	519	.msg = msg_str,
397		.src_loc = try eb.addSourceLocation(.{
398		.src_path = try eb.addString(file_path),
399		.span_start = token_starts[msg_tok],
400		.span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len),
401		.span_main = token_starts[msg_tok] + msg_off,
402		.line = @intCast(start_loc.line),
403		.column = @intCast(start_loc.column),
404		.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
405		}),
406		.notes_len = 0,
	520	.src_loc = try f.srcLoc(msg_tok),
407	521	});
408
409	522	return error.FetchFailed;
410	523	}
411	524
412	525	const Resource = union(enum) {
413	526	file: fs.File,
414	527	http_request: std.http.Client.Request,
415		git_fetch_stream: git.Session.FetchStream,
	528	git: Git,
416	529	dir: fs.IterableDir,
	530
	531	const Git = struct {
	532	fetch_stream: git.Session.FetchStream,
	533	want_oid: [git.oid_length]u8,
	534	};
	535
	536	fn deinit(resource: *Resource) void {
	537	switch (resource.*) {
	538	.file => \|*file\| file.close(),
	539	.http_request => \|*req\| req.deinit(),
	540	.git => \|*git_resource\| git_resource.fetch_stream.deinit(),
	541	.dir => \|*dir\| dir.close(),
	542	}
	543	resource.* = undefined;
	544	}
	545
	546	fn reader(resource: *Resource) std.io.AnyReader {
	547	return .{
	548	.context = resource,
	549	.readFn = read,
	550	};
	551	}
	552
	553	fn read(context: *const anyopaque, buffer: []u8) anyerror!usize {
	554	const resource: *Resource = @constCast(@ptrCast(@alignCast(context)));
	555	switch (resource.*) {
	556	.file => \|*f\| return f.read(buffer),
	557	.http_request => \|*r\| return r.read(buffer),
	558	.git => \|*g\| return g.fetch_stream.read(buffer),
	559	.dir => unreachable,
	560	}
	561	}
417	562	};
418	563
419	564	const FileType = enum {
...	...	@@ -468,30 +613,52 @@ const FileType = enum {
468	613	};
469	614
470	615	fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
471		const gpa = f.gpa;
472		const arena = f.arena_allocator.allocator();
	616	const gpa = f.arena.child_allocator;
	617	const arena = f.arena.allocator();
473	618	const eb = &f.error_bundle;
474	619
475	620	if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{
476		.file = try f.parent_package_root.openFile(uri.path, .{}),
	621	.file = f.parent_package_root.openFile(uri.path, .{}) catch \|err\| {
	622	return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{
	623	f.parent_package_root, uri.path, @errorName(err),
	624	}));
	625	},
477	626	};
478	627
	628	const http_client = f.job_queue.http_client;
	629
479	630	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
480	631	ascii.eqlIgnoreCase(uri.scheme, "https"))
481	632	{
482	633	var h = std.http.Headers{ .allocator = gpa };
483	634	defer h.deinit();
484	635
485		var req = try f.http_client.request(.GET, uri, h, .{});
	636	var req = http_client.request(.GET, uri, h, .{}) catch \|err\| {
	637	return f.fail(f.location_tok, try eb.printString(
	638	"unable to connect to server: {s}",
	639	.{@errorName(err)},
	640	));
	641	};
486	642	errdefer req.deinit(); // releases more than memory
487	643
488		try req.start(.{});
489		try req.wait();
	644	req.start(.{}) catch \|err\| {
	645	return f.fail(f.location_tok, try eb.printString(
	646	"HTTP request failed: {s}",
	647	.{@errorName(err)},
	648	));
	649	};
	650	req.wait() catch \|err\| {
	651	return f.fail(f.location_tok, try eb.printString(
	652	"invalid HTTP response: {s}",
	653	.{@errorName(err)},
	654	));
	655	};
490	656
491	657	if (req.response.status != .ok) {
492		return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{
493		@intFromEnum(req.response.status), req.response.status.phrase() orelse "",
494		});
	658	return f.fail(f.location_tok, try eb.printString(
	659	"bad HTTP response code: '{d} {s}'",
	660	.{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" },
	661	));
495	662	}
496	663
497	664	return .{ .http_request = req };
...	...	@@ -503,13 +670,21 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
503	670	var transport_uri = uri;
504	671	transport_uri.scheme = uri.scheme["git+".len..];
505	672	var redirect_uri: []u8 = undefined;
506		var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri };
507		session.discoverCapabilities(gpa, &redirect_uri) catch \|e\| switch (e) {
	673	var session: git.Session = .{ .transport = http_client, .uri = transport_uri };
	674	session.discoverCapabilities(gpa, &redirect_uri) catch \|err\| switch (err) {
508	675	error.Redirected => {
509	676	defer gpa.free(redirect_uri);
510		return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri});
	677	return f.fail(f.location_tok, try eb.printString(
	678	"repository moved to {s}",
	679	.{redirect_uri},
	680	));
	681	},
	682	else => \|e\| {
	683	return f.fail(f.location_tok, try eb.printString(
	684	"unable to discover remote git server capabilities: {s}",
	685	.{@errorName(e)},
	686	));
511	687	},
512		else => \|other\| return other,
513	688	};
514	689
515	690	const want_oid = want_oid: {
...	...	@@ -519,12 +694,22 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
519	694	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
520	695	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
521	696
522		var ref_iterator = try session.listRefs(gpa, .{
	697	var ref_iterator = session.listRefs(gpa, .{
523	698	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
524	699	.include_peeled = true,
525		});
	700	}) catch \|err\| {
	701	return f.fail(f.location_tok, try eb.printString(
	702	"unable to list refs: {s}",
	703	.{@errorName(err)},
	704	));
	705	};
526	706	defer ref_iterator.deinit();
527		while (try ref_iterator.next()) \|ref\| {
	707	while (ref_iterator.next() catch \|err\| {
	708	return f.fail(f.location_tok, try eb.printString(
	709	"unable to iterate refs: {s}",
	710	.{@errorName(err)},
	711	));
	712	}) \|ref\| {
528	713	if (std.mem.eql(u8, ref.name, want_ref) or
529	714	std.mem.eql(u8, ref.name, want_ref_head) or
530	715	std.mem.eql(u8, ref.name, want_ref_tag))
...	...	@@ -532,31 +717,46 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource {
532	717	break :want_oid ref.peeled orelse ref.oid;
533	718	}
534	719	}
535		return f.fail(f.location_tok, "ref not found: {s}", .{want_ref});
	720	return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
536	721	};
537	722	if (uri.fragment == null) {
538	723	const notes_len = 1;
539		try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref");
	724	try eb.addRootErrorMessage(.{
	725	.msg = try eb.addString("url field is missing an explicit ref"),
	726	.src_loc = try f.srcLoc(f.location_tok),
	727	.notes_len = notes_len,
	728	});
540	729	const notes_start = try eb.reserveNotes(notes_len);
541	730	eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
542	731	.msg = try eb.printString("try .url = \"{+/}#{}\",", .{
543	732	uri, std.fmt.fmtSliceHexLower(&want_oid),
544	733	}),
545	734	}));
546		return error.PackageFetchFailed;
	735	return error.FetchFailed;
547	736	}
548	737
549	738	var want_oid_buf: [git.fmt_oid_length]u8 = undefined;
550	739	_ = std.fmt.bufPrint(&want_oid_buf, "{}", .{
551	740	std.fmt.fmtSliceHexLower(&want_oid),
552	741	}) catch unreachable;
553		var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf});
	742	var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch \|err\| {
	743	return f.fail(f.location_tok, try eb.printString(
	744	"unable to create fetch stream: {s}",
	745	.{@errorName(err)},
	746	));
	747	};
554	748	errdefer fetch_stream.deinit();
555	749
556		return .{ .git_fetch_stream = fetch_stream };
	750	return .{ .git = .{
	751	.fetch_stream = fetch_stream,
	752	.want_oid = want_oid,
	753	} };
557	754	}
558	755
559		return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme});
	756	return f.fail(f.location_tok, try eb.printString(
	757	"unsupported URL scheme: {s}",
	758	.{uri.scheme},
	759	));
560	760	}
561	761
562	762	fn unpackResource(
...	...	@@ -565,52 +765,62 @@ fn unpackResource(
565	765	uri_path: []const u8,
566	766	tmp_directory: Cache.Directory,
567	767	) RunError!void {
	768	const eb = &f.error_bundle;
568	769	const file_type = switch (resource.*) {
569	770	.file => FileType.fromPath(uri_path) orelse
570		return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}),
	771	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
571	772
572	773	.http_request => \|req\| ft: {
573	774	// Content-Type takes first precedence.
574	775	const content_type = req.response.headers.getFirstValue("Content-Type") orelse
575		return f.fail(f.location_tok, "missing 'Content-Type' header", .{});
	776	return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
576	777
577	778	if (ascii.eqlIgnoreCase(content_type, "application/x-tar"))
578		return .tar;
	779	break :ft .tar;
579	780
580	781	if (ascii.eqlIgnoreCase(content_type, "application/gzip") or
581	782	ascii.eqlIgnoreCase(content_type, "application/x-gzip") or
582	783	ascii.eqlIgnoreCase(content_type, "application/tar+gzip"))
583	784	{
584		return .@"tar.gz";
	785	break :ft .@"tar.gz";
585	786	}
586	787
587	788	if (ascii.eqlIgnoreCase(content_type, "application/x-xz"))
588		return .@"tar.xz";
	789	break :ft .@"tar.xz";
589	790
590	791	if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) {
591		return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{
592		content_type,
593		});
	792	return f.fail(f.location_tok, try eb.printString(
	793	"unrecognized 'Content-Type' header: '{s}'",
	794	.{content_type},
	795	));
594	796	}
595	797
596	798	// Next, the filename from 'content-disposition: attachment' takes precedence.
597	799	if (req.response.headers.getFirstValue("Content-Disposition")) \|cd_header\| {
598		break :ft FileType.fromContentDisposition(cd_header) orelse
599		return f.fail(
600		f.location_tok,
601		"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
602		.{cd_header},
603		);
	800	break :ft FileType.fromContentDisposition(cd_header) orelse {
	801	return f.fail(f.location_tok, try eb.printString(
	802	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
	803	.{cd_header},
	804	));
	805	};
604	806	}
605	807
606	808	// Finally, the path from the URI is used.
607		break :ft FileType.fromPath(uri_path) orelse
608		return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path});
	809	break :ft FileType.fromPath(uri_path) orelse {
	810	return f.fail(f.location_tok, try eb.printString(
	811	"unknown file type: '{s}'",
	812	.{uri_path},
	813	));
	814	};
609	815	},
610		.git_fetch_stream => return .git_pack,
611		.dir => \|dir\| {
612		try f.recursiveDirectoryCopy(dir, tmp_directory.handle);
613		return;
	816
	817	.git => .git_pack,
	818
	819	.dir => \|dir\| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch \|err\| {
	820	return f.fail(f.location_tok, try eb.printString(
	821	"unable to copy directory '{s}': {s}",
	822	.{ uri_path, @errorName(err) },
	823	));
614	824	},
615	825	};
616	826
...	...	@@ -618,7 +828,14 @@ fn unpackResource(
618	828	.tar => try unpackTarball(f, tmp_directory.handle, resource.reader()),
619	829	.@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip),
620	830	.@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz),
621		.git_pack => try unpackGitPack(f, tmp_directory.handle, resource),
	831	.git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch \|err\| switch (err) {
	832	error.FetchFailed => return error.FetchFailed,
	833	error.OutOfMemory => return error.OutOfMemory,
	834	else => \|e\| return f.fail(f.location_tok, try eb.printString(
	835	"unable to unpack git files: {s}",
	836	.{@errorName(e)},
	837	)),
	838	},
622	839	}
623	840	}
624	841
...	...	@@ -628,11 +845,17 @@ fn unpackTarballCompressed(
628	845	resource: *Resource,
629	846	comptime Compression: type,
630	847	) RunError!void {
631		const gpa = f.gpa;
	848	const gpa = f.arena.child_allocator;
	849	const eb = &f.error_bundle;
632	850	const reader = resource.reader();
633	851	var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader);
634	852
635		var decompress = try Compression.decompress(gpa, br.reader());
	853	var decompress = Compression.decompress(gpa, br.reader()) catch \|err\| {
	854	return f.fail(f.location_tok, try eb.printString(
	855	"unable to decompress tarball: {s}",
	856	.{@errorName(err)},
	857	));
	858	};
636	859	defer decompress.deinit();
637	860
638	861	return unpackTarball(f, out_dir, decompress.reader());
...	...	@@ -640,11 +863,12 @@ fn unpackTarballCompressed(
640	863
641	864	fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
642	865	const eb = &f.error_bundle;
	866	const gpa = f.arena.child_allocator;
643	867
644		var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa };
	868	var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa };
645	869	defer diagnostics.deinit();
646	870
647		try std.tar.pipeToFileSystem(out_dir, reader, .{
	871	std.tar.pipeToFileSystem(out_dir, reader, .{
648	872	.diagnostics = &diagnostics,
649	873	.strip_components = 1,
650	874	// TODO: we would like to set this to executable_bit_only, but two
...	...	@@ -653,12 +877,19 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
653	877	// 2. the hashing algorithm here needs to support detecting the is_executable
654	878	// bit on Windows from the ACLs (see the isExecutable function).
655	879	.mode_mode = .ignore,
656		.filter = .{ .exclude_empty_directories = true },
657		});
	880	.exclude_empty_directories = true,
	881	}) catch \|err\| return f.fail(f.location_tok, try eb.printString(
	882	"unable to unpack tarball to temporary directory: {s}",
	883	.{@errorName(err)},
	884	));
658	885
659	886	if (diagnostics.errors.items.len > 0) {
660	887	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
661		try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball");
	888	try eb.addRootErrorMessage(.{
	889	.msg = try eb.addString("unable to unpack tarball"),
	890	.src_loc = try f.srcLoc(f.location_tok),
	891	.notes_len = notes_len,
	892	});
662	893	const notes_start = try eb.reserveNotes(notes_len);
663	894	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
664	895	switch (item) {
...	...	@@ -678,19 +909,15 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void {
678	909	},
679	910	}
680	911	}
681		return error.InvalidTarball;
	912	return error.FetchFailed;
682	913	}
683	914	}
684	915
685		fn unpackGitPack(
686		f: *Fetch,
687		out_dir: fs.Dir,
688		resource: *Resource,
689		want_oid: git.Oid,
690		) !void {
	916	fn unpackGitPack(f: Fetch, out_dir: fs.Dir, resource: Resource) anyerror!void {
691	917	const eb = &f.error_bundle;
692		const gpa = f.gpa;
693		const reader = resource.reader();
	918	const gpa = f.arena.child_allocator;
	919	const want_oid = resource.git.want_oid;
	920	const reader = resource.git.fetch_stream.reader();
694	921	// The .git directory is used to store the packfile and associated index, but
695	922	// we do not attempt to replicate the exact structure of a real .git
696	923	// directory, since that isn't relevant for fetching a package.
...	...	@@ -700,13 +927,13 @@ fn unpackGitPack(
700	927	var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true });
701	928	defer pack_file.close();
702	929	var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
703		try fifo.pump(reader.reader(), pack_file.writer());
	930	try fifo.pump(reader, pack_file.writer());
704	931	try pack_file.sync();
705	932
706	933	var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true });
707	934	defer index_file.close();
708	935	{
709		var index_prog_node = reader.prog_node.start("Index pack", 0);
	936	var index_prog_node = f.prog_node.start("Index pack", 0);
710	937	defer index_prog_node.end();
711	938	index_prog_node.activate();
712	939	var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
...	...	@@ -716,7 +943,7 @@ fn unpackGitPack(
716	943	}
717	944
718	945	{
719		var checkout_prog_node = reader.prog_node.start("Checkout", 0);
	946	var checkout_prog_node = f.prog_node.start("Checkout", 0);
720	947	defer checkout_prog_node.end();
721	948	checkout_prog_node.activate();
722	949	var repository = try git.Repository.init(gpa, pack_file, index_file);
...	...	@@ -727,7 +954,11 @@ fn unpackGitPack(
727	954
728	955	if (diagnostics.errors.items.len > 0) {
729	956	const notes_len: u32 = @intCast(diagnostics.errors.items.len);
730		try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile");
	957	try eb.addRootErrorMessage(.{
	958	.msg = try eb.addString("unable to unpack packfile"),
	959	.src_loc = try f.srcLoc(f.location_tok),
	960	.notes_len = notes_len,
	961	});
731	962	const notes_start = try eb.reserveNotes(notes_len);
732	963	for (diagnostics.errors.items, notes_start..) \|item, note_i\| {
733	964	switch (item) {
...	...	@@ -748,9 +979,10 @@ fn unpackGitPack(
748	979	try out_dir.deleteTree(".git");
749	980	}
750	981
751		fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void {
	982	fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void {
	983	const gpa = f.arena.child_allocator;
752	984	// Recursive directory copy.
753		var it = try dir.walk(f.gpa);
	985	var it = try dir.walk(gpa);
754	986	defer it.deinit();
755	987	while (try it.next()) \|entry\| {
756	988	switch (entry.kind) {
...	...	@@ -816,16 +1048,22 @@ pub fn renameTmpIntoCache(
816	1048	/// the hash are not present on the file system. Empty directories are *not
817	1049	/// hashed* and must not be present on the file system when calling this
818	1050	/// function.
819		fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest {
	1051	fn computeHash(
	1052	f: *Fetch,
	1053	tmp_directory: Cache.Directory,
	1054	filter: Filter,
	1055	) RunError!Manifest.Digest {
820	1056	// All the path name strings need to be in memory for sorting.
821		const arena = f.arena_allocator.allocator();
822		const gpa = f.gpa;
	1057	const arena = f.arena.allocator();
	1058	const gpa = f.arena.child_allocator;
	1059	const eb = &f.error_bundle;
	1060	const thread_pool = f.job_queue.thread_pool;
823	1061
824	1062	// Collect all files, recursively, then sort.
825	1063	var all_files = std.ArrayList(*HashedFile).init(gpa);
826	1064	defer all_files.deinit();
827	1065
828		var walker = try pkg_dir.walk(gpa);
	1066	var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa);
829	1067	defer walker.deinit();
830	1068
831	1069	{
...	...	@@ -834,19 +1072,28 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
834	1072	var wait_group: WaitGroup = .{};
835	1073	// `computeHash` is called from a worker thread so there must not be
836	1074	// any waiting without working or a deadlock could occur.
837		defer wait_group.waitAndWork();
838
839		while (try walker.next()) \|entry\| {
	1075	defer thread_pool.waitAndWork(&wait_group);
	1076
	1077	while (walker.next() catch \|err\| {
	1078	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
	1079	"unable to walk temporary directory '{}': {s}",
	1080	.{ tmp_directory, @errorName(err) },
	1081	) });
	1082	return error.FetchFailed;
	1083	}) \|entry\| {
840	1084	_ = filter; // TODO: apply filter rules here
841	1085
842	1086	const kind: HashedFile.Kind = switch (entry.kind) {
843	1087	.directory => continue,
844	1088	.file => .file,
845	1089	.sym_link => .sym_link,
846		else => return error.IllegalFileTypeInPackage,
	1090	else => return f.fail(f.location_tok, try eb.printString(
	1091	"package contains '{s}' which has illegal file type '{s}'",
	1092	.{ entry.path, @tagName(entry.kind) },
	1093	)),
847	1094	};
848	1095
849		if (std.mem.eql(u8, entry.path, build_zig_basename))
	1096	if (std.mem.eql(u8, entry.path, Package.build_zig_basename))
850	1097	f.has_build_zig = true;
851	1098
852	1099	const hashed_file = try arena.create(HashedFile);
...	...	@@ -859,7 +1106,9 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
859	1106	.failure = undefined, // to be populated by the worker
860	1107	};
861	1108	wait_group.start();
862		try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group });
	1109	try thread_pool.spawn(workerHashFile, .{
	1110	tmp_directory.handle, hashed_file, &wait_group,
	1111	});
863	1112
864	1113	try all_files.append(hashed_file);
865	1114	}
...	...	@@ -869,19 +1118,13 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige
869	1118
870	1119	var hasher = Manifest.Hash.init(.{});
871	1120	var any_failures = false;
872		const eb = &f.error_bundle;
873	1121	for (all_files.items) \|hashed_file\| {
874	1122	hashed_file.failure catch \|err\| {
875	1123	any_failures = true;
876	1124	try eb.addRootErrorMessage(.{
877		.msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}),
878		.src_loc = try eb.addSourceLocation(.{
879		.src_path = try eb.addString(hashed_file.fs_path),
880		.span_start = 0,
881		.span_end = 0,
882		.span_main = 0,
	1125	.msg = try eb.printString("unable to hash '{s}': {s}", .{
	1126	hashed_file.fs_path, @errorName(err),
883	1127	}),
884		.notes_len = 0,
885	1128	});
886	1129	};
887	1130	hasher.update(&hashed_file.hash);
...	...	@@ -934,7 +1177,7 @@ fn isExecutable(file: fs.File) !bool {
934	1177	const HashedFile = struct {
935	1178	fs_path: []const u8,
936	1179	normalized_path: []const u8,
937		hash: Digest,
	1180	hash: Manifest.Digest,
938	1181	failure: Error!void,
939	1182	kind: Kind,
940	1183
...	...	@@ -970,7 +1213,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 {
970	1213	return normalized;
971	1214	}
972	1215
973		pub const Filter = struct {
	1216	const Filter = struct {
974	1217	include_paths: std.StringArrayHashMapUnmanaged(void) = .{},
975	1218
976	1219	/// sub_path is relative to the tarball root.
...	...	@@ -990,12 +1233,9 @@ pub const Filter = struct {
990	1233	}
991	1234	};
992	1235
993		const build_zig_basename = @import("../Package.zig").build_zig_basename;
994		const hex_multihash_len = 2 * Manifest.multihash_len;
995
996	1236	// These are random bytes.
997		const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
998		const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
	1237	const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 };
	1238	const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc };
999	1239
1000	1240	const builtin = @import("builtin");
1001	1241	const std = @import("std");
...	...	@@ -1010,3 +1250,4 @@ const Manifest = @import("../Manifest.zig");
1010	1250	const Fetch = @This();
1011	1251	const main = @import("../main.zig");
1012	1252	const git = @import("../git.zig");
	1253	const Package = @import("../Package.zig");

src/Package/Module.zig created+32

...	...	@@ -0,0 +1,32 @@
	1	//! Corresponds to something that Zig source code can `@import`.
	2	//! Not to be confused with src/Module.zig which should be renamed
	3	//! to something else. https://github.com/ziglang/zig/issues/14307
	4
	5	/// Only files inside this directory can be imported.
	6	root: Package.Path,
	7	/// Relative to `root`. May contain path separators.
	8	root_src_path: []const u8,
	9	/// The dependency table of this module. Shared dependencies such as 'std',
	10	/// 'builtin', and 'root' are not specified in every dependency table, but
	11	/// instead only in the table of `main_pkg`. `Module.importFile` is
	12	/// responsible for detecting these names and using the correct package.
	13	deps: Deps = .{},
	14
	15	pub const Deps = std.StringHashMapUnmanaged(*Module);
	16
	17	pub const Tree = struct {
	18	/// Each `Package` exposes a `Module` with build.zig as its root source file.
	19	build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module),
	20	};
	21
	22	pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module {
	23	const new = try allocator.create(Module);
	24	new.* = m;
	25	return new;
	26	}
	27
	28	const Module = @This();
	29	const Package = @import("../Package.zig");
	30	const std = @import("std");
	31	const Allocator = std.mem.Allocator;
	32	const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest;

src/crash_report.zig+13-9

...	...	@@ -139,18 +139,22 @@ fn dumpStatusReport() !void {
139	139
140	140	var crash_heap: [16 * 4096]u8 = undefined;
141	141
142		fn writeFilePath(file: *Module.File, stream: anytype) !void {
143		if (file.pkg.root_src_directory.path) \|path\| {
144		try stream.writeAll(path);
145		try stream.writeAll(std.fs.path.sep_str);
	142	fn writeFilePath(file: *Module.File, writer: anytype) !void {
	143	if (file.mod.root.root_dir.path) \|path\| {
	144	try writer.writeAll(path);
	145	try writer.writeAll(std.fs.path.sep_str);
146	146	}
147		try stream.writeAll(file.sub_file_path);
	147	if (file.mod.root.sub_path.len > 0) {
	148	try writer.writeAll(file.mod.root.sub_path);
	149	try writer.writeAll(std.fs.path.sep_str);
	150	}
	151	try writer.writeAll(file.sub_file_path);
148	152	}
149	153
150		fn writeFullyQualifiedDeclWithFile(mod: Module, decl: Decl, stream: anytype) !void {
151		try writeFilePath(decl.getFileScope(mod), stream);
152		try stream.writeAll(": ");
153		try decl.renderFullyQualifiedDebugName(mod, stream);
	154	fn writeFullyQualifiedDeclWithFile(mod: Module, decl: Decl, writer: anytype) !void {
	155	try writeFilePath(decl.getFileScope(mod), writer);
	156	try writer.writeAll(": ");
	157	try decl.renderFullyQualifiedDebugName(mod, writer);
154	158	}
155	159
156	160	pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn {

src/main.zig+153-155

...	...	@@ -416,7 +416,7 @@ const usage_build_generic =
416	416	\\ dep: [[import=]name]
417	417	\\ --deps [dep],[dep],... Set dependency names for the root package
418	418	\\ dep: [[import=]name]
419		\\ --main-pkg-path Set the directory of the root package
	419	\\ --main-mod-path Set the directory of the root module
420	420	\\ -fPIC Force-enable Position Independent Code
421	421	\\ -fno-PIC Force-disable Position Independent Code
422	422	\\ -fPIE Force-enable Position Independent Executable
...	...	@@ -765,17 +765,11 @@ const Framework = struct {
765	765	};
766	766
767	767	const CliModule = struct {
768		mod: *Package,
	768	mod: *Package.Module,
769	769	/// still in CLI arg format
770	770	deps_str: []const u8,
771	771	};
772	772
773		fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void {
774		var it = modules.iterator();
775		while (it.next()) \|kv\| kv.value_ptr.mod.destroy(modules.allocator);
776		modules.deinit();
777		}
778
779	773	fn buildOutputType(
780	774	gpa: Allocator,
781	775	arena: Allocator,
...	...	@@ -950,8 +944,7 @@ fn buildOutputType(
950	944	// Contains every module specified via --mod. The dependencies are added
951	945	// after argument parsing is completed. We use a StringArrayHashMap to make
952	946	// error output consistent.
953		var modules = std.StringArrayHashMap(CliModule).init(gpa);
954		defer cleanupModules(&modules);
	947	var modules = std.StringArrayHashMap(CliModule).init(arena);
955	948
956	949	// The dependency string for the root package
957	950	var root_deps_str: ?[]const u8 = null;
...	...	@@ -1023,32 +1016,37 @@ fn buildOutputType(
1023	1016
1024	1017	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {
1025	1018	if (mem.eql(u8, mod_name, name)) {
1026		fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src });
	1019	fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{
	1020	mod_name, root_src,
	1021	});
1027	1022	}
1028	1023	}
1029	1024
1030	1025	var mod_it = modules.iterator();
1031	1026	while (mod_it.next()) \|kv\| {
1032	1027	if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) {
1033		fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path });
	1028	fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{
	1029	mod_name, root_src, kv.value_ptr.mod.root_src_path,
	1030	});
1034	1031	}
1035	1032	}
1036	1033
1037		try modules.ensureUnusedCapacity(1);
1038		modules.put(mod_name, .{
1039		.mod = try Package.create(
1040		gpa,
1041		fs.path.dirname(root_src),
1042		fs.path.basename(root_src),
1043		),
	1034	try modules.put(mod_name, .{
	1035	.mod = try Package.Module.create(arena, .{
	1036	.root = .{
	1037	.root_dir = Cache.Directory.cwd(),
	1038	.sub_path = fs.path.dirname(root_src) orelse "",
	1039	},
	1040	.root_src_path = fs.path.basename(root_src),
	1041	}),
1044	1042	.deps_str = deps_str,
1045		}) catch unreachable;
	1043	});
1046	1044	} else if (mem.eql(u8, arg, "--deps")) {
1047	1045	if (root_deps_str != null) {
1048	1046	fatal("only one --deps argument is allowed", .{});
1049	1047	}
1050	1048	root_deps_str = args_iter.nextOrFatal();
1051		} else if (mem.eql(u8, arg, "--main-pkg-path")) {
	1049	} else if (mem.eql(u8, arg, "--main-mod-path")) {
1052	1050	main_pkg_path = args_iter.nextOrFatal();
1053	1051	} else if (mem.eql(u8, arg, "-cflags")) {
1054	1052	extra_cflags.shrinkRetainingCapacity(0);
...	...	@@ -2461,19 +2459,26 @@ fn buildOutputType(
2461	2459	var deps_it = ModuleDepIterator.init(deps_str);
2462	2460	while (deps_it.next()) \|dep\| {
2463	2461	if (dep.expose.len == 0) {
2464		fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name });
	2462	fatal("module '{s}' depends on '{s}' with a blank name", .{
	2463	kv.key_ptr.*, dep.name,
	2464	});
2465	2465	}
2466	2466
2467	2467	for ([_][]const u8{ "std", "root", "builtin" }) \|name\| {
2468	2468	if (mem.eql(u8, dep.expose, name)) {
2469		fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose });
	2469	fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{
	2470	dep.name, dep.expose,
	2471	});
2470	2472	}
2471	2473	}
2472	2474
2473		const dep_mod = modules.get(dep.name) orelse
2474		fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name });
	2475	const dep_mod = modules.get(dep.name) orelse {
	2476	fatal("module '{s}' depends on module '{s}' which does not exist", .{
	2477	kv.key_ptr.*, dep.name,
	2478	});
	2479	};
2475	2480
2476		try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod);
	2481	try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod);
2477	2482	}
2478	2483	}
2479	2484	}
...	...	@@ -3229,31 +3234,33 @@ fn buildOutputType(
3229	3234	};
3230	3235	defer emit_implib_resolved.deinit();
3231	3236
3232		const main_pkg: ?*Package = if (root_src_file) \|unresolved_src_path\| blk: {
	3237	const main_mod: ?*Package.Module = if (root_src_file) \|unresolved_src_path\| blk: {
3233	3238	const src_path = try introspect.resolvePath(arena, unresolved_src_path);
3234	3239	if (main_pkg_path) \|unresolved_main_pkg_path\| {
3235	3240	const p = try introspect.resolvePath(arena, unresolved_main_pkg_path);
3236		if (p.len == 0) {
3237		break :blk try Package.create(gpa, null, src_path);
3238		} else {
3239		const rel_src_path = try fs.path.relative(arena, p, src_path);
3240		break :blk try Package.create(gpa, p, rel_src_path);
3241		}
	3241	break :blk try Package.Module.create(arena, .{
	3242	.root = .{
	3243	.root_dir = Cache.Directory.cwd(),
	3244	.sub_path = p,
	3245	},
	3246	.root_src_path = if (p.len == 0)
	3247	src_path
	3248	else
	3249	try fs.path.relative(arena, p, src_path),
	3250	});
3242	3251	} else {
3243		const root_src_dir_path = fs.path.dirname(src_path);
3244		break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch \|err\| {
3245		if (root_src_dir_path) \|p\| {
3246		fatal("unable to open '{s}': {s}", .{ p, @errorName(err) });
3247		} else {
3248		return err;
3249		}
3250		};
	3252	break :blk try Package.Module.create(arena, .{
	3253	.root = .{
	3254	.root_dir = Cache.Directory.cwd(),
	3255	.sub_path = fs.path.dirname(src_path) orelse "",
	3256	},
	3257	.root_src_path = fs.path.basename(src_path),
	3258	});
3251	3259	}
3252	3260	} else null;
3253		defer if (main_pkg) \|p\| p.destroy(gpa);
3254	3261
3255	3262	// Transfer packages added with --deps to the root package
3256		if (main_pkg) \|mod\| {
	3263	if (main_mod) \|mod\| {
3257	3264	var it = ModuleDepIterator.init(root_deps_str orelse "");
3258	3265	while (it.next()) \|dep\| {
3259	3266	if (dep.expose.len == 0) {
...	...	@@ -3269,7 +3276,7 @@ fn buildOutputType(
3269	3276	const dep_mod = modules.get(dep.name) orelse
3270	3277	fatal("root module depends on module '{s}' which does not exist", .{dep.name});
3271	3278
3272		try mod.add(gpa, dep.expose, dep_mod.mod);
	3279	try mod.deps.put(arena, dep.expose, dep_mod.mod);
3273	3280	}
3274	3281	}
3275	3282
...	...	@@ -3310,17 +3317,18 @@ fn buildOutputType(
3310	3317	if (arg_mode == .run) {
3311	3318	break :l global_cache_directory;
3312	3319	}
3313		if (main_pkg) \|pkg\| {
	3320	if (main_mod != null) {
3314	3321	// search upwards from cwd until we find directory with build.zig
3315	3322	const cwd_path = try process.getCwdAlloc(arena);
3316		const build_zig = "build.zig";
3317	3323	const zig_cache = "zig-cache";
3318	3324	var dirname: []const u8 = cwd_path;
3319	3325	while (true) {
3320		const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig });
	3326	const joined_path = try fs.path.join(arena, &.{
	3327	dirname, Package.build_zig_basename,
	3328	});
3321	3329	if (fs.cwd().access(joined_path, .{})) \|_\| {
3322		const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache });
3323		const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{});
	3330	const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache });
	3331	const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{});
3324	3332	cleanup_local_cache_dir = dir;
3325	3333	break :l .{ .handle = dir, .path = cache_dir_path };
3326	3334	} else \|err\| switch (err) {
...	...	@@ -3378,6 +3386,8 @@ fn buildOutputType(
3378	3386
3379	3387	gimmeMoreOfThoseSweetSweetFileDescriptors();
3380	3388
	3389	if (true) @panic("TODO restore Compilation logic");
	3390
3381	3391	const comp = Compilation.create(gpa, .{
3382	3392	.zig_lib_directory = zig_lib_directory,
3383	3393	.local_cache_directory = local_cache_directory,
...	...	@@ -3389,7 +3399,7 @@ fn buildOutputType(
3389	3399	.dynamic_linker = target_info.dynamic_linker.get(),
3390	3400	.sysroot = sysroot,
3391	3401	.output_mode = output_mode,
3392		.main_pkg = main_pkg,
	3402	.main_mod = main_mod,
3393	3403	.emit_bin = emit_bin_loc,
3394	3404	.emit_h = emit_h_resolved.data,
3395	3405	.emit_asm = emit_asm_resolved.data,
...	...	@@ -4799,32 +4809,22 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4799	4809	try thread_pool.init(.{ .allocator = gpa });
4800	4810	defer thread_pool.deinit();
4801	4811
4802		var cleanup_build_runner_dir: ?fs.Dir = null;
4803		defer if (cleanup_build_runner_dir) \|*dir\| dir.close();
4804
4805		var main_pkg: Package = if (override_build_runner) \|build_runner_path\|
	4812	var main_mod: Package.Module = if (override_build_runner) \|build_runner_path\|
4806	4813	.{
4807		.root_src_directory = blk: {
4808		if (std.fs.path.dirname(build_runner_path)) \|dirname\| {
4809		const dir = fs.cwd().openDir(dirname, .{}) catch \|err\| {
4810		fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) });
4811		};
4812		cleanup_build_runner_dir = dir;
4813		break :blk .{ .path = dirname, .handle = dir };
4814		}
4815
4816		break :blk .{ .path = null, .handle = fs.cwd() };
	4814	.root = .{
	4815	.root_dir = Cache.Directory.cwd(),
	4816	.sub_path = fs.path.dirname(build_runner_path) orelse "",
4817	4817	},
4818		.root_src_path = std.fs.path.basename(build_runner_path),
	4818	.root_src_path = fs.path.basename(build_runner_path),
4819	4819	}
4820	4820	else
4821	4821	.{
4822		.root_src_directory = zig_lib_directory,
	4822	.root = .{ .root_dir = zig_lib_directory },
4823	4823	.root_src_path = "build_runner.zig",
4824	4824	};
4825	4825
4826		var build_pkg: Package = .{
4827		.root_src_directory = build_directory,
	4826	var build_mod: Package.Module = .{
	4827	.root = .{ .root_dir = build_directory },
4828	4828	.root_src_path = build_zig_basename,
4829	4829	};
4830	4830	if (build_options.only_core_functionality) {
...	...	@@ -4833,11 +4833,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4833	4833	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
4834	4834	\\
4835	4835	);
4836		try main_pkg.add(gpa, "@dependencies", deps_pkg);
	4836	try main_mod.deps.put(arena, "@dependencies", deps_pkg);
4837	4837	} else {
4838	4838	var http_client: std.http.Client = .{ .allocator = gpa };
4839	4839	defer http_client.deinit();
4840	4840
	4841	if (true) @panic("TODO restore package fetching logic");
	4842
4841	4843	// Here we provide an import to the build runner that allows using reflection to find
4842	4844	// all of the dependencies. Without this, there would be no way to use `@import` to
4843	4845	// access dependencies by name, since `@import` requires string literals.
...	...	@@ -4857,8 +4859,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4857	4859
4858	4860	// Here we borrow main package's table and will replace it with a fresh
4859	4861	// one after this process completes.
4860		const fetch_result = build_pkg.fetchAndAddDependencies(
4861		&main_pkg,
	4862	const fetch_result = build_mod.fetchAndAddDependencies(
	4863	&main_mod,
4862	4864	arena,
4863	4865	&thread_pool,
4864	4866	&http_client,
...	...	@@ -4886,10 +4888,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4886	4888	dependencies_source.items,
4887	4889	);
4888	4890
4889		mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table);
4890		try main_pkg.add(gpa, "@dependencies", deps_pkg);
	4891	mem.swap(Package.Table, &main_mod.table, &deps_pkg.table);
	4892	try main_mod.add(gpa, "@dependencies", deps_pkg);
4891	4893	}
4892		try main_pkg.add(gpa, "@build", &build_pkg);
	4894	try main_mod.add(gpa, "@build", &build_mod);
4893	4895
4894	4896	const comp = Compilation.create(gpa, .{
4895	4897	.zig_lib_directory = zig_lib_directory,
...	...	@@ -4901,7 +4903,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
4901	4903	.is_native_abi = cross_target.isNativeAbi(),
4902	4904	.dynamic_linker = target_info.dynamic_linker.get(),
4903	4905	.output_mode = .Exe,
4904		.main_pkg = &main_pkg,
	4906	.main_mod = &main_mod,
4905	4907	.emit_bin = emit_bin,
4906	4908	.emit_h = null,
4907	4909	.optimize_mode = .Debug,
...	...	@@ -5115,12 +5117,14 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void
5115	5117	.tree = tree,
5116	5118	.tree_loaded = true,
5117	5119	.zir = undefined,
5118		.pkg = undefined,
	5120	.mod = undefined,
5119	5121	.root_decl = .none,
5120	5122	};
5121	5123
5122		file.pkg = try Package.create(gpa, null, file.sub_file_path);
5123		defer file.pkg.destroy(gpa);
	5124	file.mod = try Package.Module.create(arena, .{
	5125	.root = Package.Path.cwd(),
	5126	.root_src_path = file.sub_file_path,
	5127	});
5124	5128
5125	5129	file.zir = try AstGen.generate(gpa, file.tree);
5126	5130	file.zir_loaded = true;
...	...	@@ -5321,12 +5325,14 @@ fn fmtPathFile(
5321	5325	.tree = tree,
5322	5326	.tree_loaded = true,
5323	5327	.zir = undefined,
5324		.pkg = undefined,
	5328	.mod = undefined,
5325	5329	.root_decl = .none,
5326	5330	};
5327	5331
5328		file.pkg = try Package.create(gpa, null, file.sub_file_path);
5329		defer file.pkg.destroy(gpa);
	5332	file.mod = try Package.Module.create(fmt.arena, .{
	5333	.root = Package.Path.cwd(),
	5334	.root_src_path = file.sub_file_path,
	5335	});
5330	5336
5331	5337	if (stat.size > max_src_size)
5332	5338	return error.FileTooBig;
...	...	@@ -5387,7 +5393,7 @@ pub fn putAstErrorsIntoBundle(
5387	5393	tree: Ast,
5388	5394	path: []const u8,
5389	5395	wip_errors: *std.zig.ErrorBundle.Wip,
5390		) !void {
	5396	) Allocator.Error!void {
5391	5397	var file: Module.File = .{
5392	5398	.status = .never_loaded,
5393	5399	.source_loaded = true,
...	...	@@ -5402,12 +5408,15 @@ pub fn putAstErrorsIntoBundle(
5402	5408	.tree = tree,
5403	5409	.tree_loaded = true,
5404	5410	.zir = undefined,
5405		.pkg = undefined,
	5411	.mod = undefined,
5406	5412	.root_decl = .none,
5407	5413	};
5408	5414
5409		file.pkg = try Package.create(gpa, null, path);
5410		defer file.pkg.destroy(gpa);
	5415	file.mod = try Package.Module.create(gpa, .{
	5416	.root = Package.Path.cwd(),
	5417	.root_src_path = file.sub_file_path,
	5418	});
	5419	defer gpa.destroy(file.mod);
5411	5420
5412	5421	file.zir = try AstGen.generate(gpa, file.tree);
5413	5422	file.zir_loaded = true;
...	...	@@ -5933,7 +5942,7 @@ pub fn cmdAstCheck(
5933	5942	.stat = undefined,
5934	5943	.tree = undefined,
5935	5944	.zir = undefined,
5936		.pkg = undefined,
	5945	.mod = undefined,
5937	5946	.root_decl = .none,
5938	5947	};
5939	5948	if (zig_source_file) \|file_name\| {
...	...	@@ -5971,8 +5980,10 @@ pub fn cmdAstCheck(
5971	5980	file.stat.size = source.len;
5972	5981	}
5973	5982
5974		file.pkg = try Package.create(gpa, null, file.sub_file_path);
5975		defer file.pkg.destroy(gpa);
	5983	file.mod = try Package.Module.create(arena, .{
	5984	.root = Package.Path.cwd(),
	5985	.root_src_path = file.sub_file_path,
	5986	});
5976	5987
5977	5988	file.tree = try Ast.parse(gpa, file.source, .zig);
5978	5989	file.tree_loaded = true;
...	...	@@ -6067,7 +6078,7 @@ pub fn cmdDumpZir(
6067	6078	.stat = undefined,
6068	6079	.tree = undefined,
6069	6080	.zir = try Module.loadZirCache(gpa, f),
6070		.pkg = undefined,
	6081	.mod = undefined,
6071	6082	.root_decl = .none,
6072	6083	};
6073	6084
...	...	@@ -6136,12 +6147,14 @@ pub fn cmdChangelist(
6136	6147	},
6137	6148	.tree = undefined,
6138	6149	.zir = undefined,
6139		.pkg = undefined,
	6150	.mod = undefined,
6140	6151	.root_decl = .none,
6141	6152	};
6142	6153
6143		file.pkg = try Package.create(gpa, null, file.sub_file_path);
6144		defer file.pkg.destroy(gpa);
	6154	file.mod = try Package.Module.create(arena, .{
	6155	.root = Package.Path.cwd(),
	6156	.root_src_path = file.sub_file_path,
	6157	});
6145	6158
6146	6159	const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0);
6147	6160	const amt = try f.readAll(source);
...	...	@@ -6623,8 +6636,11 @@ fn cmdFetch(
6623	6636	args: []const []const u8,
6624	6637	) !void {
6625	6638	const color: Color = .auto;
6626		var opt_url: ?[]const u8 = null;
	6639	const work_around_btrfs_bug = builtin.os.tag == .linux and
	6640	std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND");
	6641	var opt_path_or_url: ?[]const u8 = null;
6627	6642	var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR");
	6643	var recursive = false;
6628	6644
6629	6645	{
6630	6646	var i: usize = 0;
...	...	@@ -6640,18 +6656,21 @@ fn cmdFetch(
6640	6656	i += 1;
6641	6657	override_global_cache_dir = args[i];
6642	6658	continue;
	6659	} else if (mem.eql(u8, arg, "--recursive")) {
	6660	recursive = true;
	6661	continue;
6643	6662	} else {
6644	6663	fatal("unrecognized parameter: '{s}'", .{arg});
6645	6664	}
6646		} else if (opt_url != null) {
	6665	} else if (opt_path_or_url != null) {
6647	6666	fatal("unexpected extra parameter: '{s}'", .{arg});
6648	6667	} else {
6649		opt_url = arg;
	6668	opt_path_or_url = arg;
6650	6669	}
6651	6670	}
6652	6671	}
6653	6672
6654		const url = opt_url orelse fatal("missing url or path parameter", .{});
	6673	const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{});
6655	6674
6656	6675	var thread_pool: ThreadPool = undefined;
6657	6676	try thread_pool.init(.{ .allocator = gpa });
...	...	@@ -6664,19 +6683,6 @@ fn cmdFetch(
6664	6683	const root_prog_node = progress.start("Fetch", 0);
6665	6684	defer root_prog_node.end();
6666	6685
6667		var wip_errors: std.zig.ErrorBundle.Wip = undefined;
6668		try wip_errors.init(gpa);
6669		defer wip_errors.deinit();
6670
6671		var report: Package.Report = .{
6672		.ast = null,
6673		.directory = .{
6674		.handle = fs.cwd(),
6675		.path = null,
6676		},
6677		.error_bundle = &wip_errors,
6678		};
6679
6680	6686	var global_cache_directory: Compilation.Directory = l: {
6681	6687	const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena);
6682	6688	break :l .{
...	...	@@ -6686,56 +6692,48 @@ fn cmdFetch(
6686	6692	};
6687	6693	defer global_cache_directory.handle.close();
6688	6694
6689		var readable_resource: Package.ReadableResource = rr: {
6690		if (fs.cwd().openIterableDir(url, .{})) \|dir\| {
6691		break :rr .{
6692		.path = try gpa.dupe(u8, url),
6693		.resource = .{ .dir = dir },
6694		};
6695		} else \|dir_err\| {
6696		const file_err = if (dir_err == error.NotDir) e: {
6697		if (fs.cwd().openFile(url, .{})) \|f\| {
6698		break :rr .{
6699		.path = try gpa.dupe(u8, url),
6700		.resource = .{ .file = f },
6701		};
6702		} else \|err\| break :e err;
6703		} else dir_err;
6704
6705		const uri = std.Uri.parse(url) catch \|uri_err\| {
6706		fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{
6707		url, @errorName(file_err), @errorName(uri_err),
6708		});
6709		};
6710		const fetch_location = try Package.FetchLocation.initUri(uri, 0, report);
6711		const cwd: Cache.Directory = .{
6712		.handle = fs.cwd(),
6713		.path = null,
6714		};
6715		break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report);
6716		}
	6695	var job_queue: Package.Fetch.JobQueue = .{
	6696	.http_client = &http_client,
	6697	.thread_pool = &thread_pool,
	6698	.global_cache = global_cache_directory,
	6699	.recursive = recursive,
	6700	.work_around_btrfs_bug = work_around_btrfs_bug,
	6701	};
	6702	defer job_queue.deinit();
	6703
	6704	var fetch: Package.Fetch = .{
	6705	.arena = std.heap.ArenaAllocator.init(gpa),
	6706	.location = .{ .path_or_url = path_or_url },
	6707	.location_tok = 0,
	6708	.hash_tok = 0,
	6709	.parent_package_root = undefined,
	6710	.parent_manifest_ast = null,
	6711	.prog_node = root_prog_node,
	6712	.job_queue = &job_queue,
	6713	.omit_missing_hash_error = true,
	6714
	6715	.package_root = undefined,
	6716	.error_bundle = undefined,
	6717	.manifest = null,
	6718	.manifest_ast = undefined,
	6719	.actual_hash = undefined,
	6720	.has_build_zig = false,
	6721	.oom_flag = false,
6717	6722	};
6718		defer readable_resource.deinit(gpa);
	6723	defer fetch.deinit();
6719	6724
6720		var package_location = readable_resource.unpack(
6721		gpa,
6722		&thread_pool,
6723		global_cache_directory,
6724		0,
6725		report,
6726		root_prog_node,
6727		) catch \|err\| {
6728		if (wip_errors.root_list.items.len > 0) {
6729		var errors = try wip_errors.toOwnedBundle("");
6730		defer errors.deinit(gpa);
6731		errors.renderToStdErr(renderOptions(color));
6732		process.exit(1);
6733		}
6734		fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) });
	6725	fetch.run() catch \|err\| switch (err) {
	6726	error.OutOfMemory => fatal("out of memory", .{}),
	6727	error.FetchFailed => {}, // error bundle checked below
6735	6728	};
6736		defer package_location.deinit(gpa);
6737	6729
6738		const hex_digest = Package.Manifest.hexDigest(package_location.hash);
	6730	if (fetch.error_bundle.root_list.items.len > 0) {
	6731	var errors = try fetch.error_bundle.toOwnedBundle("");
	6732	errors.renderToStdErr(renderOptions(color));
	6733	process.exit(1);
	6734	}
	6735
	6736	const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
6739	6737
6740	6738	progress.done = true;
6741	6739	progress.refresh();