Meg's Mirrors

zig

1	//! Represents one independent job whose responsibility is to:
2	//!
3	//! 1. Check the local zig package directory to see if the hash already exists.
4	//! If so, load, parse, and validate the build.zig.zon file therein, and
5	//! goto step 9. Likewise if the location is a relative path, treat this
6	//! the same as a cache hit. Otherwise, proceed.
7	//! 2. Check the global package cache for a compressed tarball matching the
8	//! hash. If it is found, unpack the contents into a temporary directory inside
9	//! project local zig cache. Rename this directory into the local zig package
10	//! directory and goto step 9, skipping step 10.
11	//! 3. Fetch and unpack a URL into a temporary directory.
12	//! 4. Load, parse, and validate the build.zig.zon file therein. It is allowed
13	//! for the file to be missing, in which case this fetched package is considered
14	//! to be a "naked" package.
15	//! 5. Apply inclusion rules of the build.zig.zon to the temporary directory by
16	//! deleting excluded files. If any files had errors for files that were
17	//! ultimately excluded, those errors should be ignored, such as failure to
18	//! create symlinks that weren't supposed to be included anyway.
19	//! 6. Compute the package hash based on the remaining files in the temporary
20	//! directory.
21	//! 7. Rename the temporary directory into the local zig package directory. If
22	//! the hash already exists, delete the temporary directory and leave the zig
23	//! package directory untouched as it may be in use. This is done even if
24	//! the hash is invalid, in case the package with the different hash is used
25	//! in the future.
26	//! 8. Validate the computed hash against the expected hash. If invalid,
27	//! this job is done.
28	//! 9. Spawn a new fetch job for each dependency in the manifest file. Use
29	//! a mutex and a hash map so that redundant jobs do not get queued up.
30	//! 10.Compress the package directory and store it into the global package
31	//! cache.
32	//!
33	//! All of this must be done with only referring to the state inside this struct
34	//! because this work will be done in a dedicated thread.
35	const Fetch = @This();
36
37	const builtin = @import("builtin");
38	const native_os = builtin.os.tag;
39
40	const std = @import("std");
41	const Io = std.Io;
42	const fs = std.fs;
43	const log = std.log.scoped(.fetch);
44	const assert = std.debug.assert;
45	const ascii = std.ascii;
46	const Allocator = std.mem.Allocator;
47	const Path = std.Build.Cache.Path;
48	const Directory = std.Build.Cache.Directory;
49	const git = @import("Fetch/git.zig");
50	const Package = @import("Package.zig");
51	const Manifest = Package.Manifest;
52	const ErrorBundle = std.zig.ErrorBundle;
53
54	arena: std.heap.ArenaAllocator,
55	location: Location,
56	location_tok: std.zig.Ast.TokenIndex,
57	hash_tok: std.zig.Ast.OptionalTokenIndex,
58	name_tok: std.zig.Ast.TokenIndex,
59	lazy_status: LazyStatus,
60	/// Same as `parent_packge_root` except it is unchanged when recursing into
61	/// relative file paths (as opposed to URL).
62	remote_package_root: Path,
63	parent_package_root: Path,
64	parent_manifest_ast: ?*const std.zig.Ast,
65	prog_node: std.Progress.Node,
66	job_queue: *JobQueue,
67	/// If true, don't add an error for a missing hash. This flag is not passed
68	/// down to recursive dependencies. It's intended to be used only be the CLI.
69	omit_missing_hash_error: bool,
70	/// If true, don't fail when a manifest file is missing the `paths` field,
71	/// which specifies inclusion rules. This is intended to be true for the first
72	/// fetch task and false for the recursive dependencies.
73	allow_missing_paths_field: bool,
74	/// If true and URL points to a Git repository, will use the latest commit.
75	use_latest_commit: bool,
76
77	// Above this are fields provided as inputs to `run`.
78	// Below this are fields populated by `run`.
79
80	/// Relative to the build root of the root package.
81	package_root: Path,
82	error_bundle: ErrorBundle.Wip,
83	manifest: Manifest,
84	manifest_ast: std.zig.Ast,
85	have_manifest: bool,
86	computed_hash: ComputedHash,
87	/// Fetch logic notices whether a package has a build.zig file and sets this flag.
88	has_build_zig: bool,
89	/// Indicates whether the task aborted due to an out-of-memory condition.
90	oom_flag: bool,
91	/// If `use_latest_commit` was true, this will be set to the commit that was used.
92	/// If the resource pointed to by the location is not a Git-repository, this
93	/// will be left unchanged.
94	latest_commit: ?git.Oid,
95
96	// This field is used by the CLI only, untouched by this file.
97
98	/// The module for this `Fetch` tasks's package, which exposes `build.zig` as
99	/// the root source file.
100	///
101	/// This could be an opaque "userdata" field because this code does not observe
102	/// this data in any way but let's have some type safety because we can.
103	cli_module: ?*@import("../Maker.zig").CliModule,
104
105	pub const LazyStatus = enum {
106	/// Not lazy.
107	eager,
108	/// Lazy, found.
109	available,
110	/// Lazy, not found.
111	unavailable,
112	};
113
114	pub const LocalStorage = struct {
115	cache_root: Path,
116	/// Path to "zig-pkg" inside the package in which the user ran `zig build`.
117	pkg_root: Path,
118	};
119
120	/// Contains shared state among all `Fetch` tasks.
121	pub const JobQueue = struct {
122	io: Io,
123	mutex: Io.Mutex = .init,
124	/// It's an array hash map so that it can be sorted before rendering the
125	/// dependencies.zig source file.
126	/// Protected by `mutex`.
127	table: Table = .{},
128	/// `table` may be missing some tasks such as ones that failed, so this
129	/// field contains references to all of them.
130	/// Protected by `mutex`.
131	all_fetches: std.ArrayList(*Fetch) = .empty,
132	prog_node: std.Progress.Node,
133
134	http_client: *std.http.Client,
135	/// This tracks `Fetch` tasks as well as recompression tasks.
136	group: Io.Group = .init,
137	global_cache: Directory,
138	/// If `null`, indicates fetch globally only.
139	local_storage: ?*const LocalStorage,
140	/// If true then, no fetching occurs, and:
141	/// * The `global_cache` directory is assumed to be the direct parent
142	/// directory of on-disk packages rather than having the "p/" directory
143	/// prefix inside of it.
144	/// * An error occurs if any non-lazy packages are not already present in
145	/// the package cache directory.
146	/// * Missing hash field causes an error, and no fetching occurs so it does
147	/// not print the correct hash like usual.
148	read_only: bool,
149	recursive: bool,
150	/// Dumps hash information to stdout which can be used to troubleshoot why
151	/// two hashes of the same package do not match.
152	/// If this is true, `recursive` must be false.
153	debug_hash: bool,
154	mode: Mode,
155	/// Set of hashes that will be additionally fetched even if they are marked
156	/// as lazy.
157	unlazy_set: UnlazySet = .{},
158	/// Identifies paths that override all packages in the tree with matching
159	/// project ids.
160	fork_set: ForkSet = .{},
161
162	pub const Mode = enum {
163	/// Non-lazy dependencies are always fetched.
164	/// Lazy dependencies are fetched only when needed.
165	needed,
166	/// Both non-lazy and lazy dependencies are always fetched.
167	all,
168	};
169	pub const Table = std.array_hash_map.Auto(Package.Hash, *Fetch);
170	pub const UnlazySet = std.array_hash_map.Auto(Package.Hash, void);
171	pub const ForkSet = std.array_hash_map.Custom(Fork, void, Fork.Context, false);
172
173	pub const Fork = struct {
174	path: Path,
175	manifest_ast: std.zig.Ast,
176	manifest: Package.Manifest,
177	uses: usize,
178
179	pub const Context = struct {
180	pub fn hash(_: @This(), a: Fork) u32 {
181	const project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
182	return @truncate(project_id.hash());
183	}
184
185	pub fn eql(_: @This(), a: Fork, b: Fork, _: usize) bool {
186	const a_project_id: Package.ProjectId = .init(a.manifest.name, a.manifest.id);
187	const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
188	return a_project_id.eql(&b_project_id);
189	}
190	};
191
192	pub const Adapter = struct {
193	pub fn hash(_: @This(), a: Package.ProjectId) u32 {
194	return @truncate(a.hash());
195	}
196
197	pub fn eql(_: @This(), a_project_id: Package.ProjectId, b: Fork, _: usize) bool {
198	const b_project_id: Package.ProjectId = .init(b.manifest.name, b.manifest.id);
199	return a_project_id.eql(&b_project_id);
200	}
201	};
202	};
203
204	pub fn deinit(jq: *JobQueue) void {
205	const io = jq.io;
206	jq.group.cancel(io);
207	if (jq.all_fetches.items.len == 0) return;
208	const gpa = jq.all_fetches.items[0].arena.child_allocator;
209	jq.table.deinit(gpa);
210	// These must be deinitialized in reverse order because subsequent
211	// `Fetch` instances are allocated in prior ones' arenas.
212	// Sorry, I know it's a bit weird, but it slightly simplifies the
213	// critical section.
214	while (jq.all_fetches.pop()) |f| f.deinit();
215	jq.all_fetches.deinit(gpa);
216	jq.* = undefined;
217	}
218
219	/// Dumps all subsequent error bundles into the first one.
220	pub fn consolidateErrors(jq: *JobQueue) !void {
221	const root = &jq.all_fetches.items[0].error_bundle;
222	const gpa = root.gpa;
223	for (jq.all_fetches.items[1..]) |fetch| {
224	if (fetch.error_bundle.root_list.items.len > 0) {
225	var bundle = try fetch.error_bundle.toOwnedBundle("");
226	defer bundle.deinit(gpa);
227	try root.addBundleAsRoots(bundle);
228	}
229	}
230	}
231
232	/// Creates the dependencies.zig source code for the build runner to obtain
233	/// via `@import("@dependencies")`.
234	pub fn createDependenciesSource(jq: *JobQueue, w: *Io.Writer) Io.Writer.Error!void {
235	const keys = jq.table.keys();
236
237	assert(keys.len != 0); // caller should have added the first one
238	if (keys.len == 1) {
239	// This is the first one. It must have no dependencies.
240	return createEmptyDependenciesSource(w);
241	}
242
243	try w.writeAll("pub const packages = struct {\n");
244
245	// Ensure the generated .zig file is deterministic.
246	jq.table.sortUnstable(@as(struct {
247	keys: []const Package.Hash,
248	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
249	return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
250	}
251	}, .{ .keys = keys }));
252
253	for (keys, jq.table.values()) |*hash, fetch| {
254	if (fetch == jq.all_fetches.items[0]) {
255	// The first one is a dummy package for the current project.
256	continue;
257	}
258
259	const hash_slice = hash.toSlice();
260
261	try w.print(
262	\\ pub const {f} = struct {{
263	\\
264	, .{std.zig.fmtId(hash_slice)});
265
266	lazy: {
267	switch (fetch.lazy_status) {
268	.eager => break :lazy,
269	.available => {
270	try w.writeAll(
271	\\ pub const available = true;
272	\\
273	);
274	break :lazy;
275	},
276	.unavailable => {
277	try w.writeAll(
278	\\ pub const available = false;
279	\\ };
280	\\
281	);
282	continue;
283	},
284	}
285	}
286
287	try w.print(
288	\\ pub const build_root = "{f}";
289	\\
290	, .{std.fmt.alt(fetch.package_root, .formatEscapeString)});
291
292	if (fetch.has_build_zig) {
293	try w.print(
294	\\ pub const build_zig = @import("{f}");
295	\\
296	, .{std.zig.fmtString(hash_slice)});
297	}
298
299	if (fetch.have_manifest) {
300	const manifest = &fetch.manifest;
301	try w.writeAll(
302	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{
303	\\
304	);
305	for (manifest.dependencies.keys(), manifest.dependencies.values()) |name, dep| {
306	const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
307	try w.print(
308	" .{{ \"{f}\", \"{f}\" }},\n",
309	.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
310	);
311	}
312
313	try w.writeAll(
314	\\ };
315	\\ };
316	\\
317	);
318	} else {
319	try w.writeAll(
320	\\ pub const deps: []const struct { []const u8, []const u8 } = &.{};
321	\\ };
322	\\
323	);
324	}
325	}
326
327	try w.writeAll(
328	\\};
329	\\
330	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{
331	\\
332	);
333
334	const root_fetch = jq.all_fetches.items[0];
335	assert(root_fetch.have_manifest);
336	const root_manifest = &root_fetch.manifest;
337
338	for (root_manifest.dependencies.keys(), root_manifest.dependencies.values()) |name, dep| {
339	const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
340	try w.print(
341	" .{{ \"{f}\", \"{f}\" }},\n",
342	.{ std.zig.fmtString(name), std.zig.fmtString(h.toSlice()) },
343	);
344	}
345	try w.writeAll("};\n");
346	}
347
348	pub fn createEmptyDependenciesSource(w: *Io.Writer) Io.Writer.Error!void {
349	try w.writeAll(
350	\\pub const packages = struct {};
351	\\pub const root_deps: []const struct { []const u8, []const u8 } = &.{};
352	\\
353	);
354	}
355
356	fn recompress(jq: *JobQueue, package_hash: Package.Hash, package_root: Path) Io.Cancelable!void {
357	const pkg_hash_slice = package_hash.toSlice();
358
359	const prog_node = jq.prog_node.startFmt(0, "recompress {s}", .{pkg_hash_slice});
360	defer prog_node.end();
361
362	var dest_sub_path_buf: ["p/".len + Package.Hash.max_len + ".tar.gz".len]u8 = undefined;
363	const dest_path: Path = .{
364	.root_dir = jq.global_cache,
365	.sub_path = std.mem.print(&dest_sub_path_buf, "p/{s}.tar.gz", .{pkg_hash_slice}) catch unreachable,
366	};
367
368	const gpa = jq.http_client.allocator;
369
370	var arena_instance = std.heap.ArenaAllocator.init(gpa);
371	defer arena_instance.deinit();
372	const arena = arena_instance.allocator();
373
374	recompressFallible(jq, arena, dest_path, pkg_hash_slice, package_root, prog_node) catch |err| switch (err) {
375	error.Canceled => |e| return e,
376	error.ReadFailed => comptime unreachable,
377	error.WriteFailed => comptime unreachable,
378	else => |e| log.warn("failed caching recompressed tarball to {f}: {t}", .{ dest_path, e }),
379	};
380	}
381
382	fn recompressFallible(
383	jq: *JobQueue,
384	arena: Allocator,
385	dest_path: Path,
386	pkg_hash_slice: []const u8,
387	package_root: Path,
388	prog_node: std.Progress.Node,
389	) !void {
390	const gpa = jq.http_client.allocator;
391	const io = jq.io;
392
393	// We have to walk the file system up front in order to sort the file
394	// list for determinism purposes. The hash of the recompressed file is
395	// not critical because the true hash is based on the content alone.
396	// However, if we want Zig users to be able to share cached package
397	// data with each other via peer-to-peer protocols, we benefit greatly
398	// from the data being identical on everyone's computers.
399	var scanned_files: std.ArrayList(ScannedFile) = .empty;
400	defer scanned_files.deinit(gpa);
401
402	var pkg_dir = try package_root.root_dir.handle.openDir(io, package_root.sub_path, .{ .iterate = true });
403	defer pkg_dir.close(io);
404
405	{
406	var walker = try pkg_dir.walk(gpa);
407	defer walker.deinit();
408
409	while (try walker.next(io)) |entry| {
410	const symlink = switch (entry.kind) {
411	.directory => continue,
412	.file => false,
413	.sym_link => true,
414	else => return error.IllegalFileType,
415	};
416	const entry_path = try arena.dupe(u8, entry.path);
417	// If necessary, normalize path separators to POSIX-style since the tar format requires that.
418	if (comptime (std.fs.path.sep != std.fs.path.sep_posix)) {
419	std.mem.replaceScalar(u8, entry_path, std.fs.path.sep, std.fs.path.sep_posix);
420	}
421	try scanned_files.append(gpa, .{
422	.ptr = entry_path.ptr,
423	.len = @intCast(entry_path.len),
424	.symlink = symlink,
425	});
426	}
427
428	std.mem.sortUnstable(ScannedFile, scanned_files.items, {}, stringCmp);
429	}
430
431	prog_node.setEstimatedTotalItems(scanned_files.items.len);
432
433	var atomic_file = try dest_path.root_dir.handle.createFileAtomic(io, dest_path.sub_path, .{
434	.make_path = true,
435	.replace = true,
436	});
437	defer atomic_file.deinit(io);
438
439	var file_write_buffer: [4096]u8 = undefined;
440	var file_writer = atomic_file.file.writer(io, &file_write_buffer);
441
442	var compress_buffer: [std.compress.flate.max_window_len]u8 = undefined;
443	var compress = std.compress.flate.Compress.init(&file_writer.interface, &compress_buffer, .gzip, .level_9) catch |err| switch (err) {
444	error.WriteFailed => return file_writer.err.?,
445	};
446
447	var archiver: std.tar.Writer = .{ .underlying_writer = &compress.writer };
448	archiver.prefix = pkg_hash_slice;
449
450	var file_read_buffer: [4096]u8 = undefined;
451	var link_buf: [fs.max_path_bytes]u8 = undefined;
452
453	for (scanned_files.items) |scanned_file| {
454	const entry_path = scanned_file.ptr[0..scanned_file.len];
455	if (scanned_file.symlink) {
456	const link_name = link_buf[0..try pkg_dir.readLink(io, entry_path, &link_buf)];
457	archiver.writeLink(entry_path, link_name, .{}) catch |err| switch (err) {
458	error.WriteFailed => return file_writer.err.?,
459	else => |e| return e,
460	};
461	} else {
462	var file = try pkg_dir.openFile(io, entry_path, .{});
463	defer file.close(io);
464	var file_reader: Io.File.Reader = .init(file, io, &file_read_buffer);
465	archiver.writeFile(entry_path, &file_reader, 0) catch |err| switch (err) {
466	error.ReadFailed => return file_reader.err.?,
467	error.WriteFailed => return file_writer.err.?,
468	else => |e| return e,
469	};
470	}
471	prog_node.completeOne();
472	}
473
474	// intentionally omitting the pointless trailer
475	//try archiver.finish();
476	compress.finish() catch |err| switch (err) {
477	error.WriteFailed => return file_writer.err.?,
478	};
479	try file_writer.flush();
480	try atomic_file.replace(io);
481	}
482	};
483
484	const ScannedFile = struct {
485	ptr: [*]const u8,
486	len: u32,
487	symlink: bool,
488	};
489
490	fn stringCmp(_: void, lhs: ScannedFile, rhs: ScannedFile) bool {
491	return std.mem.lessThan(u8, lhs.ptr[0..lhs.len], rhs.ptr[0..rhs.len]);
492	}
493
494	pub const Location = union(enum) {
495	remote: Remote,
496	/// A directory found inside the parent package.
497	relative_path: Path,
498	/// Recursive Fetch tasks will never use this Location, but it may be
499	/// passed in by the CLI. Indicates the file contents here should be copied
500	/// into the global package cache. It may be a file relative to the cwd or
501	/// absolute, in which case it should be treated exactly like a `file://`
502	/// URL, or a directory, in which case it should be treated as an
503	/// already-unpacked directory (but still needs to be copied into the
504	/// global package cache and have inclusion rules applied).
505	path_or_url: []const u8,
506
507	pub const Remote = struct {
508	url: []const u8,
509	/// If this is null it means the user omitted the hash field from a dependency.
510	/// It will be an error but the logic should still fetch and print the discovered hash.
511	hash: ?Package.Hash,
512	};
513	};
514
515	pub const RunError = error{
516	OutOfMemory,
517	Canceled,
518	/// This error code is intended to be handled by inspecting the
519	/// `error_bundle` field.
520	FetchFailed,
521	};
522
523	pub fn run(f: *Fetch) RunError!void {
524	const job_queue = f.job_queue;
525	const io = job_queue.io;
526	const eb = &f.error_bundle;
527	const arena = f.arena.allocator();
528	const gpa = f.arena.child_allocator;
529
530	try eb.init(gpa);
531
532	// Check the global zig package cache to see if the hash already exists. If
533	// so, load, parse, and validate the build.zig.zon file therein, and skip
534	// ahead to queuing up jobs for dependencies. Likewise if the location is a
535	// relative path, treat this the same as a cache hit. Otherwise, proceed.
536
537	const remote = switch (f.location) {
538	.relative_path => |pkg_root| {
539	if (fs.path.isAbsolute(pkg_root.sub_path)) return f.fail(
540	f.location_tok,
541	try eb.addString("expected path relative to build root; found absolute path"),
542	);
543	if (f.hash_tok.unwrap()) |hash_tok| return f.fail(
544	hash_tok,
545	try eb.addString("path-based dependencies are not hashed"),
546	);
547	// Packages fetched by URL may not use relative paths to escape outside the
548	// fetched package directory from within the package cache.
549
550	// This code path is only reachable recursively and the sub_path
551	// will already have been resolved to no longer have extra ".." or
552	// "." components.
553	assert(job_queue.local_storage != null);
554	assert(pkg_root.root_dir.eql(f.remote_package_root.root_dir));
555	if (!std.mem.startsWith(u8, pkg_root.sub_path, f.remote_package_root.sub_path)) return f.fail(
556	f.location_tok,
557	try eb.printString("dependency path outside project: '{f}'", .{pkg_root}),
558	);
559	f.package_root = pkg_root;
560	try loadManifest(f, pkg_root);
561	if (!f.has_build_zig) try checkBuildFileExistence(f);
562	if (!job_queue.recursive) return;
563	return queueJobsForDeps(f);
564	},
565	.remote => |remote| remote,
566	.path_or_url => |path_or_url| {
567	if (Io.Dir.cwd().openDir(io, path_or_url, .{ .iterate = true })) |dir| {
568	var resource: Resource = .{ .dir = dir };
569	return f.runResource(path_or_url, &resource, null, false);
570	} else |dir_err| {
571	var server_header_buffer: [init_resource_buffer_size]u8 = undefined;
572
573	const file_err = if (dir_err == error.NotDir) e: {
574	if (Io.Dir.cwd().openFile(io, path_or_url, .{})) |file| {
575	var resource: Resource = .{ .file = file.reader(io, &server_header_buffer) };
576	return f.runResource(path_or_url, &resource, null, false);
577	} else |err| break :e err;
578	} else dir_err;
579
580	const uri = std.Uri.parse(path_or_url) catch |uri_err| {
581	return f.fail(0, try eb.printString(
582	"'{s}' could not be recognized as a file path ({t}) or an URL ({t})",
583	.{ path_or_url, file_err, uri_err },
584	));
585	};
586	var resource: Resource = undefined;
587	try f.initResource(uri, &resource, &server_header_buffer);
588	return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, null, false);
589	}
590	},
591	};
592
593	var resource_buffer: [init_resource_buffer_size]u8 = undefined;
594
595	if (remote.hash) |expected_hash| {
596	const expected_project_id: Package.ProjectId = expected_hash.projectId();
597	if (job_queue.fork_set.getKeyPtrAdapted(expected_project_id, @as(JobQueue.Fork.Adapter, .{}))) |fork| {
598	log.debug("using fork {f} for {s}", .{ fork.path, fork.manifest.name });
599	fork.uses += 1;
600	f.package_root = fork.path;
601	f.remote_package_root = f.package_root;
602	f.manifest_ast = fork.manifest_ast;
603	f.manifest = fork.manifest;
604	f.have_manifest = true;
605	try checkBuildFileExistence(f);
606	if (!job_queue.recursive) return;
607	return queueJobsForDeps(f);
608	}
609
610	if (job_queue.local_storage) |ls| {
611	const package_root = try ls.pkg_root.join(arena, expected_hash.toSlice());
612	if (package_root.root_dir.handle.access(io, package_root.sub_path, .{})) |_| {
613	assert(f.lazy_status != .unavailable);
614	f.package_root = package_root;
615	f.remote_package_root = f.package_root;
616	try loadManifest(f, f.package_root);
617	try checkBuildFileExistence(f);
618	if (!job_queue.recursive) return;
619	return queueJobsForDeps(f);
620	} else |err| switch (err) {
621	error.FileNotFound => {
622	log.debug("FileNotFound: {f}", .{package_root});
623	if (job_queue.read_only and f.lazy_status == .eager) return f.fail(
624	f.name_tok,
625	try eb.printString("package not found at '{f}'", .{package_root}),
626	);
627	},
628	error.Canceled => |e| return e,
629	else => |e| {
630	try eb.addRootErrorMessage(.{
631	.msg = try eb.printString("unable to open package cache directory {f}: {t}", .{
632	package_root, e,
633	}),
634	});
635	return error.FetchFailed;
636	},
637	}
638	}
639
640	// Check global cache before remote fetch.
641	const cached_tarball_sub_path = try std.fmt.allocPrint(arena, "p/{s}.tar.gz", .{expected_hash.toSlice()});
642	const cached_tarball_path: Path = .{
643	.root_dir = job_queue.global_cache,
644	.sub_path = cached_tarball_sub_path,
645	};
646	if (cached_tarball_path.root_dir.handle.openFile(io, cached_tarball_path.sub_path, .{})) |file| {
647	log.debug("found global cached tarball {f}", .{cached_tarball_path});
648	var resource: Resource = .{ .file = file.reader(io, &resource_buffer) };
649	return f.runResource(cached_tarball_sub_path, &resource, remote.hash, true);
650	} else |err| switch (err) {
651	error.FileNotFound => log.debug("FileNotFound: {f}", .{cached_tarball_path}),
652	error.Canceled => |e| return e,
653	else => |e| {
654	try eb.addRootErrorMessage(.{
655	.msg = try eb.printString("unable to open globally cached package {f}: {t}", .{
656	cached_tarball_path, e,
657	}),
658	});
659	return error.FetchFailed;
660	},
661	}
662
663	switch (f.lazy_status) {
664	.eager => {},
665	.available => if (!job_queue.unlazy_set.contains(expected_hash)) {
666	f.lazy_status = .unavailable;
667	return;
668	},
669	.unavailable => unreachable,
670	}
671	} else if (job_queue.read_only) {
672	try eb.addRootErrorMessage(.{
673	.msg = try eb.addString("dependency is missing hash field"),
674	.src_loc = try f.srcLoc(f.location_tok),
675	});
676	return error.FetchFailed;
677	}
678
679	// Fetch and unpack the remote into a temporary directory.
680	const uri = std.Uri.parse(remote.url) catch |err| return f.fail(
681	f.location_tok,
682	try eb.printString("invalid URI: {t}", .{err}),
683	);
684	var resource: Resource = undefined;
685	try f.initResource(uri, &resource, &resource_buffer);
686	return f.runResource(try uri.path.toRawMaybeAlloc(arena), &resource, remote.hash, false);
687	}
688
689	pub fn deinit(f: *Fetch) void {
690	f.error_bundle.deinit();
691	f.arena.deinit();
692	}
693
694	/// Consumes `resource`, even if an error is returned.
695	fn runResource(
696	f: *Fetch,
697	uri_path: []const u8,
698	resource: *Resource,
699	remote_hash: ?Package.Hash,
700	disable_recompress: bool,
701	) RunError!void {
702	const job_queue = f.job_queue;
703	assert(!job_queue.read_only);
704
705	const io = job_queue.io;
706	defer resource.deinit(io);
707
708	const arena = f.arena.allocator();
709	const eb = &f.error_bundle;
710	const rand_int = r: {
711	var x: u64 = undefined;
712	io.random(@ptrCast(&x));
713	break :r x;
714	};
715	const tmp_dir_sub_path = ".tmp-" ++ std.fmt.hex(rand_int);
716	const tmp_tmp_dir_sub_path = "tmp/" ++ tmp_dir_sub_path;
717	const tmp_directory_path: Path = if (job_queue.local_storage) |ls|
718	try ls.pkg_root.join(arena, tmp_dir_sub_path)
719	else
720	.{
721	.root_dir = job_queue.global_cache,
722	.sub_path = tmp_tmp_dir_sub_path,
723	};
724
725	const package_sub_path = blk: {
726	var tmp_directory: Directory = .{
727	.path = tmp_directory_path.sub_path,
728	.handle = handle: {
729	const dir = tmp_directory_path.root_dir.handle.createDirPathOpen(io, tmp_directory_path.sub_path, .{
730	.open_options = .{ .iterate = true },
731	}) catch |err| {
732	try eb.addRootErrorMessage(.{
733	.msg = try eb.printString("unable to create temporary directory '{f}': {t}", .{
734	tmp_directory_path, err,
735	}),
736	});
737	return error.FetchFailed;
738	};
739	break :handle dir;
740	},
741	};
742	defer tmp_directory.handle.close(io);
743
744	// Fetch and unpack a resource into a temporary directory.
745	var unpack_result = try unpackResource(f, resource, uri_path, tmp_directory);
746
747	const pkg_path: Path = .{ .root_dir = tmp_directory, .sub_path = unpack_result.root_dir };
748
749	// Load, parse, and validate the unpacked build.zig.zon file. It is allowed
750	// for the file to be missing, in which case this fetched package is
751	// considered to be a "naked" package.
752	try loadManifest(f, pkg_path);
753
754	const filter: Filter = .{
755	.include_paths = if (f.have_manifest) f.manifest.paths else .{},
756	};
757
758	// Ignore errors that were excluded by manifest, such as failure to
759	// create symlinks that weren't supposed to be included anyway.
760	try unpack_result.validate(f, filter);
761
762	// Apply the manifest's inclusion rules to the temporary directory by
763	// deleting excluded files.
764	// Empty directories have already been omitted by `unpackResource`.
765	// Compute the package hash based on the remaining files in the temporary
766	// directory.
767	f.computed_hash = try computeHash(f, pkg_path, filter);
768
769	if (unpack_result.root_dir.len > 0)
770	break :blk try tmp_directory_path.join(arena, unpack_result.root_dir);
771
772	break :blk tmp_directory_path;
773	};
774
775	const computed_package_hash = computedPackageHash(f);
776
777	// Rename the temporary directory into the local zig package directory. If
778	// the hash already exists, delete the temporary directory and leave the
779	// zig package directory untouched as it may be in use. This is done even
780	// if the hash is invalid, in case the package with the different hash is
781	// used in the future.
782	if (job_queue.local_storage) |ls| {
783	f.package_root = try ls.pkg_root.join(arena, computed_package_hash.toSlice());
784	renameTmpIntoCache(io, package_sub_path, f.package_root) catch |err| {
785	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
786	"failed to rename temporary directory {f} into package cache directory {f}: {t}",
787	.{ package_sub_path, f.package_root, err },
788	) });
789	return error.FetchFailed;
790	};
791	} else {
792	f.package_root = tmp_directory_path;
793	}
794	f.remote_package_root = f.package_root;
795
796	if (!disable_recompress) {
797	// Spin off a task to recompress the tarball, with filtered files deleted, into
798	// the global cache.
799	job_queue.group.async(io, JobQueue.recompress, .{ job_queue, computed_package_hash, f.package_root });
800	}
801
802	// Remove temporary directory root if not already renamed to global cache.
803	if (!package_sub_path.eql(tmp_directory_path)) {
804	tmp_directory_path.root_dir.handle.deleteDir(io, tmp_directory_path.sub_path) catch |err| switch (err) {
805	error.Canceled => |e| return e,
806	else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_directory_path, e }),
807	};
808	}
809
810	// Validate the computed hash against the expected hash. If invalid, this
811	// job is done.
812
813	if (remote_hash) |declared_hash| {
814	const hash_tok = f.hash_tok.unwrap().?;
815	if (!computed_package_hash.eql(&declared_hash)) {
816	return f.fail(hash_tok, try eb.printString(
817	"hash mismatch: manifest declares {s} but the fetched package has {s}",
818	.{ declared_hash.toSlice(), computed_package_hash.toSlice() },
819	));
820	}
821	} else if (!f.omit_missing_hash_error) {
822	const notes_len = 1;
823	try eb.addRootErrorMessage(.{
824	.msg = try eb.addString("dependency is missing hash field"),
825	.src_loc = try f.srcLoc(f.location_tok),
826	.notes_len = notes_len,
827	});
828	const notes_start = try eb.reserveNotes(notes_len);
829	eb.extra.items[notes_start] = @backingInt(try eb.addErrorMessage(.{
830	.msg = try eb.printString("expected .hash = {q},", .{computed_package_hash.toSlice()}),
831	}));
832	return error.FetchFailed;
833	}
834
835	// Spawn a new fetch job for each dependency in the manifest file. Use
836	// a mutex and a hash map so that redundant jobs do not get queued up.
837	if (!job_queue.recursive) return;
838	return queueJobsForDeps(f);
839	}
840
841	pub fn computedPackageHash(f: *const Fetch) Package.Hash {
842	const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
843	if (f.have_manifest) {
844	const man = &f.manifest;
845	var version_buffer: [32]u8 = undefined;
846	const version: []const u8 = std.mem.print(&version_buffer, "{f}", .{man.version}) catch &version_buffer;
847	return .init(f.computed_hash.digest, man.name, version, man.id, saturated_size);
848	}
849	// In the future build.zig.zon fields will be added to allow overriding these values
850	// for naked tarballs.
851	return .init(f.computed_hash.digest, "N", "V", 0xffff, saturated_size);
852	}
853
854	/// `computeHash` gets a free check for the existence of `build.zig`, but when
855	/// not computing a hash, we need to do a syscall to check for it.
856	fn checkBuildFileExistence(f: *Fetch) RunError!void {
857	const io = f.job_queue.io;
858	const eb = &f.error_bundle;
859	if (f.package_root.access(io, std.zig.build_zig_basename, .{})) |_| {
860	f.has_build_zig = true;
861	} else |err| switch (err) {
862	error.FileNotFound => {},
863	else => |e| {
864	try eb.addRootErrorMessage(.{
865	.msg = try eb.printString("unable to access {f}/{s}: {t}", .{
866	f.package_root, std.zig.build_zig_basename, e,
867	}),
868	});
869	return error.FetchFailed;
870	},
871	}
872	}
873
874	/// This function populates `f.manifest` or leaves it `null`.
875	fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void {
876	const io = f.job_queue.io;
877	const eb = &f.error_bundle;
878	const arena = f.arena.allocator();
879	const manifest_path = try pkg_root.join(arena, Manifest.basename);
880
881	Manifest.load(
882	io,
883	arena,
884	manifest_path,
885	&f.manifest_ast,
886	eb,
887	&f.manifest,
888	f.allow_missing_paths_field,
889	) catch |err| switch (err) {
890	error.FileNotFound => return,
891	error.Canceled => |e| return e,
892	error.ErrorsBundled => return error.FetchFailed,
893	else => |e| {
894	try eb.addRootErrorMessage(.{
895	.msg = try eb.printString("unable to load package manifest '{f}': {t}", .{ manifest_path, e }),
896	});
897	return error.FetchFailed;
898	},
899	};
900	f.have_manifest = true;
901	}
902
903	fn queueJobsForDeps(f: *Fetch) RunError!void {
904	const io = f.job_queue.io;
905
906	assert(f.job_queue.recursive);
907
908	// If the package does not have a build.zig.zon file then there are no dependencies.
909	if (!f.have_manifest) return;
910	const manifest = &f.manifest;
911
912	const new_fetches, const prog_names = nf: {
913	const parent_arena = f.arena.allocator();
914	const gpa = f.arena.child_allocator;
915	const cache_root = f.job_queue.global_cache;
916	const dep_names = manifest.dependencies.keys();
917	const deps = manifest.dependencies.values();
918	// Grab the new tasks into a temporary buffer so we can unlock that mutex
919	// as fast as possible.
920	// This overallocates any fetches that get skipped by the `continue` in the
921	// loop below.
922	const new_fetches = try parent_arena.alloc(Fetch, deps.len);
923	const prog_names = try parent_arena.alloc([]const u8, deps.len);
924	var new_fetch_index: usize = 0;
925
926	try f.job_queue.mutex.lock(io);
927	defer f.job_queue.mutex.unlock(io);
928
929	try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len);
930	try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len));
931
932	// There are four cases here:
933	// * Correct hash is provided by manifest.
934	// - Hash map already has the entry, no need to add it again.
935	// * Incorrect hash is provided by manifest.
936	// - Hash mismatch error emitted; `queueJobsForDeps` is not called.
937	// * Hash is not provided by manifest.
938	// - Hash missing error emitted; `queueJobsForDeps` is not called.
939	// * path-based location is used without a hash.
940	// - Hash is added to the table based on the path alone before
941	// calling run(); no need to add it again.
942	//
943	// If we add a dep as lazy and then later try to add the same dep as eager,
944	// eagerness takes precedence and the existing entry is updated and re-scheduled
945	// for fetching.
946
947	for (dep_names, deps) |dep_name, dep| {
948	var promoted_existing_to_eager = false;
949	const new_fetch = &new_fetches[new_fetch_index];
950	const location: Location = switch (dep.location) {
951	.url => |url| .{
952	.remote = .{
953	.url = url,
954	.hash = h: {
955	const h = dep.hash orelse break :h null;
956	const pkg_hash: Package.Hash = .fromSlice(h);
957	if (h.len == 0) break :h pkg_hash;
958	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
959	if (gop.found_existing) {
960	if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
961	gop.value_ptr.*.lazy_status = .eager;
962	promoted_existing_to_eager = true;
963	} else {
964	continue;
965	}
966	}
967	gop.value_ptr.* = new_fetch;
968	break :h pkg_hash;
969	},
970	},
971	},
972	.path => |rel_path| l: {
973	// This might produce an invalid path, which is checked for
974	// at the beginning of run().
975	const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
976	const pkg_hash = relativePathDigest(new_root, cache_root);
977	const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
978	if (gop.found_existing) {
979	if (!dep.lazy and gop.value_ptr.*.lazy_status != .eager) {
980	gop.value_ptr.*.lazy_status = .eager;
981	promoted_existing_to_eager = true;
982	} else {
983	continue;
984	}
985	}
986	gop.value_ptr.* = new_fetch;
987	break :l .{ .relative_path = new_root };
988	},
989	};
990	prog_names[new_fetch_index] = dep_name;
991	new_fetch_index += 1;
992	if (!promoted_existing_to_eager) {
993	f.job_queue.all_fetches.appendAssumeCapacity(new_fetch);
994	}
995	new_fetch.* = .{
996	.arena = std.heap.ArenaAllocator.init(gpa),
997	.location = location,
998	.location_tok = dep.location_tok,
999	.hash_tok = dep.hash_tok,
1000	.name_tok = dep.name_tok,
1001	.lazy_status = switch (f.job_queue.mode) {
1002	.needed => if (dep.lazy) .available else .eager,
1003	.all => .eager,
1004	},
1005	.parent_package_root = f.package_root,
1006	.remote_package_root = f.remote_package_root,
1007	.parent_manifest_ast = &f.manifest_ast,
1008	.prog_node = f.prog_node,
1009	.job_queue = f.job_queue,
1010	.omit_missing_hash_error = false,
1011	.allow_missing_paths_field = true,
1012	.use_latest_commit = false,
1013
1014	.package_root = undefined,
1015	.error_bundle = undefined,
1016	.manifest = undefined,
1017	.manifest_ast = undefined,
1018	.have_manifest = false,
1019	.computed_hash = undefined,
1020	.has_build_zig = false,
1021	.oom_flag = false,
1022	.latest_commit = null,
1023
1024	.cli_module = null,
1025	};
1026	}
1027
1028	f.prog_node.increaseEstimatedTotalItems(new_fetch_index);
1029
1030	break :nf .{ new_fetches[0..new_fetch_index], prog_names[0..new_fetch_index] };
1031	};
1032
1033	// Now it's time to dispatch tasks.
1034	for (new_fetches, prog_names) |*new_fetch, prog_name| {
1035	f.job_queue.group.async(io, workerRun, .{ new_fetch, prog_name });
1036	}
1037	}
1038
1039	pub fn relativePathDigest(pkg_root: Path, cache_root: Directory) Package.Hash {
1040	return .initPath(pkg_root.sub_path, pkg_root.root_dir.eql(cache_root));
1041	}
1042
1043	pub fn workerRun(f: *Fetch, prog_name: []const u8) Io.Cancelable!void {
1044	const prog_node = f.prog_node.start(prog_name, 0);
1045	defer prog_node.end();
1046
1047	run(f) catch |err| switch (err) {
1048	error.OutOfMemory => f.oom_flag = true,
1049	error.Canceled => |e| return e,
1050	error.FetchFailed => {
1051	// Nothing to do because the errors are already reported in `error_bundle`,
1052	// and a reference is kept to the `Fetch` task inside `all_fetches`.
1053	},
1054	};
1055	}
1056
1057	fn srcLoc(
1058	f: *Fetch,
1059	tok: std.zig.Ast.TokenIndex,
1060	) Allocator.Error!ErrorBundle.SourceLocationIndex {
1061	const ast = f.parent_manifest_ast orelse return .none;
1062	const eb = &f.error_bundle;
1063	const start_loc = ast.tokenLocation(0, tok);
1064	const src_path = try eb.printString("{f}" ++ fs.path.sep_str ++ Manifest.basename, .{f.parent_package_root});
1065	const msg_off = 0;
1066	return eb.addSourceLocation(.{
1067	.src_path = src_path,
1068	.span_start = ast.tokenStart(tok),
1069	.span_end = @intCast(ast.tokenStart(tok) + ast.tokenSlice(tok).len),
1070	.span_main = ast.tokenStart(tok) + msg_off,
1071	.line = @intCast(start_loc.line),
1072	.column = @intCast(start_loc.column),
1073	.source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]),
1074	});
1075	}
1076
1077	fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError {
1078	const eb = &f.error_bundle;
1079	try eb.addRootErrorMessage(.{
1080	.msg = msg_str,
1081	.src_loc = try f.srcLoc(msg_tok),
1082	});
1083	return error.FetchFailed;
1084	}
1085
1086	const Resource = union(enum) {
1087	file: Io.File.Reader,
1088	http_request: HttpRequest,
1089	git: Git,
1090	dir: Io.Dir,
1091
1092	const Git = struct {
1093	session: git.Session,
1094	fetch_stream: git.Session.FetchStream,
1095	want_oid: git.Oid,
1096	};
1097
1098	const HttpRequest = struct {
1099	request: std.http.Client.Request,
1100	response: std.http.Client.Response,
1101	transfer_buffer: []u8,
1102	decompress: std.http.Decompress,
1103	decompress_buffer: []u8,
1104	};
1105
1106	fn deinit(resource: *Resource, io: Io) void {
1107	switch (resource.*) {
1108	.file => |*file_reader| file_reader.file.close(io),
1109	.http_request => |*http_request| http_request.request.deinit(),
1110	.git => |*git_resource| {
1111	git_resource.fetch_stream.deinit();
1112	},
1113	.dir => |*dir| dir.close(io),
1114	}
1115	resource.* = undefined;
1116	}
1117
1118	fn reader(resource: *Resource) *Io.Reader {
1119	return switch (resource.*) {
1120	.file => |*file_reader| return &file_reader.interface,
1121	.http_request => |*http_request| return http_request.response.readerDecompressing(
1122	http_request.transfer_buffer,
1123	&http_request.decompress,
1124	http_request.decompress_buffer,
1125	),
1126	.git => |*g| return &g.fetch_stream.reader,
1127	.dir => unreachable,
1128	};
1129	}
1130	};
1131
1132	const FileType = enum {
1133	tar,
1134	@"tar.gz",
1135	@"tar.xz",
1136	@"tar.zst",
1137	git_pack,
1138	zip,
1139
1140	fn fromPath(file_path: []const u8) ?FileType {
1141	if (ascii.endsWithIgnoreCase(file_path, ".tar")) return .tar;
1142	if (ascii.endsWithIgnoreCase(file_path, ".tgz")) return .@"tar.gz";
1143	if (ascii.endsWithIgnoreCase(file_path, ".tar.gz")) return .@"tar.gz";
1144	if (ascii.endsWithIgnoreCase(file_path, ".txz")) return .@"tar.xz";
1145	if (ascii.endsWithIgnoreCase(file_path, ".tar.xz")) return .@"tar.xz";
1146	if (ascii.endsWithIgnoreCase(file_path, ".tzst")) return .@"tar.zst";
1147	if (ascii.endsWithIgnoreCase(file_path, ".tar.zst")) return .@"tar.zst";
1148	if (ascii.endsWithIgnoreCase(file_path, ".zip")) return .zip;
1149	if (ascii.endsWithIgnoreCase(file_path, ".jar")) return .zip;
1150	return null;
1151	}
1152
1153	/// Parameter is a content-disposition header value.
1154	fn fromContentDisposition(cd_header: []const u8) ?FileType {
1155	const attach_end = ascii.findIgnoreCase(cd_header, "attachment;") orelse
1156	return null;
1157
1158	var value_start = ascii.findIgnoreCasePos(cd_header, attach_end + 1, "filename") orelse
1159	return null;
1160	value_start += "filename".len;
1161	if (cd_header[value_start] == '*') {
1162	value_start += 1;
1163	}
1164	if (cd_header[value_start] != '=') return null;
1165	value_start += 1;
1166
1167	var value_end = std.mem.findPos(u8, cd_header, value_start, ";") orelse cd_header.len;
1168	if (cd_header[value_end - 1] == '\"') {
1169	value_end -= 1;
1170	}
1171	return fromPath(cd_header[value_start..value_end]);
1172	}
1173
1174	test fromContentDisposition {
1175	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attaChment; FILENAME=\"stuff.tar.gz\"; size=42"));
1176	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; filename*=\"stuff.tar.gz\""));
1177	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("ATTACHMENT; filename=\"stuff.tar.xz\""));
1178	try std.testing.expectEqual(@as(?FileType, .@"tar.xz"), fromContentDisposition("attachment; FileName=\"stuff.tar.xz\""));
1179	try std.testing.expectEqual(@as(?FileType, .@"tar.gz"), fromContentDisposition("attachment; FileName*=UTF-8\'\'xyz%2Fstuff.tar.gz"));
1180	try std.testing.expectEqual(@as(?FileType, .tar), fromContentDisposition("attachment; FileName=\"stuff.tar\""));
1181
1182	try std.testing.expect(fromContentDisposition("attachment FileName=\"stuff.tar.gz\"") == null);
1183	try std.testing.expect(fromContentDisposition("attachment; FileName\"stuff.gz\"") == null);
1184	try std.testing.expect(fromContentDisposition("attachment; size=42") == null);
1185	try std.testing.expect(fromContentDisposition("inline; size=42") == null);
1186	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\"; attachment;") == null);
1187	try std.testing.expect(fromContentDisposition("FileName=\"stuff.tar.gz\";") == null);
1188	}
1189	};
1190
1191	const init_resource_buffer_size = git.Packet.max_data_length;
1192
1193	fn initResource(f: *Fetch, uri: std.Uri, resource: *Resource, reader_buffer: []u8) RunError!void {
1194	const io = f.job_queue.io;
1195	const arena = f.arena.allocator();
1196	const eb = &f.error_bundle;
1197
1198	if (ascii.eqlIgnoreCase(uri.scheme, "file")) {
1199	const path = try uri.path.toRawMaybeAlloc(arena);
1200	const file = f.parent_package_root.openFile(io, path, .{}) catch |err| {
1201	return f.fail(f.location_tok, try eb.printString("unable to open {f}/{s}: {t}", .{
1202	f.parent_package_root, path, err,
1203	}));
1204	};
1205	resource.* = .{ .file = file.reader(io, reader_buffer) };
1206	return;
1207	}
1208
1209	const http_client = f.job_queue.http_client;
1210
1211	if (ascii.eqlIgnoreCase(uri.scheme, "http") or
1212	ascii.eqlIgnoreCase(uri.scheme, "https"))
1213	{
1214	resource.* = .{ .http_request = .{
1215	.request = http_client.request(.GET, uri, .{}) catch |err|
1216	return f.fail(f.location_tok, try eb.printString("server connection failed: {t}", .{err})),
1217	.response = undefined,
1218	.transfer_buffer = reader_buffer,
1219	.decompress_buffer = &.{},
1220	.decompress = undefined,
1221	} };
1222	const request = &resource.http_request.request;
1223	errdefer request.deinit();
1224
1225	request.sendBodiless() catch |err|
1226	return f.fail(f.location_tok, try eb.printString("HTTP request failed: {t}", .{err}));
1227
1228	var redirect_buffer: [8000]u8 = undefined;
1229	const response = &resource.http_request.response;
1230	response.* = request.receiveHead(&redirect_buffer) catch |err| switch (err) {
1231	error.ReadFailed => {
1232	return f.fail(f.location_tok, try eb.printString("HTTP response read failure: {t}", .{
1233	request.connection.?.getReadError().?,
1234	}));
1235	},
1236	else => |e| return f.fail(f.location_tok, try eb.printString("invalid HTTP response: {t}", .{e})),
1237	};
1238
1239	if (response.head.status != .ok) return f.fail(f.location_tok, try eb.printString(
1240	"bad HTTP response code: '{d} {s}'",
1241	.{ response.head.status, response.head.status.phrase() orelse "" },
1242	));
1243
1244	resource.http_request.decompress_buffer = try arena.alloc(u8, response.head.content_encoding.minBufferCapacity());
1245	return;
1246	}
1247
1248	if (ascii.eqlIgnoreCase(uri.scheme, "git+http") or
1249	ascii.eqlIgnoreCase(uri.scheme, "git+https"))
1250	{
1251	var transport_uri = uri;
1252	transport_uri.scheme = uri.scheme["git+".len..];
1253	var session = git.Session.init(arena, http_client, transport_uri, reader_buffer) catch |err| {
1254	return f.fail(
1255	f.location_tok,
1256	try eb.printString("unable to discover remote git server capabilities: {t}", .{err}),
1257	);
1258	};
1259
1260	const want_oid = want_oid: {
1261	const want_ref =
1262	if (uri.fragment) |fragment| try fragment.toRawMaybeAlloc(arena) else "HEAD";
1263	if (git.Oid.parseAny(want_ref)) |oid| break :want_oid oid else |_| {}
1264
1265	const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref});
1266	const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref});
1267
1268	var ref_iterator: git.Session.RefIterator = undefined;
1269	session.listRefs(&ref_iterator, .{
1270	.ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag },
1271	.include_peeled = true,
1272	.buffer = reader_buffer,
1273	}) catch |err| return f.fail(f.location_tok, try eb.printString("unable to list refs: {t}", .{err}));
1274	defer ref_iterator.deinit();
1275	while (ref_iterator.next() catch |err| {
1276	return f.fail(f.location_tok, try eb.printString(
1277	"unable to iterate refs: {s}",
1278	.{@errorName(err)},
1279	));
1280	}) |ref| {
1281	if (std.mem.eql(u8, ref.name, want_ref) or
1282	std.mem.eql(u8, ref.name, want_ref_head) or
1283	std.mem.eql(u8, ref.name, want_ref_tag))
1284	{
1285	break :want_oid ref.peeled orelse ref.oid;
1286	}
1287	}
1288	return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref}));
1289	};
1290	if (f.use_latest_commit) {
1291	f.latest_commit = want_oid;
1292	} else if (uri.fragment == null) {
1293	const notes_len = 1;
1294	try eb.addRootErrorMessage(.{
1295	.msg = try eb.addString("url field is missing an explicit ref"),
1296	.src_loc = try f.srcLoc(f.location_tok),
1297	.notes_len = notes_len,
1298	});
1299	const notes_start = try eb.reserveNotes(notes_len);
1300	eb.extra.items[notes_start] = @backingInt(try eb.addErrorMessage(.{
1301	.msg = try eb.printString("try .url = \"{f}#{f}\",", .{
1302	uri.fmt(.{ .scheme = true, .authority = true, .path = true }),
1303	want_oid,
1304	}),
1305	}));
1306	return error.FetchFailed;
1307	}
1308
1309	var want_oid_hex_buf: [git.Oid.max_formatted_length]u8 = undefined;
1310	const want_oid_hex = std.mem.print(&want_oid_hex_buf, "{f}", .{want_oid}) catch unreachable;
1311	resource.* = .{ .git = .{
1312	.session = session,
1313	.fetch_stream = undefined,
1314	.want_oid = want_oid,
1315	} };
1316	const fetch_stream = &resource.git.fetch_stream;
1317	session.fetch(fetch_stream, &.{want_oid_hex}, reader_buffer) catch |err| {
1318	return f.fail(f.location_tok, try eb.printString("unable to create fetch stream: {t}", .{err}));
1319	};
1320	errdefer fetch_stream.deinit(fetch_stream);
1321
1322	return;
1323	}
1324
1325	return f.fail(f.location_tok, try eb.printString("unsupported URL scheme: {s}", .{uri.scheme}));
1326	}
1327
1328	fn unpackResource(
1329	f: *Fetch,
1330	resource: *Resource,
1331	uri_path: []const u8,
1332	tmp_directory: Directory,
1333	) RunError!UnpackResult {
1334	const eb = &f.error_bundle;
1335	const file_type = switch (resource.*) {
1336	.file => FileType.fromPath(uri_path) orelse
1337	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})),
1338
1339	.http_request => |*http_request| ft: {
1340	const head = &http_request.response.head;
1341
1342	// Content-Type takes first precedence.
1343	const content_type = head.content_type orelse
1344	return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header"));
1345
1346	// Extract the MIME type, ignoring charset and boundary directives
1347	const mime_type_end = std.mem.find(u8, content_type, ";") orelse content_type.len;
1348	const mime_type = content_type[0..mime_type_end];
1349
1350	if (ascii.eqlIgnoreCase(mime_type, "application/x-tar"))
1351	break :ft .tar;
1352
1353	if (ascii.eqlIgnoreCase(mime_type, "application/gzip") or
1354	ascii.eqlIgnoreCase(mime_type, "application/x-gzip") or
1355	ascii.eqlIgnoreCase(mime_type, "application/tar+gzip") or
1356	ascii.eqlIgnoreCase(mime_type, "application/x-tar-gz") or
1357	ascii.eqlIgnoreCase(mime_type, "application/x-gtar-compressed"))
1358	{
1359	break :ft .@"tar.gz";
1360	}
1361
1362	if (ascii.eqlIgnoreCase(mime_type, "application/x-xz"))
1363	break :ft .@"tar.xz";
1364
1365	if (ascii.eqlIgnoreCase(mime_type, "application/zstd"))
1366	break :ft .@"tar.zst";
1367
1368	if (ascii.eqlIgnoreCase(mime_type, "application/zip") or
1369	ascii.eqlIgnoreCase(mime_type, "application/x-zip-compressed") or
1370	ascii.eqlIgnoreCase(mime_type, "application/java-archive"))
1371	{
1372	break :ft .zip;
1373	}
1374
1375	if (!ascii.eqlIgnoreCase(mime_type, "application/octet-stream") and
1376	!ascii.eqlIgnoreCase(mime_type, "application/x-compressed"))
1377	{
1378	return f.fail(f.location_tok, try eb.printString(
1379	"unrecognized 'Content-Type' header: '{s}'",
1380	.{content_type},
1381	));
1382	}
1383
1384	// Next, the filename from 'content-disposition: attachment' takes precedence.
1385	if (head.content_disposition) |cd_header| {
1386	break :ft FileType.fromContentDisposition(cd_header) orelse {
1387	return f.fail(f.location_tok, try eb.printString(
1388	"unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream",
1389	.{cd_header},
1390	));
1391	};
1392	}
1393
1394	// Finally, the path from the URI is used.
1395	break :ft FileType.fromPath(uri_path) orelse {
1396	return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path}));
1397	};
1398	},
1399
1400	.git => .git_pack,
1401
1402	.dir => |dir| {
1403	f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| {
1404	return f.fail(f.location_tok, try eb.printString("unable to copy directory '{s}': {t}", .{
1405	uri_path, err,
1406	}));
1407	};
1408	return .{};
1409	},
1410	};
1411
1412	switch (file_type) {
1413	.tar => {
1414	return unpackTarball(f, tmp_directory.handle, resource.reader());
1415	},
1416	.@"tar.gz" => {
1417	var flate_buffer: [std.compress.flate.max_window_len]u8 = undefined;
1418	var decompress: std.compress.flate.Decompress = .init(resource.reader(), .gzip, &flate_buffer);
1419	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1420	},
1421	.@"tar.xz" => {
1422	const gpa = f.arena.child_allocator;
1423	var decompress = std.compress.xz.Decompress.init(resource.reader(), gpa, &.{}) catch |err|
1424	return f.fail(f.location_tok, try eb.printString("unable to decompress tarball: {t}", .{err}));
1425	defer decompress.deinit();
1426	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1427	},
1428	.@"tar.zst" => {
1429	const window_len = std.compress.zstd.default_window_len;
1430	const window_buffer = try f.arena.allocator().alloc(u8, window_len + std.compress.zstd.block_size_max);
1431	var decompress: std.compress.zstd.Decompress = .init(resource.reader(), window_buffer, .{
1432	.verify_checksum = false,
1433	.window_len = window_len,
1434	});
1435	return try unpackTarball(f, tmp_directory.handle, &decompress.reader);
1436	},
1437	.git_pack => return unpackGitPack(f, tmp_directory.handle, &resource.git) catch |err| switch (err) {
1438	error.FetchFailed, error.OutOfMemory => |e| return e,
1439	else => |e| return f.fail(f.location_tok, try eb.printString("unable to unpack git files: {t}", .{e})),
1440	},
1441	.zip => return unzip(f, tmp_directory.handle, resource.reader()) catch |err| switch (err) {
1442	error.ReadFailed => return f.fail(f.location_tok, try eb.printString(
1443	"failed reading resource: {t}",
1444	.{err},
1445	)),
1446	else => |e| return e,
1447	},
1448	}
1449	}
1450
1451	fn unpackTarball(f: *Fetch, out_dir: Io.Dir, reader: *Io.Reader) RunError!UnpackResult {
1452	const eb = &f.error_bundle;
1453	const arena = f.arena.allocator();
1454	const io = f.job_queue.io;
1455
1456	var diagnostics: std.tar.Diagnostics = .{ .allocator = arena };
1457
1458	std.tar.extract(io, out_dir, reader, .{
1459	.diagnostics = &diagnostics,
1460	.strip_components = 0,
1461	.mode_mode = .ignore,
1462	.exclude_empty_directories = true,
1463	}) catch |err| return f.fail(
1464	f.location_tok,
1465	try eb.printString("unable to unpack tarball to temporary directory: {t}", .{err}),
1466	);
1467
1468	var res: UnpackResult = .{ .root_dir = diagnostics.root_dir };
1469	if (diagnostics.errors.items.len > 0) {
1470	try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack tarball");
1471	for (diagnostics.errors.items) |item| {
1472	switch (item) {
1473	.unable_to_create_file => |i| res.unableToCreateFile(stripRoot(i.file_name, res.root_dir), i.code),
1474	.unable_to_create_sym_link => |i| res.unableToCreateSymLink(stripRoot(i.file_name, res.root_dir), i.link_name, i.code),
1475	.unsupported_file_type => |i| res.unsupportedFileType(stripRoot(i.file_name, res.root_dir), @backingInt(i.file_type)),
1476	.components_outside_stripped_prefix => unreachable, // unreachable with strip_components = 0
1477	}
1478	}
1479	}
1480	return res;
1481	}
1482
1483	fn unzip(
1484	f: *Fetch,
1485	out_dir: Io.Dir,
1486	reader: *Io.Reader,
1487	) error{ ReadFailed, OutOfMemory, Canceled, FetchFailed }!UnpackResult {
1488	// We write the entire contents to a file first because zip files
1489	// must be processed back to front and they could be too large to
1490	// load into memory.
1491
1492	const io = f.job_queue.io;
1493	const cache_root = f.job_queue.global_cache;
1494	const prefix = "tmp/";
1495	const suffix = ".zip";
1496	const eb = &f.error_bundle;
1497	const random_len = @sizeOf(u64) * 2;
1498
1499	var zip_path: [prefix.len + random_len + suffix.len]u8 = undefined;
1500	zip_path[0..prefix.len].* = prefix.*;
1501	zip_path[prefix.len + random_len ..].* = suffix.*;
1502
1503	var zip_file = while (true) {
1504	const random_integer = r: {
1505	var x: u64 = undefined;
1506	io.random(@ptrCast(&x));
1507	break :r x;
1508	};
1509	zip_path[prefix.len..][0..random_len].* = std.fmt.hex(random_integer);
1510
1511	break cache_root.handle.createFile(io, &zip_path, .{
1512	.exclusive = true,
1513	.read = true,
1514	}) catch |err| switch (err) {
1515	error.PathAlreadyExists => continue,
1516	error.FileNotFound => {
1517	cache_root.handle.createDir(io, prefix, .default_dir) catch |dir_err| switch (dir_err) {
1518	error.Canceled => |e| return e,
1519	// error.PathAlreadyExists is considered a failure here because
1520	// it implies that the prefix is not a directory.
1521	else => |e| return f.fail(
1522	f.location_tok,
1523	try eb.printString("failed to create temporary directory: {t}", .{e}),
1524	),
1525	};
1526	continue;
1527	},
1528	error.Canceled => |e| return e,
1529	else => |e| return f.fail(
1530	f.location_tok,
1531	try eb.printString("failed to create temporary zip file: {t}", .{e}),
1532	),
1533	};
1534	};
1535	defer zip_file.close(io);
1536	var zip_file_buffer: [4096]u8 = undefined;
1537	var zip_file_reader = b: {
1538	var zip_file_writer = zip_file.writer(io, &zip_file_buffer);
1539
1540	_ = reader.streamRemaining(&zip_file_writer.interface) catch |err| switch (err) {
1541	error.ReadFailed => |e| return e,
1542	error.WriteFailed => return f.fail(
1543	f.location_tok,
1544	try eb.printString("failed writing temporary zip file: {t}", .{err}),
1545	),
1546	};
1547	zip_file_writer.interface.flush() catch |err| return f.fail(
1548	f.location_tok,
1549	try eb.printString("failed writing temporary zip file: {t}", .{err}),
1550	);
1551	break :b zip_file_writer.moveToReader();
1552	};
1553
1554	var diagnostics: std.zip.Diagnostics = .{ .allocator = f.arena.allocator() };
1555	// no need to deinit since we are using an arena allocator
1556
1557	zip_file_reader.seekTo(0) catch |err|
1558	return f.fail(f.location_tok, try eb.printString("failed to seek temporary zip file: {t}", .{err}));
1559	std.zip.extract(out_dir, &zip_file_reader, .{
1560	.allow_backslashes = true,
1561	.diagnostics = &diagnostics,
1562	}) catch |err| return f.fail(f.location_tok, try eb.printString("zip extract failed: {t}", .{err}));
1563
1564	cache_root.handle.deleteFile(io, &zip_path) catch |err|
1565	return f.fail(f.location_tok, try eb.printString("delete temporary zip failed: {t}", .{err}));
1566
1567	return .{ .root_dir = diagnostics.root_dir };
1568	}
1569
1570	fn unpackGitPack(f: *Fetch, out_dir: Io.Dir, resource: *Resource.Git) anyerror!UnpackResult {
1571	const io = f.job_queue.io;
1572	const arena = f.arena.allocator();
1573	// TODO don't try to get a gpa from an arena. expose this dependency higher up
1574	// because the backing of arena could be page allocator
1575	const gpa = f.arena.child_allocator;
1576	const object_format: git.Oid.Format = resource.want_oid;
1577
1578	var res: UnpackResult = .{};
1579	// The .git directory is used to store the packfile and associated index, but
1580	// we do not attempt to replicate the exact structure of a real .git
1581	// directory, since that isn't relevant for fetching a package.
1582	{
1583	var pack_dir = try out_dir.createDirPathOpen(io, ".git", .{});
1584	defer pack_dir.close(io);
1585	var pack_file = try pack_dir.createFile(io, "pkg.pack", .{ .read = true });
1586	defer pack_file.close(io);
1587	var pack_file_buffer: [4096]u8 = undefined;
1588	var pack_file_reader = b: {
1589	var pack_file_writer = pack_file.writer(io, &pack_file_buffer);
1590	const fetch_reader = &resource.fetch_stream.reader;
1591	_ = try fetch_reader.streamRemaining(&pack_file_writer.interface);
1592	try pack_file_writer.interface.flush();
1593	break :b pack_file_writer.moveToReader();
1594	};
1595
1596	var index_file = try pack_dir.createFile(io, "pkg.idx", .{ .read = true });
1597	defer index_file.close(io);
1598	var index_file_buffer: [2000]u8 = undefined;
1599	var index_file_writer = index_file.writer(io, &index_file_buffer);
1600	{
1601	const index_prog_node = f.prog_node.start("Index pack", 0);
1602	defer index_prog_node.end();
1603	try git.indexPack(gpa, object_format, &pack_file_reader, &index_file_writer);
1604	}
1605
1606	{
1607	var index_file_reader = index_file.reader(io, &index_file_buffer);
1608	const checkout_prog_node = f.prog_node.start("Checkout", 0);
1609	defer checkout_prog_node.end();
1610	var repository: git.Repository = undefined;
1611	try repository.init(gpa, object_format, &pack_file_reader, &index_file_reader);
1612	defer repository.deinit();
1613	var diagnostics: git.Diagnostics = .{ .allocator = arena };
1614	try repository.checkout(io, out_dir, resource.want_oid, &diagnostics);
1615
1616	if (diagnostics.errors.items.len > 0) {
1617	try res.allocErrors(arena, diagnostics.errors.items.len, "unable to unpack packfile");
1618	for (diagnostics.errors.items) |item| {
1619	switch (item) {
1620	.unable_to_create_file => |i| res.unableToCreateFile(i.file_name, i.code),
1621	.unable_to_create_sym_link => |i| res.unableToCreateSymLink(i.file_name, i.link_name, i.code),
1622	}
1623	}
1624	}
1625	}
1626	}
1627
1628	try out_dir.deleteTree(io, ".git");
1629	return res;
1630	}
1631
1632	fn recursiveDirectoryCopy(f: *Fetch, dir: Io.Dir, tmp_dir: Io.Dir) anyerror!void {
1633	const gpa = f.arena.child_allocator;
1634	const io = f.job_queue.io;
1635	// Recursive directory copy.
1636	var it = try dir.walk(gpa);
1637	defer it.deinit();
1638	while (try it.next(io)) |entry| {
1639	switch (entry.kind) {
1640	.directory => {}, // omit empty directories
1641	.file => {
1642	dir.copyFile(entry.path, tmp_dir, entry.path, io, .{}) catch |err| switch (err) {
1643	error.FileNotFound => {
1644	if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname);
1645	try dir.copyFile(entry.path, tmp_dir, entry.path, io, .{});
1646	},
1647	else => |e| return e,
1648	};
1649	},
1650	.sym_link => {
1651	var buf: [fs.max_path_bytes]u8 = undefined;
1652	const link_name = buf[0..try dir.readLink(io, entry.path, &buf)];
1653	// TODO: if this would create a symlink to outside
1654	// the destination directory, fail with an error instead.
1655	tmp_dir.symLink(io, link_name, entry.path, .{}) catch |err| switch (err) {
1656	error.FileNotFound => {
1657	if (fs.path.dirname(entry.path)) |dirname| try tmp_dir.createDirPath(io, dirname);
1658	try tmp_dir.symLink(io, link_name, entry.path, .{});
1659	},
1660	else => |e| return e,
1661	};
1662	},
1663	else => return error.IllegalFileTypeInPackage,
1664	}
1665	}
1666	}
1667
1668	pub fn renameTmpIntoCache(io: Io, tmp_path: Path, dest_path: Path) !void {
1669	var handled_missing_dir = false;
1670	while (true) {
1671	Io.Dir.rename(
1672	tmp_path.root_dir.handle,
1673	tmp_path.sub_path,
1674	dest_path.root_dir.handle,
1675	dest_path.sub_path,
1676	io,
1677	) catch |err| switch (err) {
1678	error.FileNotFound => {
1679	if (handled_missing_dir) return err;
1680	const parent_sub_path = Io.Dir.path.dirname(dest_path.sub_path).?;
1681	dest_path.root_dir.handle.createDir(io, parent_sub_path, .default_dir) catch |er| switch (er) {
1682	error.PathAlreadyExists => handled_missing_dir = true,
1683	else => |e| return e,
1684	};
1685	continue;
1686	},
1687	error.DirNotEmpty, error.AccessDenied => {
1688	// Package has been already downloaded and may already be in use on the system.
1689	tmp_path.root_dir.handle.deleteTree(io, tmp_path.sub_path) catch |er| switch (er) {
1690	error.Canceled => |e| return e,
1691	// Garbage files leftover in zig-cache/tmp/ is, as they say
1692	// on Star Trek, "operating within normal parameters".
1693	else => |e| log.warn("failed to delete temporary directory {f}: {t}", .{ tmp_path, e }),
1694	};
1695	},
1696	else => |e| return e,
1697	};
1698	break;
1699	}
1700	}
1701
1702	const ComputedHash = struct {
1703	digest: Package.Hash.Digest,
1704	total_size: u64,
1705	};
1706
1707	/// Assumes that files not included in the package have already been filtered
1708	/// prior to calling this function. This ensures that files not protected by
1709	/// the hash are not present on the file system. Empty directories are *not
1710	/// hashed* and must not be present on the file system when calling this
1711	/// function.
1712	fn computeHash(f: *Fetch, pkg_path: Path, filter: Filter) RunError!ComputedHash {
1713	const io = f.job_queue.io;
1714	// All the path name strings need to be in memory for sorting.
1715	const arena = f.arena.allocator();
1716	const gpa = f.arena.child_allocator;
1717	const eb = &f.error_bundle;
1718	const root_dir = pkg_path.root_dir.handle;
1719
1720	// Collect all files, recursively, then sort.
1721	var all_files = std.array_list.Managed(*HashedFile).init(gpa);
1722	defer all_files.deinit();
1723
1724	var deleted_files = std.array_list.Managed(*DeletedFile).init(gpa);
1725	defer deleted_files.deinit();
1726
1727	// Track directories which had any files deleted from them so that empty directories
1728	// can be deleted.
1729	var sus_dirs: std.array_hash_map.String(void) = .empty;
1730	defer sus_dirs.deinit(gpa);
1731
1732	var walker = try root_dir.walk(gpa);
1733	defer walker.deinit();
1734
1735	// Total number of bytes of file contents included in the package.
1736	var total_size: u64 = 0;
1737
1738	{
1739	// The final hash will be a hash of each file hashed independently. This
1740	// allows hashing in parallel.
1741	var group: Io.Group = .init;
1742	defer group.cancel(io);
1743
1744	while (walker.next(io) catch |err| {
1745	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1746	"unable to walk temporary directory '{f}': {t}",
1747	.{ pkg_path, err },
1748	) });
1749	return error.FetchFailed;
1750	}) |entry| {
1751	if (entry.kind == .directory) continue;
1752
1753	const entry_pkg_path = stripRoot(entry.path, pkg_path.sub_path);
1754	if (!filter.includePath(entry_pkg_path)) {
1755	// Delete instead of including in hash calculation.
1756	const fs_path = try arena.dupe(u8, entry.path);
1757
1758	// Also track the parent directory in case it becomes empty.
1759	if (fs.path.dirname(fs_path)) |parent|
1760	try sus_dirs.put(gpa, parent, {});
1761
1762	const deleted_file = try arena.create(DeletedFile);
1763	deleted_file.* = .{
1764	.fs_path = fs_path,
1765	.failure = undefined, // to be populated by the worker
1766	};
1767	group.async(io, workerDeleteFile, .{ io, root_dir, deleted_file });
1768	try deleted_files.append(deleted_file);
1769	continue;
1770	}
1771
1772	const kind: HashedFile.Kind = switch (entry.kind) {
1773	.directory => unreachable,
1774	.file => .file,
1775	.sym_link => .link,
1776	else => return f.fail(f.location_tok, try eb.printString(
1777	"package contains '{s}' which has illegal file type '{t}'",
1778	.{ entry.path, entry.kind },
1779	)),
1780	};
1781
1782	if (std.mem.eql(u8, entry_pkg_path, std.zig.build_zig_basename))
1783	f.has_build_zig = true;
1784
1785	const fs_path = try arena.dupe(u8, entry.path);
1786	const hashed_file = try arena.create(HashedFile);
1787	hashed_file.* = .{
1788	.fs_path = fs_path,
1789	.normalized_path = try normalizePathAlloc(arena, entry_pkg_path),
1790	.kind = kind,
1791	.hash = undefined, // to be populated by the worker
1792	.failure = undefined, // to be populated by the worker
1793	.size = undefined, // to be populated by the worker
1794	};
1795	group.async(io, workerHashFile, .{ io, root_dir, hashed_file });
1796	try all_files.append(hashed_file);
1797	}
1798
1799	try group.await(io);
1800	}
1801
1802	{
1803	// Sort by length, descending, so that child directories get removed first.
1804	sus_dirs.sortUnstable(@as(struct {
1805	keys: []const []const u8,
1806	pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
1807	return ctx.keys[b_index].len < ctx.keys[a_index].len;
1808	}
1809	}, .{ .keys = sus_dirs.keys() }));
1810
1811	// During this loop, more entries will be added, so we must loop by index.
1812	var i: usize = 0;
1813	while (i < sus_dirs.count()) : (i += 1) {
1814	const sus_dir = sus_dirs.keys()[i];
1815	root_dir.deleteDir(io, sus_dir) catch |err| switch (err) {
1816	error.DirNotEmpty => continue,
1817	error.FileNotFound => continue,
1818	else => |e| {
1819	try eb.addRootErrorMessage(.{ .msg = try eb.printString(
1820	"unable to delete empty directory '{s}': {s}",
1821	.{ sus_dir, @errorName(e) },
1822	) });
1823	return error.FetchFailed;
1824	},
1825	};
1826	if (fs.path.dirname(sus_dir)) |parent| {
1827	try sus_dirs.put(gpa, parent, {});
1828	}
1829	}
1830	}
1831
1832	std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
1833
1834	var hasher = Package.Hash.Algo.init(.{});
1835	var any_failures = false;
1836	for (all_files.items) |hashed_file| {
1837	hashed_file.failure catch |err| {
1838	any_failures = true;
1839	try eb.addRootErrorMessage(.{
1840	.msg = try eb.printString("unable to hash '{s}': {s}", .{
1841	hashed_file.fs_path, @errorName(err),
1842	}),
1843	});
1844	};
1845	hasher.update(&hashed_file.hash);
1846	total_size += hashed_file.size;
1847	}
1848	for (deleted_files.items) |deleted_file| {
1849	deleted_file.failure catch |err| {
1850	any_failures = true;
1851	try eb.addRootErrorMessage(.{
1852	.msg = try eb.printString("failed to delete excluded path '{s}' from package: {s}", .{
1853	deleted_file.fs_path, @errorName(err),
1854	}),
1855	});
1856	};
1857	}
1858
1859	if (any_failures) return error.FetchFailed;
1860
1861	if (f.job_queue.debug_hash) {
1862	assert(!f.job_queue.recursive);
1863	// Print something to stdout that can be text diffed to figure out why
1864	// the package hash is different.
1865	dumpHashInfo(io, all_files.items) catch |err|
1866	std.process.fatal("unable to write to stdout: {t}", .{err});
1867	}
1868
1869	return .{
1870	.digest = hasher.finalResult(),
1871	.total_size = total_size,
1872	};
1873	}
1874
1875	fn dumpHashInfo(io: Io, all_files: []const *const HashedFile) !void {
1876	var stdout_buffer: [1024]u8 = undefined;
1877	var stdout_writer: Io.File.Writer = .initStreaming(.stdout(), io, &stdout_buffer);
1878	dumpHashInfoWriter(&stdout_writer.interface, all_files) catch |err| switch (err) {
1879	error.WriteFailed => return stdout_writer.err.?,
1880	};
1881	try stdout_writer.flush();
1882	}
1883
1884	fn dumpHashInfoWriter(w: *Io.Writer, all_files: []const *const HashedFile) Io.Writer.Error!void {
1885	for (all_files) |hashed_file| {
1886	try w.print("{t}: {x}: {s}\n", .{ hashed_file.kind, &hashed_file.hash, hashed_file.normalized_path });
1887	}
1888	}
1889
1890	fn workerHashFile(io: Io, dir: Io.Dir, hashed_file: *HashedFile) void {
1891	hashed_file.failure = hashFileFallible(io, dir, hashed_file);
1892	}
1893
1894	fn workerDeleteFile(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) void {
1895	deleted_file.failure = deleteFileFallible(io, dir, deleted_file);
1896	}
1897
1898	fn hashFileFallible(io: Io, dir: Io.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
1899	var buf: [8000]u8 = undefined;
1900	var hasher = Package.Hash.Algo.init(.{});
1901	hasher.update(hashed_file.normalized_path);
1902	var file_size: u64 = 0;
1903
1904	switch (hashed_file.kind) {
1905	.file => {
1906	var file = try dir.openFile(io, hashed_file.fs_path, .{});
1907	defer file.close(io);
1908	// Hard-coded false executable bit: https://github.com/ziglang/zig/issues/17463
1909	hasher.update(&.{ 0, 0 });
1910	var file_header: FileHeader = .{};
1911	while (true) {
1912	const bytes_read = try file.readPositional(io, &.{&buf}, file_size);
1913	if (bytes_read == 0) break;
1914	file_size += bytes_read;
1915	hasher.update(buf[0..bytes_read]);
1916	file_header.update(buf[0..bytes_read]);
1917	}
1918	if (file_header.isExecutable()) {
1919	try setExecutable(io, file);
1920	}
1921	},
1922	.link => {
1923	const link_name = buf[0..try dir.readLink(io, hashed_file.fs_path, &buf)];
1924	if (fs.path.sep != canonical_sep) {
1925	// Package hashes are intended to be consistent across
1926	// platforms which means we must normalize path separators
1927	// inside symlinks.
1928	normalizePath(link_name);
1929	}
1930	hasher.update(link_name);
1931	},
1932	}
1933	hasher.final(&hashed_file.hash);
1934	hashed_file.size = file_size;
1935	}
1936
1937	fn deleteFileFallible(io: Io, dir: Io.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
1938	try dir.deleteFile(io, deleted_file.fs_path);
1939	}
1940
1941	fn setExecutable(io: Io, file: Io.File) !void {
1942	if (!Io.File.Permissions.has_executable_bit) return;
1943	try file.setPermissions(io, .executable_file);
1944	}
1945
1946	const DeletedFile = struct {
1947	fs_path: []const u8,
1948	failure: Error!void,
1949
1950	const Error =
1951	Io.Dir.DeleteFileError ||
1952	Io.Dir.DeleteDirError;
1953	};
1954
1955	const HashedFile = struct {
1956	fs_path: []const u8,
1957	normalized_path: []const u8,
1958	hash: Package.Hash.Digest,
1959	failure: Error!void,
1960	kind: Kind,
1961	size: u64,
1962
1963	const Error =
1964	Io.File.OpenError ||
1965	Io.File.ReadPositionalError ||
1966	Io.File.StatError ||
1967	Io.File.SetPermissionsError ||
1968	Io.Dir.ReadLinkError;
1969
1970	const Kind = enum { file, link };
1971
1972	fn lessThan(context: void, lhs: *const HashedFile, rhs: *const HashedFile) bool {
1973	_ = context;
1974	return std.mem.lessThan(u8, lhs.normalized_path, rhs.normalized_path);
1975	}
1976	};
1977
1978	/// Strips root directory name from file system path.
1979	fn stripRoot(fs_path: []const u8, root_dir: []const u8) []const u8 {
1980	if (root_dir.len == 0 or fs_path.len <= root_dir.len) return fs_path;
1981
1982	if (std.mem.eql(u8, fs_path[0..root_dir.len], root_dir) and fs.path.isSep(fs_path[root_dir.len])) {
1983	return fs_path[root_dir.len + 1 ..];
1984	}
1985
1986	return fs_path;
1987	}
1988
1989	/// Make a file system path identical independently of operating system path inconsistencies.
1990	/// This converts backslashes into forward slashes.
1991	fn normalizePathAlloc(arena: Allocator, pkg_path: []const u8) ![]const u8 {
1992	const normalized = try arena.dupe(u8, pkg_path);
1993	if (fs.path.sep == canonical_sep) return normalized;
1994	normalizePath(normalized);
1995	return normalized;
1996	}
1997
1998	const canonical_sep = fs.path.sep_posix;
1999
2000	fn normalizePath(bytes: []u8) void {
2001	assert(fs.path.sep != canonical_sep);
2002	std.mem.replaceScalar(u8, bytes, fs.path.sep, canonical_sep);
2003	}
2004
2005	const Filter = struct {
2006	include_paths: std.array_hash_map.String(void) = .empty,
2007
2008	/// sub_path is relative to the package root.
2009	pub fn includePath(self: *const Filter, sub_path: []const u8) bool {
2010	if (self.include_paths.count() == 0) return true;
2011	if (self.include_paths.contains("")) return true;
2012	if (self.include_paths.contains(".")) return true;
2013	if (self.include_paths.contains(sub_path)) return true;
2014
2015	// Check if any included paths are parent directories of sub_path.
2016	var dirname = sub_path;
2017	while (std.fs.path.dirname(dirname)) |next_dirname| {
2018	if (self.include_paths.contains(next_dirname)) return true;
2019	dirname = next_dirname;
2020	}
2021
2022	return false;
2023	}
2024
2025	test includePath {
2026	const gpa = std.testing.allocator;
2027	var filter: Filter = .{};
2028	defer filter.include_paths.deinit(gpa);
2029
2030	try filter.include_paths.put(gpa, "src", {});
2031	try std.testing.expect(filter.includePath("src/core/unix/SDL_poll.c"));
2032	try std.testing.expect(!filter.includePath(".gitignore"));
2033	}
2034	};
2035
2036	pub fn depDigest(pkg_root: Path, cache_root: Directory, dep: Manifest.Dependency) ?Package.Hash {
2037	if (dep.hash) |h| return .fromSlice(h);
2038
2039	switch (dep.location) {
2040	.url => return null,
2041	.path => |rel_path| {
2042	var buf: [fs.max_path_bytes]u8 = undefined;
2043	var fba = std.heap.FixedBufferAllocator.init(&buf);
2044	const new_root = pkg_root.resolvePosix(fba.allocator(), rel_path) catch
2045	return null;
2046	return relativePathDigest(new_root, cache_root);
2047	},
2048	}
2049	}
2050
2051	// Detects executable header: ELF or Macho-O magic header or shebang line.
2052	const FileHeader = struct {
2053	header: [4]u8 = undefined,
2054	bytes_read: usize = 0,
2055
2056	pub fn update(self: *FileHeader, buf: []const u8) void {
2057	if (self.bytes_read >= self.header.len) return;
2058	const n = @min(self.header.len - self.bytes_read, buf.len);
2059	@memcpy(self.header[self.bytes_read..][0..n], buf[0..n]);
2060	self.bytes_read += n;
2061	}
2062
2063	fn isScript(self: *FileHeader) bool {
2064	const shebang = "#!";
2065	return std.mem.eql(u8, self.header[0..@min(self.bytes_read, shebang.len)], shebang);
2066	}
2067
2068	fn isElf(self: *FileHeader) bool {
2069	const elf_magic = std.elf.MAGIC;
2070	return std.mem.eql(u8, self.header[0..@min(self.bytes_read, elf_magic.len)], elf_magic);
2071	}
2072
2073	fn isMachO(self: *FileHeader) bool {
2074	if (self.bytes_read < 4) return false;
2075	const magic_number = std.mem.readInt(u32, &self.header, builtin.cpu.arch.endian());
2076	return magic_number == std.macho.MH_MAGIC or
2077	magic_number == std.macho.MH_MAGIC_64 or
2078	magic_number == std.macho.FAT_MAGIC or
2079	magic_number == std.macho.FAT_MAGIC_64 or
2080	magic_number == std.macho.MH_CIGAM or
2081	magic_number == std.macho.MH_CIGAM_64 or
2082	magic_number == std.macho.FAT_CIGAM or
2083	magic_number == std.macho.FAT_CIGAM_64;
2084	}
2085
2086	pub fn isExecutable(self: *FileHeader) bool {
2087	return self.isScript() or self.isElf() or self.isMachO();
2088	}
2089	};
2090
2091	test FileHeader {
2092	var h: FileHeader = .{};
2093	try std.testing.expect(!h.isExecutable());
2094
2095	const elf_magic = std.elf.MAGIC;
2096	h.update(elf_magic[0..2]);
2097	try std.testing.expect(!h.isExecutable());
2098	h.update(elf_magic[2..4]);
2099	try std.testing.expect(h.isExecutable());
2100
2101	h.update(elf_magic[2..4]);
2102	try std.testing.expect(h.isExecutable());
2103
2104	const macho64_magic_bytes = [_]u8{ 0xCF, 0xFA, 0xED, 0xFE };
2105	h.bytes_read = 0;
2106	h.update(&macho64_magic_bytes);
2107	try std.testing.expect(h.isExecutable());
2108
2109	const macho64_cigam_bytes = [_]u8{ 0xFE, 0xED, 0xFA, 0xCF };
2110	h.bytes_read = 0;
2111	h.update(&macho64_cigam_bytes);
2112	try std.testing.expect(h.isExecutable());
2113	}
2114
2115	// Result of the `unpackResource` operation. Enables collecting errors from
2116	// tar/git diagnostic, filtering that errors by manifest inclusion rules and
2117	// emitting remaining errors to an `ErrorBundle`.
2118	const UnpackResult = struct {
2119	errors: []Error = undefined,
2120	errors_count: usize = 0,
2121	root_error_message: []const u8 = "",
2122
2123	// A non empty value means that the package contents are inside a
2124	// sub-directory indicated by the named path.
2125	root_dir: []const u8 = "",
2126
2127	const Error = union(enum) {
2128	unable_to_create_sym_link: struct {
2129	code: anyerror,
2130	file_name: []const u8,
2131	link_name: []const u8,
2132	},
2133	unable_to_create_file: struct {
2134	code: anyerror,
2135	file_name: []const u8,
2136	},
2137	unsupported_file_type: struct {
2138	file_name: []const u8,
2139	file_type: u8,
2140	},
2141
2142	fn excluded(self: Error, filter: Filter) bool {
2143	const file_name = switch (self) {
2144	.unable_to_create_file => |info| info.file_name,
2145	.unable_to_create_sym_link => |info| info.file_name,
2146	.unsupported_file_type => |info| info.file_name,
2147	};
2148	return !filter.includePath(file_name);
2149	}
2150	};
2151
2152	fn allocErrors(self: *UnpackResult, arena: std.mem.Allocator, n: usize, root_error_message: []const u8) !void {
2153	self.root_error_message = try arena.dupe(u8, root_error_message);
2154	self.errors = try arena.alloc(UnpackResult.Error, n);
2155	}
2156
2157	fn hasErrors(self: *UnpackResult) bool {
2158	return self.errors_count > 0;
2159	}
2160
2161	fn unableToCreateFile(self: *UnpackResult, file_name: []const u8, err: anyerror) void {
2162	self.errors[self.errors_count] = .{ .unable_to_create_file = .{
2163	.code = err,
2164	.file_name = file_name,
2165	} };
2166	self.errors_count += 1;
2167	}
2168
2169	fn unableToCreateSymLink(self: *UnpackResult, file_name: []const u8, link_name: []const u8, err: anyerror) void {
2170	self.errors[self.errors_count] = .{ .unable_to_create_sym_link = .{
2171	.code = err,
2172	.file_name = file_name,
2173	.link_name = link_name,
2174	} };
2175	self.errors_count += 1;
2176	}
2177
2178	fn unsupportedFileType(self: *UnpackResult, file_name: []const u8, file_type: u8) void {
2179	self.errors[self.errors_count] = .{ .unsupported_file_type = .{
2180	.file_name = file_name,
2181	.file_type = file_type,
2182	} };
2183	self.errors_count += 1;
2184	}
2185
2186	fn validate(self: *UnpackResult, f: *Fetch, filter: Filter) !void {
2187	if (self.errors_count == 0) return;
2188
2189	var unfiltered_errors: u32 = 0;
2190	for (self.errors) |item| {
2191	if (item.excluded(filter)) continue;
2192	unfiltered_errors += 1;
2193	}
2194	if (unfiltered_errors == 0) return;
2195
2196	// Emmit errors to an `ErrorBundle`.
2197	const eb = &f.error_bundle;
2198	try eb.addRootErrorMessage(.{
2199	.msg = try eb.addString(self.root_error_message),
2200	.src_loc = try f.srcLoc(f.location_tok),
2201	.notes_len = unfiltered_errors,
2202	});
2203	var note_i: u32 = try eb.reserveNotes(unfiltered_errors);
2204	for (self.errors) |item| {
2205	if (item.excluded(filter)) continue;
2206	switch (item) {
2207	.unable_to_create_sym_link => |info| {
2208	eb.extra.items[note_i] = @backingInt(try eb.addErrorMessage(.{
2209	.msg = try eb.printString("unable to create symlink from '{s}' to '{s}': {s}", .{
2210	info.file_name, info.link_name, @errorName(info.code),
2211	}),
2212	}));
2213	},
2214	.unable_to_create_file => |info| {
2215	eb.extra.items[note_i] = @backingInt(try eb.addErrorMessage(.{
2216	.msg = try eb.printString("unable to create file '{s}': {s}", .{
2217	info.file_name, @errorName(info.code),
2218	}),
2219	}));
2220	},
2221	.unsupported_file_type => |info| {
2222	eb.extra.items[note_i] = @backingInt(try eb.addErrorMessage(.{
2223	.msg = try eb.printString("file '{s}' has unsupported type '{c}'", .{
2224	info.file_name, info.file_type,
2225	}),
2226	}));
2227	},
2228	}
2229	note_i += 1;
2230	}
2231
2232	return error.FetchFailed;
2233	}
2234
2235	test validate {
2236	const gpa = std.testing.allocator;
2237	var arena_instance = std.heap.ArenaAllocator.init(gpa);
2238	defer arena_instance.deinit();
2239	const arena = arena_instance.allocator();
2240
2241	// fill UnpackResult with errors
2242	var res: UnpackResult = .{};
2243	try res.allocErrors(arena, 4, "unable to unpack");
2244	try std.testing.expectEqual(0, res.errors_count);
2245	res.unableToCreateFile("dir1/file1", error.File1);
2246	res.unableToCreateSymLink("dir2/file2", "filename", error.SymlinkError);
2247	res.unableToCreateFile("dir1/file3", error.File3);
2248	res.unsupportedFileType("dir2/file4", 'x');
2249	try std.testing.expectEqual(4, res.errors_count);
2250
2251	// create filter, includes dir2, excludes dir1
2252	var filter: Filter = .{};
2253	try filter.include_paths.put(arena, "dir2", {});
2254
2255	// init Fetch
2256	var fetch: Fetch = undefined;
2257	fetch.parent_manifest_ast = null;
2258	fetch.location_tok = 0;
2259	try fetch.error_bundle.init(gpa);
2260	defer fetch.error_bundle.deinit();
2261
2262	// validate errors with filter
2263	try std.testing.expectError(error.FetchFailed, res.validate(&fetch, filter));
2264
2265	// output errors to string
2266	var errors = try fetch.error_bundle.toOwnedBundle("");
2267	defer errors.deinit(gpa);
2268	var aw: Io.Writer.Allocating = .init(gpa);
2269	defer aw.deinit();
2270	try errors.renderToWriter(.{}, &aw.writer);
2271	try std.testing.expectEqualStrings(
2272	\\error: unable to unpack
2273	\\ note: unable to create symlink from 'dir2/file2' to 'filename': SymlinkError
2274	\\ note: file 'dir2/file4' has unsupported type 'x'
2275	\\
2276	, aw.written());
2277	}
2278	};
2279
2280	test {
2281	_ = Filter;
2282	_ = FileType;
2283	_ = UnpackResult;
2284	}