Commit 57df1871541a43ef86534e3998d17203d6614920 | zig

author	Andrew Kelley <andrew@ziglang.org> 2026-08-23 21:42:17-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2026-08-24 14:34:27-07:00
log	57df1871541a43ef86534e3998d17203d6614920
tree	f8c1208492411c6faa49c28608064950b414fe4d
parent	857d8ac650b3154bfd320911f8d8fd0bd7ed9ea7

WIP: rework std.Build.Cache

- implement std.Build functions:
  - `dependOnDirectoryMetadata`
  - `dependOnDirectoryContents`
  - `dependOnFileMetadata`
  - `dependOnFileContents`
- switch cache manifest to binary format. saves about 25% of file size,
  which is not a ton, but this is in the really hot path, so any amount
  we can reduce file system cache pressure is going to help a lot.
  - also means saving and loading is simpler - just copying bytes in and
    out of memory plus a tiny bit of validation.
- batch cache manifest memory allocations together. one big allocation
  for the entire manifest file contents, with the files hash map only
  tracking starting offsets for each entry, and the input files as a
  simple array list with the extra information needed for input files
  (so they can be lazily loaded only on cache miss)
- remove the manifest file size limit
- input files that have requested content to be loaded is loaded
  together into one big allocation rather than separately tracked.
- each file entry in the cache now tracks two additional bits:
  - kind: file or directory
  - tracking mode: contents or metadata
    - when tracking a directory, "contents" are considered to be the
      sorted list of file names contained in the directory (non recursive)
- rename "hit" to "check" and make it return enum instead of bool
- merge most "add...File" function variants together into one that has
  an "options bag" parameter.
- take advantage of async I/O when checking for cache hit to compute
  hashes concurrently. Care is taken to never cancel useful work, and
  never wait for unnecessary work to complete.

bonus:
- build.zig: use std.log for printing log messages

7 files changed, 626 insertions(+), 626 deletions(-)

build.zig
lib
- std
  - Build.zig
  - Build
    Cache.zig
    Configuration.zig
  - fs
    path.zig
src
- Zcu
  - PerThread.zig
test
- src
  - Cases.zig

build.zig+12-9

...	...	@@ -263,8 +263,8 @@ pub fn build(b: *std.Build) !void {
263	263	const opt_version_string = b.option([]const u8, "version-string", "Override Zig version string. Default is to find out with git.");
264	264	const version_slice = if (opt_version_string) \|version\| version else v: {
265	265	if (!std.process.can_spawn) {
266		std.debug.print("error: version info cannot be retrieved from git. Zig version must be provided using -Dversion-string\n", .{});
267		std.process.exit(1);
	266	std.log.info("version info can be provided explicitly via \"-Dversion-string\"", .{});
	267	std.process.fatal("version info cannot be retrieved from git", .{});
268	268	}
269	269
270	270	// Ensure git version changes get picked up.
...	...	@@ -310,8 +310,9 @@ pub fn build(b: *std.Build) !void {
310	310	0 => {
311	311	// Tagged release version (e.g. 0.10.0).
312	312	if (!mem.eql(u8, git_describe, version_string)) {
313		std.debug.print("Zig version '{s}' does not match Git tag '{s}'\n", .{ version_string, git_describe });
314		std.process.exit(1);
	313	std.process.fatal("zig version {q} does not match Git tag {q}", .{
	314	version_string, git_describe,
	315	});
315	316	}
316	317	break :v version_string;
317	318	},
...	...	@@ -324,13 +325,14 @@ pub fn build(b: *std.Build) !void {
324	325
325	326	const ancestor_ver = try std.SemanticVersion.parse(tagged_ancestor);
326	327	if (zig_version.order(ancestor_ver) != .gt) {
327		std.debug.print("Zig version '{f}' must be greater than tagged ancestor '{f}'\n", .{ zig_version, ancestor_ver });
328		std.process.exit(1);
	328	std.process.fatal("zig version {f} must be greater than tagged ancestor {qf}", .{
	329	zig_version, ancestor_ver,
	330	});
329	331	}
330	332
331	333	// Check that the commit hash is prefixed with a 'g' (a Git convention).
332	334	if (commit_id.len < 1 or commit_id[0] != 'g') {
333		std.debug.print("Unexpected `git describe` output: {s}\n", .{git_describe});
	335	std.log.warn("unexpected \"git describe\" output: {s}", .{git_describe});
334	336	break :v version_string;
335	337	}
336	338
...	...	@@ -338,7 +340,7 @@ pub fn build(b: *std.Build) !void {
338	340	break :v b.fmt("{s}-dev.{s}+{s}", .{ version_string, commit_height, commit_id[1..] });
339	341	},
340	342	else => {
341		std.debug.print("Unexpected `git describe` output: {s}\n", .{git_describe});
	343	std.log.warn("unexpected \"git describe\" output: {s}", .{git_describe});
342	344	break :v version_string;
343	345	},
344	346	}
...	...	@@ -354,7 +356,8 @@ pub fn build(b: *std.Build) !void {
354	356	const file_contents = cwd.readFileAlloc(io, config_h_path, arena, .limited(max_config_h_bytes)) catch unreachable;
355	357	break :blk parseConfigH(b, file_contents);
356	358	} else {
357		std.log.warn("config.h could not be located automatically. Consider providing it explicitly via \"-Dconfig_h\"", .{});
	359	std.log.warn("config.h could not be located automatically", .{});
	360	std.log.info("config.h can be provided explicitly via \"-Dconfig_h\"", .{});
358	361	break :blk null;
359	362	}
360	363	};

lib/std/Build.zig+34-2

...	...	@@ -2636,6 +2636,35 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void {
2636	2636	/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
2637	2637	/// of `zig build` into a cache miss.
2638	2638	///
	2639	/// If any file is created, deleted, or renamed in this directory, leaving the
	2640	/// directory in a different state than last configuration with respect to
	2641	/// existence and naming of entries, the configure phase will be repeated.
	2642	///
	2643	/// Only a subset of `LazyPath` are supported:
	2644	/// - Relative to cwd
	2645	/// - Relative to any package root
	2646	/// - Relative to zig cache or zig installation
	2647	///
	2648	/// If the directory would be inside one of the search prefixes, then the dependency
	2649	/// cannot be tracked; `Graph.poisonCache` must be used instead.
	2650	///
	2651	/// Not recursive.
	2652	pub fn dependOnDirectoryContents(b: *Build, lazy_path: LazyPath) void {
	2653	validateConfigureDependency(lazy_path);
	2654	const graph = b.graph;
	2655	graph.configure_dependencies.append(graph.arena, .{
	2656	.lazy_path = lazy_path.dupe(graph),
	2657	.is_directory = true,
	2658	.metadata_only = false,
	2659	}) catch @panic("OOM");
	2660	}
	2661
	2662	/// Indicates that the build.zig logic depends on a particular directory's last
	2663	/// modification date.
	2664	///
	2665	/// This is an alternative to `Graph.poisonCache` that avoids making every invocation
	2666	/// of `zig build` into a cache miss.
	2667	///
2639	2668	/// If any file is created, deleted, or renamed in this directory, the
2640	2669	/// configure phase will be repeated.
2641	2670	///
...	...	@@ -2646,12 +2675,15 @@ pub fn dependOnFileMetadata(b: *Build, lazy_path: LazyPath) void {
2646	2675	///
2647	2676	/// If the directory would be inside one of the search prefixes, then the dependency
2648	2677	/// cannot be tracked; `Graph.poisonCache` must be used instead.
2649		pub fn dependOnDirectory(b: *Build, lazy_path: LazyPath) void {
	2678	///
	2679	/// Not recursive.
	2680	pub fn dependOnDirectoryMetadata(b: *Build, lazy_path: LazyPath) void {
2650	2681	validateConfigureDependency(lazy_path);
2651	2682	const graph = b.graph;
2652	2683	graph.configure_dependencies.append(graph.arena, .{
2653	2684	.lazy_path = lazy_path.dupe(graph),
2654		.mode = .directory,
	2685	.is_directory = true,
	2686	.metadata_only = true,
2655	2687	}) catch @panic("OOM");
2656	2688	}
2657	2689

lib/std/Build/Cache.zig+565-605

...	...	@@ -62,7 +62,7 @@ pub const PrefixedPath = struct {
62	62	sub_path: []const u8,
63	63
64	64	fn eql(a: PrefixedPath, b: PrefixedPath) bool {
65		return a.prefix == b.prefix and std.mem.eql(u8, a.sub_path, b.sub_path);
	65	return a.prefix == b.prefix and mem.eql(u8, a.sub_path, b.sub_path);
66	66	}
67	67
68	68	fn hash(pp: PrefixedPath) u32 {
...	...	@@ -118,7 +118,7 @@ fn getPrefixSubpath(gpa: Allocator, cwd: []const u8, prefix: []const u8, path: [
118	118	return error.NotASubPath;
119	119	}
120	120	const first_component = component_iterator.first();
121		if (first_component != null and std.mem.eql(u8, first_component.?.name, "..")) {
	121	if (first_component != null and mem.eql(u8, first_component.?.name, "..")) {
122	122	return error.NotASubPath;
123	123	}
124	124	return relative;
...	...	@@ -130,10 +130,6 @@ pub const hex_digest_len = bin_digest_len * 2;
130	130	pub const BinDigest = [bin_digest_len]u8;
131	131	pub const HexDigest = [hex_digest_len]u8;
132	132
133		/// This is currently just an arbitrary non-empty string that can't match another manifest line.
134		const manifest_header = "0";
135		pub const manifest_file_size_max = 100 * 1024 * 1024;
136
137	133	/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
138	134	/// provides enough collision resistance for the Manifest use cases, while being one of our
139	135	/// fastest options right now.
...	...	@@ -149,50 +145,6 @@ pub const hasher_init: Hasher = Hasher.init(&.{
149	145	0x77, 0xd6, 0xf0, 0x60,
150	146	});
151	147
152		pub const File = struct {
153		prefixed_path: PrefixedPath,
154		max_file_size: ?usize,
155		/// Populated if the user calls `addOpenedFile`.
156		/// The handle is not owned here.
157		handle: ?Io.File,
158		stat: Stat,
159		bin_digest: BinDigest,
160		contents: ?[]const u8,
161
162		pub const Stat = struct {
163		inode: Io.File.INode,
164		size: u64,
165		mtime: Io.Timestamp,
166
167		pub fn fromFs(fs_stat: Io.File.Stat) Stat {
168		return .{
169		.inode = fs_stat.inode,
170		.size = fs_stat.size,
171		.mtime = fs_stat.mtime,
172		};
173		}
174		};
175
176		pub fn deinit(self: *File, gpa: Allocator) void {
177		gpa.free(self.prefixed_path.sub_path);
178		if (self.contents) \|contents\| {
179		gpa.free(contents);
180		self.contents = null;
181		}
182		self.* = undefined;
183		}
184
185		pub fn updateMaxSize(file: *File, new_max_size: ?usize) void {
186		const new = new_max_size orelse return;
187		file.max_file_size = if (file.max_file_size) \|old\| @max(old, new) else new;
188		}
189
190		pub fn updateHandle(file: *File, new_handle: ?Io.File) void {
191		const handle = new_handle orelse return;
192		file.handle = handle;
193		}
194		};
195
196	148	pub const HashHelper = struct {
197	149	hasher: Hasher = hasher_init,
198	150
...	...	@@ -319,10 +271,15 @@ pub const Lock = struct {
319	271	}
320	272	};
321	273
	274	/// Format: a series of consecutive `Manifest.File`, followed by a final
	275	/// terminating zero byte to distinguish empty manifest file from manifest with
	276	/// zero files.
322	277	pub const Manifest = struct {
323	278	cache: *Cache,
324	279	/// Current state for incremental hashing.
325	280	hash: HashHelper,
	281	hex_digest: HexDigest,
	282	/// When this is null, `Manifest` is in "pre-check" phase. Otherwise it is in "post-check" phase.
326	283	manifest_file: ?Io.File,
327	284	manifest_dirty: bool,
328	285	/// Set this flag to true before calling hit() in order to indicate that
...	...	@@ -335,12 +292,140 @@ pub const Manifest = struct {
335	292	// order to obtain a problematic timestamp for the next call. Calls after that
336	293	// will then use the same timestamp, to avoid unnecessary filesystem writes.
337	294	want_refresh_timestamp: bool = true,
338		files: Files = .{},
339		hex_digest: HexDigest,
	295	/// Uses `Cache.gpa`.
	296	files: Files = .empty,
	297	/// Indexes line up with `files`, but only up until `hit` is called. Uses
	298	/// `Cache.gpa`.
	299	input_files: std.ArrayList(InputFile) = .empty,
340	300	diagnostic: Diagnostic = .none,
341	301	/// Keeps track of the last time we performed a file system write to observe
342	302	/// what time the file system thinks it is, according to its own granularity.
343	303	recent_problematic_timestamp: Io.Timestamp = .zero,
	304	/// The entire manifest file contents, except for the final terminating
	305	/// zero byte. However maintains always at least 1 unused capacity so the
	306	/// final terminating byte can be added without allocation. Uses
	307	/// `Cache.gpa`.
	308	contents: std.ArrayList(u8) = .empty,
	309	/// All contents from all `input_files` whose contents were requested,
	310	/// concatenated. Total byte size will be less than `max_input_content_len`
	311	/// otherwise an error is returned.
	312	all_input_content: std.ArrayList(u8) = .empty,
	313	max_input_content_len: usize = std.math.maxInt(u32),
	314
	315	pub const Files = std.array_hash_map.Custom(File.Offset, void, File.HashContext, false);
	316
	317	/// Source files whose prefix and relative path are included when computing
	318	/// the cache manifest digest. It's the information needed to lazily hash
	319	/// the input files only when a cache miss occurs.
	320	///
	321	/// `File.prefix`, `File.path`, and `File.mode` will be always populated,
	322	/// but the other fields of `File` will be populated depending on the
	323	/// fields of `InputFile`.
	324	pub const InputFile = struct {
	325	request_handle: bool,
	326	have_handle: bool,
	327	/// Determines whether `File.size`, `File.inode`, and `File.mtime` are populated.
	328	have_stat: bool,
	329	/// Determines whether `File.digest` is populated.
	330	have_digest: bool,
	331	contents: enum (usize) {
	332	requested = std.math.maxInt(u32) - 1,
	333	not_requested = std.math.maxInt(u32),
	334	/// Byte offset index into `Manifest.all_input_content`.
	335	_,
	336	},
	337	/// `have_handle` determines whether this is populated.
	338	handle: Io.File,
	339
	340	/// Index into `Manifest.input_files`.
	341	pub const Index = enum(u32) {
	342	_,
	343	};
	344	};
	345
	346	/// The data per tracked input file that is stored in the manifest file.
	347	pub const File = extern struct {
	348	size: u64,
	349	inode: u64,
	350	digest: BinDigest,
	351	/// Nanoseconds.
	352	mtime: i64,
	353	/// Starting with this field and continuing into the path, excluding the null byte,
	354	/// is the string that is hashed for the manifest digest.
	355	flags: Flags,
	356	/// Terminated by zero byte, then followed by padding until 8-byte aligned.
	357	path_start: [0]u8,
	358
	359	pub const Flags = packed struct (u8) {
	360	is_directory: bool,
	361	metadata_only: bool,
	362	prefix: u6,
	363	};
	364
	365	/// Byte index within `Manifest.contents` where the entry starts.
	366	pub const Offset = enum(u32) {
	367	_,
	368
	369	pub fn get(offset: Offset, m: const Manifest) File {
	370	return @ptrCast(m.contents.items[@backingInt(offset)..][0..@sizeOf(File)]);
	371	}
	372
	373	pub fn getFallible(offset: Offset, m: const Manifest) error{EndOfStream}!File {
	374	if (@backingInt(offset) + @sizeOf(File) >= m.contents.len) return error.EndOfStream;
	375	return get(offset, m);
	376	}
	377	};
	378
	379	pub const HashContext = struct {
	380	manifest: *const Manifest,
	381
	382	pub fn hash(this: @This(), off: Offset) u32 {
	383	const file = off.get(this.manifest);
	384	return @truncate(std.hash.Wyhash.hash(file.prefix, file.path()));
	385	}
	386
	387	pub fn eql(this: @This(), a_off: Offset, b_off: Offset, b_index: usize) bool {
	388	_ = b_index;
	389	const a = a_off.get(this.manifest);
	390	const b = b_off.get(this.manifest);
	391	return a.prefix == b.prefix and mem.eql(u8, a.path(), b.path());
	392	}
	393	};
	394
	395
	396	pub fn path(file: *const File) [:0]const u8 {
	397	return pathFallible(file) catch unreachable;
	398	}
	399
	400	pub fn pathFallible(file: *const File) error{EndOfStream}![:0]const u8 {
	401	const ptr: [*]u8 = &file.path_start;
	402	const len = mem.findScalar(u8, ptr, 0) orelse return error.EndOfStream;
	403	return ptr[0..len :0];
	404	}
	405
	406	fn manifestDigestHash(file: const File, hasher: Hasher) void {
	407	const path_ptr: [*]u8 = &file.path_start;
	408	const path_len = mem.findScalar(u8, path_ptr, 0).?;
	409	comptime assert(@offsetOf(File, "path_start") - @offsetOf(File, "flags") == 1);
	410	// Includes flags and sentinel.
	411	const hash_string = (path_ptr - 1)[0..path_len + 2];
	412	hasher.update(hash_string);
	413	}
	414
	415	fn setStat(file: File, m: Manifest, stat: Stat) Io.Cancelable!void {
	416	file.size = stat.size;
	417	file.inode = stat.inode;
	418	file.mtime = stat.mtime;
	419
	420	if (try m.isProblematicTimestamp(stat.mtime)) {
	421	// The actual file has an unreliable timestamp; force it to be hashed.
	422	file.stat.mtime = 0;
	423	file.stat.inode = 0;
	424	}
	425	}
	426
	427	};
	428
344	429
345	430	pub const Diagnostic = union(enum) {
346	431	none,
...	...	@@ -358,110 +443,118 @@ pub const Manifest = struct {
358	443	};
359	444	};
360	445
361		pub const Files = std.array_hash_map.Custom(File, void, FilesContext, false);
362
363		pub const FilesContext = struct {
364		pub fn hash(fc: FilesContext, file: File) u32 {
365		_ = fc;
366		return file.prefixed_path.hash();
367		}
368
369		pub fn eql(fc: FilesContext, a: File, b: File, b_index: usize) bool {
370		_ = fc;
371		_ = b_index;
372		return a.prefixed_path.eql(b.prefixed_path);
373		}
	446	pub const Stat = struct {
	447	size: u64,
	448	inode: Io.File.INode,
	449	mtime: Io.Timestamp,
374	450	};
375	451
376		const FilesAdapter = struct {
377		pub fn eql(context: @This(), a: PrefixedPath, b: File, b_index: usize) bool {
378		_ = context;
379		_ = b_index;
380		return a.eql(b.prefixed_path);
381		}
	452	pub const AddInputFileOptions = struct {
	453	handle: ?Io.File = null,
	454	stat: ?Stat = null,
	455	request_handle: bool = false,
	456	request_contents: bool = false,
	457	is_directory: bool = false,
	458	metadata_only: bool = false,
382	459
383		pub fn hash(context: @This(), key: PrefixedPath) u32 {
384		_ = context;
385		return key.hash();
386		}
387	460	};
388	461
	462	pub const AddInputFileError = error {
	463	/// The same file path has been added to the cache manifest both as a
	464	/// directory and as a normal file, making the intended caching
	465	/// behavior ambiguous.
	466	IsDirectoryAmbiguous,
	467	} \|\| Allocator.Error;
	468
389	469	/// Add a file as a dependency of process being cached. When `hit` is
390	470	/// called, the file's contents will be checked to ensure that it matches
391	471	/// the contents from previous times.
392	472	///
393		/// Max file size will be used to determine the amount of space the file contents
394		/// are allowed to take up in memory. If max_file_size is null, then the contents
395		/// will not be loaded into memory.
396		///
397		/// Returns the index of the entry in the `files` array list. You can use it
398		/// to access the contents of the file after calling `hit()` like so:
399		///
400		/// ```
401		/// var file_contents = cache_hash.files.keys()[file_index].contents.?;
402		/// ```
403		pub fn addFilePath(m: *Manifest, file_path: Path, max_file_size: ?usize) !usize {
404		return addOpenedFile(m, file_path, null, max_file_size);
405		}
406
407		/// Same as `addFilePath` except the file has already been opened.
408		pub fn addOpenedFile(m: *Manifest, path: Path, handle: ?Io.File, max_file_size: ?usize) !usize {
	473	/// The contents of the input file may be requested and subsequently
	474	/// obtained via methods of the returned `InputFile.Index` after calling
	475	/// `hit`.
	476	pub fn addInputFile(m: *Manifest, path: Path, options: AddInputFileOptions) Allocator.Error!InputFile.Index {
409	477	const gpa = m.cache.gpa;
410	478	try m.files.ensureUnusedCapacity(gpa, 1);
411		const resolved_path = try std.fs.path.resolve(gpa, &.{
412		path.root_dir.path orelse ".",
413		path.subPathOrDot(),
414		});
415		errdefer gpa.free(resolved_path);
416		const prefixed_path = try m.cache.findPrefixResolved(resolved_path);
417		return addFileInner(m, prefixed_path, handle, max_file_size);
418		}
419
420		fn addFileInner(self: *Manifest, prefixed_path: PrefixedPath, handle: ?Io.File, max_file_size: ?usize) usize {
421		const gop = self.files.getOrPutAssumeCapacityAdapted(prefixed_path, FilesAdapter{});
422		if (gop.found_existing) {
423		self.cache.gpa.free(prefixed_path.sub_path);
424		gop.key_ptr.updateMaxSize(max_file_size);
425		gop.key_ptr.updateHandle(handle);
426		return gop.index;
427		}
428		gop.key_ptr.* = .{
429		.prefixed_path = prefixed_path,
430		.contents = null,
431		.max_file_size = max_file_size,
432		.stat = undefined,
433		.bin_digest = undefined,
434		.handle = handle,
435		};
	479	try m.input_files.ensureUnusedCapacity(gpa, 1);
436	480
437		self.hash.add(prefixed_path.prefix);
438		self.hash.addBytes(prefixed_path.sub_path);
	481	const prev_contents_len = m.contents.items.len;
	482	const header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)));
	483	errdefer m.contents.shrinkRetainingCapacity(prev_contents_len);
439	484
440		return gop.index;
441		}
	485	header.* = .{
	486	.flags = .{
	487	.prefix = try m.cache.findAppendPrefixedPath(&m.contents, path),
	488	.is_directory = options.is_directory,
	489	.metadata_only = options.metadata_only,
	490	},
	491	.size = undefined,
	492	.inode = undefined,
	493	.mtime = undefined,
	494	.digest = undefined,
	495	};
	496	assert(m.contents.items.len % @alignOf(File) == 0);
442	497
443		pub fn addOptionalFilePath(self: *Manifest, optional_file_path: ?Path) !void {
444		self.hash.add(optional_file_path != null);
445		const file_path = optional_file_path orelse return;
446		_ = try self.addFilePath(file_path, null);
	498	const gop = try m.files.getOrPutAssumeCapacityContext(@fromBackingInt(prev_contents_len), .{
	499	.manifest = m,
	500	});
	501	if (gop.found_existing) {
	502	m.contents.shrinkRetainingCapacity(prev_contents_len);
	503	const existing_input_file = &m.input_files.items[gop.index];
	504	if (options.handle) \|handle\| {
	505	existing_input_file.handle = handle;
	506	existing_input_file.have_handle = true;
	507	}
	508	if (options.request_contents) switch (existing_input_file.contents) {
	509	.requested, .not_requested => existing_input_file.contents = .requested,
	510	_ => {},
	511	};
	512	const existing_header = &m.files.keys()[gop.index];
	513	if (options.stat) \|stat\| {
	514	existing_input_file.have_stat = true;
	515	existing_header.size = stat.size;
	516	existing_header.inode = stat.inode;
	517	existing_header.mtime = stat.mtime;
	518	}
	519	if (existing_header.flags.is_directory != options.is_directory)
	520	return error.IsDirectoryAmbiguous;
	521	if (!options.metadata_only)
	522	existing_header.flags.metadata_only = false;
	523	} else {
	524	m.input_files.appendAssumeCapacity(.{
	525	.request_handle = options.request_handle,
	526	.have_handle = options.handle != null,
	527	.handle = if (options.handle) \|handle\| handle else undefined,
	528	.contents = if (options.request_contents) .requested else .not_requested,
	529	.have_digest = false,
	530	.have_stat = options.stat != null,
	531	});
	532	assert(m.input_files.items.len - 1 == gop.index);
	533	if (options.stat) \|stat\| {
	534	header.size = stat.size;
	535	header.inode = stat.inode;
	536	header.mtime = stat.mtime;
	537	}
	538	}
	539	return @fromBackingInt(gop.index);
447	540	}
448	541
449		pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
450		assert(self.manifest_file == null);
451		return self.addDepFileMaybePost(dir, dep_file_sub_path);
	542	pub fn addInputFileOptional(m: *Manifest, opt_path: ?Path, options: AddInputFileOptions) Allocator.Error!void {
	543	m.hash.add(opt_path != null);
	544	_ = try addInputFile(m, opt_path orelse return, options);
452	545	}
453	546
454		pub const HitError = error{
	547	pub const CheckError = error{
455	548	/// Unable to check the cache for a reason that has been recorded into
456	549	/// the `diagnostic` field.
457	550	CacheCheckFailed,
458	551	/// A cache manifest file exists however it could not be parsed.
459	552	InvalidFormat,
460		OutOfMemory,
461		Canceled,
462		};
	553	} \|\| Allocator.Error \|\| Io.Cancelable;
	554
	555	pub const CheckStatus = enum { hit, miss };
463	556
464		/// Check the cache to see if the input exists in it. If it exists, returns `true`.
	557	/// Check the cache to see if the input exists in it.
465	558	/// A hex encoding of its hash is available by calling `final`.
466	559	///
467	560	/// This function will also acquire an exclusive lock to the manifest file. This means
...	...	@@ -473,50 +566,48 @@ pub const Manifest = struct {
473	566	/// The lock on the manifest file is released when `deinit` is called. As another
474	567	/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
475	568	/// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
476		pub fn hit(man: *Manifest, parent_progress_node: std.Progress.Node) HitError!bool {
	569	pub fn check(man: *Manifest, parent_progress_node: std.Progress.Node) CheckError!CheckStatus {
477	570	const node = parent_progress_node.start("Reusing Cache Artifacts", 0);
478	571	defer node.end();
479		return hitInner(man);
	572	return checkProgressless(man);
480	573	}
481	574
482		pub fn hitInner(self: *Manifest) HitError!bool {
483		assert(self.manifest_file == null);
	575	pub fn checkProgressless(man: *Manifest) CheckError!CheckStatus {
	576	assert(man.manifest_file == null);
484	577
485		self.diagnostic = .none;
	578	for (man.files.keys()[0..man.input_files.items.len]) \|file_off\| {
	579	file_off.get(man).manifestDigestHash(&man.hash.hasher);
	580	}
486	581
487		const ext = ".txt";
488		var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined;
	582	man.diagnostic = .none;
489	583
490	584	var bin_digest: BinDigest = undefined;
491		self.hash.hasher.final(&bin_digest);
492
493		self.hex_digest = binToHex(bin_digest);
	585	man.hash.hasher.final(&bin_digest);
	586	man.hex_digest = binToHex(bin_digest);
494	587
495		@memcpy(manifest_file_path[0..self.hex_digest.len], &self.hex_digest);
496		manifest_file_path[hex_digest_len..][0..ext.len].* = ext.*;
497
498		const io = self.cache.io;
	588	const manifest_file_path = &man.hex_digest;
	589	const io = man.cache.io;
499	590
500	591	// We'll try to open the cache with an exclusive lock, but if that would block
501	592	// and `want_shared_lock` is set, a shared lock might be sufficient, so we'll
502	593	// open with a shared lock instead.
503	594	while (true) {
504		if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{
	595	if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{
505	596	.read = true,
506	597	.truncate = false,
507	598	.lock = .exclusive,
508		.lock_nonblocking = self.want_shared_lock,
	599	.lock_nonblocking = man.want_shared_lock,
509	600	})) \|manifest_file\| {
510		self.manifest_file = manifest_file;
511		self.have_exclusive_lock = true;
	601	man.manifest_file = manifest_file;
	602	man.have_exclusive_lock = true;
512	603	break;
513	604	} else \|err\| switch (err) {
514	605	error.WouldBlock => {
515		self.manifest_file = self.cache.manifest_dir.openFile(io, &manifest_file_path, .{
	606	man.manifest_file = man.cache.manifest_dir.openFile(io, manifest_file_path, .{
516	607	.mode = .read_write,
517	608	.lock = .shared,
518	609	}) catch \|e\| {
519		self.diagnostic = .{ .manifest_create = e };
	610	man.diagnostic = .{ .manifest_create = e };
520	611	return error.CacheCheckFailed;
521	612	};
522	613	break;
...	...	@@ -532,315 +623,262 @@ pub const Manifest = struct {
532	623	// failure was a race, or ENOENT, indicating deletion of
533	624	// the directory of our open handle.
534	625	if (!builtin.os.tag.isDarwin()) {
535		self.diagnostic = .{ .manifest_create = error.FileNotFound };
	626	man.diagnostic = .{ .manifest_create = error.FileNotFound };
536	627	return error.CacheCheckFailed;
537	628	}
538	629
539		if (self.cache.manifest_dir.createFile(io, &manifest_file_path, .{
	630	if (man.cache.manifest_dir.createFile(io, manifest_file_path, .{
540	631	.read = true,
541	632	.truncate = false,
542	633	.lock = .exclusive,
543		.lock_nonblocking = self.want_shared_lock,
	634	.lock_nonblocking = man.want_shared_lock,
544	635	.exclusive = true,
545	636	})) \|manifest_file\| {
546		self.manifest_file = manifest_file;
547		self.have_exclusive_lock = true;
	637	man.manifest_file = manifest_file;
	638	man.have_exclusive_lock = true;
548	639	break;
549	640	} else \|excl_err\| switch (excl_err) {
550	641	error.WouldBlock, error.PathAlreadyExists => continue,
551	642	error.FileNotFound => {
552		self.diagnostic = .{ .manifest_create = error.FileNotFound };
	643	man.diagnostic = .{ .manifest_create = error.FileNotFound };
553	644	return error.CacheCheckFailed;
554	645	},
555	646	error.Canceled => \|e\| return e,
556	647	else => \|e\| {
557		self.diagnostic = .{ .manifest_create = e };
	648	man.diagnostic = .{ .manifest_create = e };
558	649	return error.CacheCheckFailed;
559	650	},
560	651	}
561	652	},
562	653	error.Canceled => \|e\| return e,
563	654	else => \|e\| {
564		self.diagnostic = .{ .manifest_create = e };
	655	man.diagnostic = .{ .manifest_create = e };
565	656	return error.CacheCheckFailed;
566	657	},
567	658	}
568	659	}
569	660
570		self.want_refresh_timestamp = true;
571
572		const input_file_count = self.files.entries.len;
	661	man.want_refresh_timestamp = true;
573	662
574	663	// We're going to construct a second hash. Its input will begin with the digest we've
575	664	// already computed (`bin_digest`), and then it'll have the digests of each input file,
576	665	// including "post" files (see `addFilePost`). If this is a hit, we learn the set of "post"
577	666	// files from the manifest on disk. If this is a miss, we'll learn those from future calls
578		// to `addFilePost` etc. As such, the state of `self.hash.hasher` after this function
	667	// to `addFilePost` etc. As such, the state of `man.hash.hasher` after this function
579	668	// depends on whether this is a hit or a miss.
580	669	//
581		// If we return `true` indicating a cache hit, then `self.hash.hasher` must already include
	670	// If we return `CacheStatus.hit`, then `man.hash.hasher` must already include
582	671	// the digests of the "post" files, so the caller can call `final`. Otherwise, on a cache
583		// miss, `self.hash.hasher` will include the digests of all non-"post" files -- that is,
	672	// miss, `man.hash.hasher` will include the digests of all non-"post" files -- that is,
584	673	// the ones we've already been told about. The rest will be discovered through calls to
585	674	// `addFilePost` etc, which will update the hasher. After all files are added, the user can
586	675	// use `final`, and will at some point `writeManifest` the file list to disk.
587	676
588		self.hash.hasher = hasher_init;
589		self.hash.hasher.update(&bin_digest);
	677	man.hash.hasher = hasher_init;
	678	man.hash.hasher.update(&bin_digest);
590	679
591	680	hit: {
592		const file_digests_populated: usize = digests: {
593		switch (try self.hitWithCurrentLock()) {
	681	digests: {
	682	switch (try man.checkLocked()) {
594	683	.hit => break :hit,
595		.miss => \|m\| if (!try self.upgradeToExclusiveLock()) {
596		break :digests m.file_digests_populated;
597		},
	684	.miss => if (!try man.upgradeToExclusiveLock()) break :digests,
598	685	}
599	686	// We've just had a miss with the shared lock, and upgraded to an exclusive lock. Someone
600	687	// else might have modified the digest, so we need to check again before deciding to miss.
601		// Before trying again, we must reset `self.hash.hasher` and `self.files`.
	688	// Before trying again, we must reset `man.hash.hasher` and `man.files`.
602	689	// This is basically just the first half of `unhit`.
603		self.hash.hasher = hasher_init;
604		self.hash.hasher.update(&bin_digest);
605		while (self.files.count() != input_file_count) {
606		var file = self.files.pop().?;
607		file.key.deinit(self.cache.gpa);
608		}
609		switch (try self.hitWithCurrentLock()) {
	690	man.hash.hasher = hasher_init;
	691	man.hash.hasher.update(&bin_digest);
	692	man.shrinkFilesToInput();
	693	switch (try man.checkLocked()) {
610	694	.hit => break :hit,
611		.miss => \|m\| break :digests m.file_digests_populated,
612		}
613		};
614
615		// This is a guaranteed cache miss. We're almost ready to return `false`, but there's a
616		// little bookkeeping to do first. The first `file_digests_populated` entries in `files`
617		// have their `bin_digest` populated; there may be some left in `input_file_count` which
618		// we'll need to populate ourselves. Other than that, this is basically `unhit`.
619		self.manifest_dirty = true;
620		self.hash.hasher = hasher_init;
621		self.hash.hasher.update(&bin_digest);
622		while (self.files.count() != input_file_count) {
623		var file = self.files.pop().?;
624		file.key.deinit(self.cache.gpa);
625		}
626		for (self.files.keys(), 0..) \|*file, idx\| {
627		if (idx < file_digests_populated) {
628		// `bin_digest` is already populated by `hitWithCurrentLock`, so we can use it directly.
629		self.hash.hasher.update(&file.bin_digest);
630		} else {
631		self.populateFileHash(file) catch \|err\| {
632		self.diagnostic = .{ .file_hash = .{
633		.file_index = idx,
634		.err = err,
635		} };
636		return error.CacheCheckFailed;
637		};
	695	.miss => break :digests,
638	696	}
639	697	}
640		return false;
	698
	699	// Cache miss. `checkLocked` guarantees that all input files have their digests populated
	700	// unless it returns an error.
	701	man.manifest_dirty = true;
	702	// All input file digests are already populated by `checkLocked`, so we can call `unhit` directly.
	703	unhit(man, &bin_digest);
	704	return .miss;
641	705	}
642	706
643		if (self.want_shared_lock) {
644		self.downgradeToSharedLock() catch \|err\| {
645		self.diagnostic = .{ .manifest_lock = err };
	707	if (man.want_shared_lock) {
	708	man.downgradeToSharedLock() catch \|err\| {
	709	man.diagnostic = .{ .manifest_lock = err };
646	710	return error.CacheCheckFailed;
647	711	};
648	712	}
649	713
650		return true;
	714	return .hit;
	715	}
	716
	717	fn shrinkFilesToInput(m: *Manifest) void {
	718	if (m.files.count() <= m.input_files.items.len) return;
	719	const off = m.files.keys()[m.input_files.items.len];
	720	m.contents.shrinkRetainingCapacity(@backingInt(off));
	721	assert(m.contents.len % @alignOf(File) == 0);
	722	m.files.shrinkRetainingCapacity(m.input_files.items.len);
651	723	}
652	724
653	725	/// Assumes that `self.hash.hasher` has been updated only with the original digest and that
654	726	/// `self.files` contains only the original input files.
655		fn hitWithCurrentLock(self: *Manifest) HitError!union(enum) {
656		hit,
657		miss: struct {
658		file_digests_populated: usize,
659		},
660		} {
661		const gpa = self.cache.gpa;
662		const io = self.cache.io;
663		const input_file_count = self.files.entries.len;
664		var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded
665		var manifest_reader = self.manifest_file.?.reader(io, &tiny_buffer); // Reads positionally from zero.
666		const limit: std.Io.Limit = .limited(manifest_file_size_max);
667		const file_contents = manifest_reader.interface.allocRemaining(gpa, limit) catch \|err\| switch (err) {
	727	fn checkLocked(m: *Manifest) CheckError!CheckStatus {
	728	const gpa = m.cache.gpa;
	729	const io = m.cache.io;
	730
	731	var manifest_reader = m.manifest_file.?.reader(io, &.{}); // Reads positionally from zero.
	732	m.contents.clearRetainingCapacity();
	733	manifest_reader.interface.appendRemainingUnlimited(gpa, &m.contents) catch \|err\| switch (err) {
668	734	error.OutOfMemory => \|e\| return e,
669		error.StreamTooLong => return error.OutOfMemory,
670	735	error.ReadFailed => {
671		self.diagnostic = .{ .manifest_read = manifest_reader.err.? };
	736	m.diagnostic = .{ .manifest_read = manifest_reader.err.? };
672	737	return error.CacheCheckFailed;
673	738	},
674	739	};
675		defer gpa.free(file_contents);
676
677		var any_file_changed = false;
678		var line_iter = mem.tokenizeScalar(u8, file_contents, '\n');
679		var idx: usize = 0;
680		const header_valid = valid: {
681		const line = line_iter.next() orelse break :valid false;
682		break :valid std.mem.eql(u8, line, manifest_header);
	740
	741	// Guess number of files based on manifest contents len to reduce allocations.
	742	try m.files.ensureUnusedCapacity(gpa, m.contents.len / (@sizeOf(File) + 32));
	743
	744	var file_index: usize = 0;
	745	var off: usize = 0;
	746
	747	// This group we always want to compute the hash digests, even on a cache miss.
	748	var input_group: Io.Group = .init;
	749	defer input_group.cancel(io);
	750
	751	// This group we would like to cancel as soon as a cache miss is discovered.
	752	const PostResult = union(enum) {
	753	checkFile: CheckFileResult,
683	754	};
684		if (!header_valid) {
685		return .{ .miss = .{ .file_digests_populated = 0 } };
	755	var post_select_buffer: [10]PostResult = undefined;
	756	var post_select: Io.Select(PostResult) = .init(&post_select_buffer);
	757	var post_select_remaining: usize = 0;
	758	defer post_select.cancel(io);
	759
	760	while (off + 1 < m.contents.len) {
	761	const file_off: File.Offset = @fromBackingInt(off);
	762	const file = try File.getFallible(file_off, m);
	763	if (file.flags.prefix >= m.cache.prefixes_len) return error.InvalidFormat;
	764	const path = try file.pathFallible();
	765	if (path.len == 0) return error.InvalidFormat;
	766
	767	if (file_index < m.input_files.items.len) {
	768	if (m.files.keys()[file_index] != file_off) return error.InvalidFormat;
	769
	770	input_group.async(io, checkFile, .{m.cache, file, path});
	771	} else {
	772	try m.files.put(gpa, file_off);
	773
	774	post_select.async(.checkFile, checkFile, .{m.cache, file, path});
	775	post_select_remaining += 1;
	776	}
	777
	778	file_index += 1;
	779	off += @sizeOf(File) + path.len + 1;
686	780	}
687		while (line_iter.next()) \|line\| {
688		defer idx += 1;
689
690		var iter = mem.tokenizeScalar(u8, line, ' ');
691		const size = iter.next() orelse return error.InvalidFormat;
692		const inode = iter.next() orelse return error.InvalidFormat;
693		const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
694		const digest_str = iter.next() orelse return error.InvalidFormat;
695		const prefix_str = iter.next() orelse return error.InvalidFormat;
696		const file_path = iter.rest();
697
698		const stat_size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
699		const stat_inode = fmt.parseInt(Io.File.INode, inode, 10) catch return error.InvalidFormat;
700		const stat_mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
701		const file_bin_digest = b: {
702		if (digest_str.len != hex_digest_len) return error.InvalidFormat;
703		var bd: BinDigest = undefined;
704		_ = fmt.hexToBytes(&bd, digest_str) catch return error.InvalidFormat;
705		break :b bd;
706		};
707	781
708		const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
709		if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
710
711		if (file_path.len == 0) return error.InvalidFormat;
712
713		const cache_hash_file = f: {
714		const prefixed_path: PrefixedPath = .{
715		.prefix = prefix,
716		.sub_path = file_path, // expires with file_contents
717		};
718		if (idx < input_file_count) {
719		const file = &self.files.keys()[idx];
720		if (!file.prefixed_path.eql(prefixed_path))
721		return error.InvalidFormat;
722
723		file.stat = .{
724		.size = stat_size,
725		.inode = stat_inode,
726		.mtime = .{ .nanoseconds = stat_mtime },
727		};
728		file.bin_digest = file_bin_digest;
729		break :f file;
730		}
731		const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
732		errdefer _ = self.files.pop();
733		if (!gop.found_existing) {
734		gop.key_ptr.* = .{
735		.prefixed_path = .{
736		.prefix = prefix,
737		.sub_path = try gpa.dupe(u8, file_path),
738		},
739		.contents = null,
740		.max_file_size = null,
741		.handle = null,
742		.stat = .{
743		.size = stat_size,
744		.inode = stat_inode,
745		.mtime = .{ .nanoseconds = stat_mtime },
746		},
747		.bin_digest = file_bin_digest,
748		};
749		}
750		break :f gop.key_ptr;
751		};
	782	// Final terminating zero byte to distinguish empty manifest file from
	783	// manifest with zero files.
	784	const file_valid = off + 1 == m.contents.len and m.contents[off] == 0;
	785	if (!file_valid or file_index < m.input_files.items.len) {
	786	try input_group.await(io);
	787	return .miss;
	788	}
752	789
753		const pp = cache_hash_file.prefixed_path;
754		const dir = self.cache.prefixes()[pp.prefix].handle;
755		const this_file = dir.openFile(io, pp.sub_path, .{ .mode = .read_only }) catch \|err\| switch (err) {
756		error.FileNotFound => {
757		// Every digest before this one has been populated successfully.
758		return .{ .miss = .{ .file_digests_populated = idx } };
759		},
760		error.Canceled => \|e\| return e,
761		else => \|e\| {
762		self.diagnostic = .{ .file_open = .{
763		.file_index = idx,
764		.err = e,
765		} };
766		return error.CacheCheckFailed;
	790	// Don't track the trailing zero byte in contents.
	791	m.contents.len -= 1;
	792
	793	var post_await_buffer: [10]PostResult = undefined;
	794	while (post_select_remaining > 0) {
	795	const n = try post_select.awaitMany(&post_await_buffer, 1);
	796	post_select_remaining -= n;
	797	for (post_await_buffer[0..n]) \|u\| switch (u) {
	798	.checkFile => \|result\| switch (result) {
	799	.hit => continue,
	800	.miss => {
	801	post_select.cancelDiscard();
	802	try input_group.await(io);
	803	return .miss;
	804	},
	805	.fail => \|diagnostic\| {
	806	m.diagnostic = diagnostic;
	807	return error.CacheCheckFailed;
	808	},
767	809	},
768	810	};
769		defer this_file.close(io);
	811	}
770	812
771		const actual_stat = this_file.stat(io) catch \|err\| {
772		self.diagnostic = .{ .file_stat = .{
773		.file_index = idx,
774		.err = err,
775		} };
776		return error.CacheCheckFailed;
777		};
778		const size_match = actual_stat.size == cache_hash_file.stat.size;
779		const mtime_match = actual_stat.mtime.nanoseconds == cache_hash_file.stat.mtime.nanoseconds;
780		const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
781
782		if (!size_match or !mtime_match or !inode_match) {
783		cache_hash_file.stat = .{
784		.size = actual_stat.size,
785		.mtime = actual_stat.mtime,
786		.inode = actual_stat.inode,
787		};
788
789		if (try self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
790		// The actual file has an unreliable timestamp, force it to be hashed
791		cache_hash_file.stat.mtime = .zero;
792		cache_hash_file.stat.inode = 0;
793		}
	813	try input_group.await(io);
794	814
795		var actual_digest: BinDigest = undefined;
796		hashFile(io, this_file, &actual_digest) catch \|err\| {
797		self.diagnostic = .{ .file_read = .{
798		.file_index = idx,
799		.err = err,
800		} };
801		return error.CacheCheckFailed;
802		};
	815	for (m.files.keys()) \|file_off\| {
	816	m.hash.hasher.update(&file_off.get(m).digest);
	817	}
803	818
804		if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
805		cache_hash_file.bin_digest = actual_digest;
806		// keep going until we have the input file digests
807		any_file_changed = true;
808		}
809		}
	819	return .hit;
	820	}
810	821
811		if (!any_file_changed) {
812		self.hash.hasher.update(&cache_hash_file.bin_digest);
813		}
814		}
	822	const CheckFileResult = union(enum) {
	823	hit,
	824	miss,
	825	fail: Diagnostic,
	826	};
815	827
816		// If the manifest was somehow missing one of our input files, or if any file hash has changed,
817		// then this is a cache miss. However, we have successfully populated some or all of the file
818		// digests.
819		if (any_file_changed or idx < input_file_count) {
820		return .{ .miss = .{ .file_digests_populated = idx } };
	828	/// Runs concurrently with other `checkFile`.
	829	fn checkFile(cache: const Cache, file: File, file_path: [:0]const u8) Io.Cancelable!CheckFileResult {
	830	const io = cache.io;
	831	const dir = cache.prefixes()[file.flags.prefix].handle;
	832
	833	const this_file = dir.openFile(io, file_path, .{ .mode = .read_only }) catch \|err\| switch (err) {
	834	error.FileNotFound => return .miss,
	835	error.Canceled => \|e\| return e,
	836	else => \|e\| return .{ .fail = .{ .file_open = .{
	837	.file_index = file_index,
	838	.err = e,
	839	} }},
	840	};
	841	defer this_file.close(io);
	842
	843	const actual_stat = this_file.stat(io) catch \|err\| return .{ .fail = .{ .file_stat = .{
	844	.file_index = file_index,
	845	.err = err,
	846	} }};
	847	const size_match = actual_stat.size == file.size;
	848	const mtime_match = actual_stat.mtime.nanoseconds == file.mtime;
	849	const inode_match = actual_stat.inode == file.inode;
	850
	851	if (!size_match or !mtime_match or !inode_match) {
	852	try file.setStat(actual_stat);
	853
	854	var actual_digest: BinDigest = undefined;
	855	hashFile(io, this_file, &actual_digest) catch \|err\| return .{ .fail = .{ .file_read = .{
	856	.file_index = file_index,
	857	.err = err,
	858	} }};
	859
	860	if (!mem.eql(u8, &file.digest, &actual_digest)) {
	861	file.digest = actual_digest;
	862	return .miss;
	863	}
821	864	}
822	865
823	866	return .hit;
824	867	}
825	868
826		/// Reset `self.hash.hasher` to the state it should be in after `hit` returns `false`.
	869	/// Reset `man.hash.hasher` to the state it should be in after `hit` returns `CheckStatus.miss`.
827	870	/// The hasher contains the original input digest, and all original input file digests (i.e.
828	871	/// not including post files).
829		/// Assumes that `bin_digest` is populated for all files up to `input_file_count`. As such,
830		/// this is not necessarily safe to call within `hit`.
831		pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
	872	///
	873	/// Assumes that `bin_digest` is populated for all input files.
	874	pub fn unhit(man: *Manifest, bin_digest: BinDigest) void {
832	875	// Reset the hash.
833		self.hash.hasher = hasher_init;
834		self.hash.hasher.update(&bin_digest);
835
836		// Remove files not in the initial hash.
837		while (self.files.count() != input_file_count) {
838		var file = self.files.pop().?;
839		file.key.deinit(self.cache.gpa);
840		}
841
842		for (self.files.keys()) \|file\| {
843		self.hash.hasher.update(&file.bin_digest);
	876	man.hash.hasher = hasher_init;
	877	man.hash.hasher.update(&bin_digest);
	878	man.shrinkFilesToInput();
	879	for (man.files.keys()) \|off\| {
	880	const file = off.get(man);
	881	man.hash.hasher.update(&file.digest);
844	882	}
845	883	}
846	884
...	...	@@ -887,203 +925,133 @@ pub const Manifest = struct {
887	925	return timestamp.nanoseconds >= man.recent_problematic_timestamp.nanoseconds;
888	926	}
889	927
890		fn populateFileHash(self: Manifest, ch_file: File) !void {
891		const io = self.cache.io;
892
893		if (ch_file.handle) \|handle\| {
894		return populateFileHashHandle(self, ch_file, handle);
895		} else {
896		const pp = ch_file.prefixed_path;
897		const dir = self.cache.prefixes()[pp.prefix].handle;
898		const handle = try dir.openFile(io, pp.sub_path, .{});
899		defer handle.close(io);
900		return populateFileHashHandle(self, ch_file, handle);
901		}
902		}
903
904		fn populateFileHashHandle(self: Manifest, ch_file: File, io_file: Io.File) !void {
905		const io = self.cache.io;
906		const gpa = self.cache.gpa;
	928	pub const AddFilePostOptions = struct {
	929	handle: union(enum) {
	930	file: ?Io.File,
	931	dir: ?Io.Dir,
	932	} = .{ .file = null },
	933	stat: ?Stat = null,
	934	contents: ?[]const u8 = null,
	935	metadata_only: bool = false,
	936	};
907	937
908		const actual_stat = try io_file.stat(io);
909		ch_file.stat = .{
910		.size = actual_stat.size,
911		.mtime = actual_stat.mtime,
912		.inode = actual_stat.inode,
913		};
	938	pub const AddFilePostError = error {
	939	/// The same file path has been added to the cache manifest both as a
	940	/// directory and as a normal file, making the intended caching
	941	/// behavior ambiguous.
	942	IsDirectoryAmbiguous,
	943	} \|\| Allocator.Error;
	944
	945	/// Add a file as a dependency of process being cached, after cache miss
	946	/// occurs.
	947	pub fn addFilePost(m: *Manifest, path: Path, options: AddFilePostOptions) AddFilePostError!void {
	948	assert(m.manifest_file != null);
	949	const cache = m.cache;
	950	const gpa = cache.gpa;
	951	const io = cache.io;
	952	const is_directory = options.handle == .dir;
914	953
915		if (try self.isProblematicTimestamp(ch_file.stat.mtime)) {
916		// The actual file has an unreliable timestamp, force it to be hashed
917		ch_file.stat.mtime = .zero;
918		ch_file.stat.inode = 0;
919		}
	954	try m.files.ensureUnusedCapacity(gpa, 1);
920	955
921		if (ch_file.max_file_size) \|max_file_size\| {
922		if (ch_file.stat.size > max_file_size) return error.FileTooBig;
	956	const prev_contents_len = m.contents.items.len;
	957	const new_header: *File = @ptrCast(try m.contents.addManyAsSlice(gpa, @sizeOf(File)));
	958	errdefer m.contents.shrinkRetainingCapacity(prev_contents_len);
923	959
924		// Hash while reading from disk, to keep the contents in the cpu
925		// cache while doing hashing.
926		const contents = try gpa.alloc(u8, @intCast(ch_file.stat.size));
927		errdefer gpa.free(contents);
	960	new_header.* = .{
	961	.flags = .{
	962	.prefix = try cache.findAppendPrefixedPath(&m.contents, path),
	963	.is_directory = is_directory,
	964	.metadata_only = options.metadata_only,
	965	},
	966	.size = undefined,
	967	.inode = undefined,
	968	.mtime = undefined,
	969	.digest = @splat(0),
	970	};
	971	assert(m.contents.items.len % @alignOf(File) == 0);
928	972
929		var hasher = hasher_init;
930		var off: usize = 0;
931		while (true) {
932		const bytes_read = try io_file.readPositional(io, &.{contents[off..]}, off);
933		if (bytes_read == 0) break;
934		hasher.update(contents[off..][0..bytes_read]);
935		off += bytes_read;
	973	const gop = m.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{
	974	.manifest = m,
	975	});
	976	m.files.lockPointers();
	977	defer m.files.unlockPointers();
	978
	979	const header = if (gop.found_existing) h: {
	980	m.contents.shrinkRetainingCapacity(prev_contents_len);
	981	const existing_off = gop.key_ptr.*;
	982	const header = existing_off.get(m);
	983	if (header.flags.is_directory != is_directory)
	984	return error.IsDirectoryAmbiguous;
	985	if (!options.metadata_only)
	986	header.flags.metadata_only = false;
	987	break :h header;
	988	} else new_header;
	989
	990	if (options.stat) \|stat\| {
	991	try header.setStat(m, stat);
	992	if (header.metadata_only) {
	993	return;
	994	} else if (options.contents) \|contents\| {
	995	var hasher = hasher_init;
	996	hasher.update(contents);
	997	hasher.final(&header.digest);
	998	return;
936	999	}
937		hasher.final(&ch_file.bin_digest);
938
939		ch_file.contents = contents;
940		} else {
941		try hashFile(io, io_file, &ch_file.bin_digest);
942	1000	}
943	1001
944		self.hash.hasher.update(&ch_file.bin_digest);
945		}
	1002	const need_stat = options.stat == null;
946	1003
947		/// Add a file as a dependency of process being cached, after the initial hash has been
948		/// calculated. This is useful for processes that don't know all the files that
949		/// are depended on ahead of time. For example, a source file that can import other files
950		/// will need to be recompiled if the imported file is changed.
951		pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
952		assert(self.manifest_file != null);
953
954		const gpa = self.cache.gpa;
955		const prefixed_path = try self.cache.findPrefix(file_path);
956		errdefer gpa.free(prefixed_path.sub_path);
957
958		const gop = try self.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
959		errdefer _ = self.files.pop();
	1004	switch (options.handle) {
	1005	.dir => \|opt_handle\| if (opt_handle) \|handle\| {
	1006	try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);
	1007	} else {
	1008	const dir = cache.prefixes()[header.flags.prefix].handle;
	1009	const handle = try dir.openDir(io, header.path(), .{ .access_sub_paths = false, .iterate = true, });
	1010	defer handle.close(io);
	1011	try populateDirectory(m, header, need_stat, handle, options.contents, header.metadata_only);
	1012	},
960	1013
961		if (gop.found_existing) {
962		gpa.free(prefixed_path.sub_path);
963		return gop.key_ptr.contents.?;
	1014	.file => \|opt_handle\| if (opt_handle) \|handle\| {
	1015	try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only);
	1016	} else {
	1017	const dir = cache.prefixes()[header.flags.prefix].handle;
	1018	const handle = try dir.openFile(io, header.path(), .{ .mode = .read_only });
	1019	defer handle.close(io);
	1020	try populateFile(m, header, need_stat, handle, options.contents, header.metadata_only);
	1021	},
964	1022	}
965
966		gop.key_ptr.* = .{
967		.prefixed_path = prefixed_path,
968		.max_file_size = max_file_size,
969		.stat = undefined,
970		.bin_digest = undefined,
971		.contents = null,
972		.handle = null,
973		};
974
975		self.files.lockPointers();
976		defer self.files.unlockPointers();
977
978		try self.populateFileHash(gop.key_ptr);
979		return gop.key_ptr.contents.?;
980		}
981
982		/// Add a file as a dependency of process being cached, after the initial hash has been
983		/// calculated.
984		///
985		/// This is useful for processes that don't know the all the files that are
986		/// depended on ahead of time. For example, a source file that can import
987		/// other files will need to be recompiled if the imported file is changed.
988		pub fn addFilePost(man: *Manifest, file_path: []const u8) !void {
989		assert(man.manifest_file != null);
990		const gpa = man.cache.gpa;
991		const prefixed_path = try man.cache.findPrefix(file_path);
992		var keep = false;
993		defer if (!keep) gpa.free(prefixed_path.sub_path);
994		keep = try addPrefixedPathPost(man, prefixed_path);
995	1023	}
996	1024
997		pub fn addPathPost(man: *Manifest, path: Path) !void {
998		assert(man.manifest_file != null);
999		const gpa = man.cache.gpa;
1000		const prefixed_path: PrefixedPath = try man.cache.findPrefixPath(path);
1001		var keep = false;
1002		defer if (!keep) gpa.free(prefixed_path.sub_path);
1003		keep = try addPrefixedPathPost(man, prefixed_path);
1004		}
1005
1006		/// Low level function. `prefixed_path` references cloned memory. Returns
1007		/// whether or not `prefixed_path.sub_path` should be kept.
1008		pub fn addPrefixedPathPost(man: *Manifest, prefixed_path: PrefixedPath) !bool {
1009		assert(man.manifest_file != null);
1010		const gpa = man.cache.gpa;
1011
1012		const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
1013		errdefer _ = man.files.pop();
1014
1015		if (gop.found_existing) return false;
1016
1017		gop.key_ptr.* = .{
1018		.prefixed_path = prefixed_path,
1019		.max_file_size = null,
1020		.handle = null,
1021		.stat = undefined,
1022		.bin_digest = undefined,
1023		.contents = null,
1024		};
1025
1026		man.files.lockPointers();
1027		defer man.files.unlockPointers();
1028
1029		try man.populateFileHash(gop.key_ptr);
1030		return true;
1031		}
1032
1033		/// Like `addFilePost` but when the file contents have already been loaded from disk.
1034		pub fn addFilePostContents(
1035		man: *Manifest,
1036		file_path: []const u8,
1037		bytes: []const u8,
1038		stat: File.Stat,
1039		) !void {
1040		assert(man.manifest_file != null);
1041		const gpa = man.cache.gpa;
1042		const prefixed_path = try man.cache.findPrefix(file_path);
1043		var keep = false;
1044		defer if (!keep) gpa.free(prefixed_path.sub_path);
1045		keep = try addPrefixedPathPostContents(man, prefixed_path, bytes, stat);
1046		}
1047
1048		/// Low level function. `prefixed_path` references cloned memory. Returns
1049		/// whether or not `prefixed_path.sub_path` should be kept.
1050		pub fn addPrefixedPathPostContents(
1051		man: *Manifest,
1052		prefixed_path: PrefixedPath,
1053		bytes: []const u8,
1054		stat: File.Stat,
1055		) !bool {
1056		const gpa = man.cache.gpa;
1057		const gop = try man.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
1058		errdefer _ = man.files.pop();
	1025	fn populateFile(m: Manifest, file: File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void {
	1026	const io = m.cache.io;
1059	1027
1060		if (gop.found_existing) return false;
1061
1062		const new_file = gop.key_ptr;
1063
1064		new_file.* = .{
1065		.prefixed_path = prefixed_path,
1066		.max_file_size = null,
1067		.handle = null,
1068		.stat = stat,
1069		.bin_digest = undefined,
1070		.contents = null,
1071		};
1072
1073		if (try man.isProblematicTimestamp(new_file.stat.mtime)) {
1074		// The actual file has an unreliable timestamp, force it to be hashed
1075		new_file.stat.mtime = .zero;
1076		new_file.stat.inode = 0;
	1028	if (need_stat) {
	1029	const stat = try handle.stat(io);
	1030	try file.setStat(m, stat);
1077	1031	}
1078
1079		{
	1032	if (metadata_only) return;
	1033	if (contents) \|bytes\| {
1080	1034	var hasher = hasher_init;
1081	1035	hasher.update(bytes);
1082		hasher.final(&new_file.bin_digest);
	1036	hasher.final(&file.digest);
	1037	} else {
	1038	try hashFile(io, handle, &file.digest);
1083	1039	}
	1040	}
1084	1041
1085		man.hash.hasher.update(&new_file.bin_digest);
1086		return true;
	1042	fn populateDirectory(m: Manifest, file: File, need_stat: bool, handle: Io.File, contents: ?[]const u8, metadata_only: bool,) !void {
	1043	_ = m;
	1044	_ = file;
	1045	_ = need_stat;
	1046	_ = handle;
	1047	_ = contents;
	1048	_ = metadata_only;
	1049	@panic("TODO");
	1050	}
	1051
	1052	pub fn addDepFile(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
	1053	assert(self.manifest_file == null);
	1054	return self.addDepFileMaybePost(dir, dep_file_sub_path);
1087	1055	}
1088	1056
1089	1057	pub fn addDepFilePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
...	...	@@ -1094,7 +1062,7 @@ pub const Manifest = struct {
1094	1062	fn addDepFileMaybePost(self: *Manifest, dir: Io.Dir, dep_file_sub_path: []const u8) !void {
1095	1063	const gpa = self.cache.gpa;
1096	1064	const io = self.cache.io;
1097		const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(manifest_file_size_max));
	1065	const dep_file_contents = try dir.readFileAlloc(io, dep_file_sub_path, gpa, .limited(file_size_max));
1098	1066	defer gpa.free(dep_file_contents);
1099	1067
1100	1068	var error_buf: std.ArrayList(u8) = .empty;
...	...	@@ -1151,39 +1119,24 @@ pub const Manifest = struct {
1151	1119
1152	1120	/// If `want_shared_lock` is true, this function automatically downgrades the
1153	1121	/// lock from exclusive to shared.
1154		pub fn writeManifest(self: *Manifest) !void {
1155		assert(self.have_exclusive_lock);
1156		const io = self.cache.io;
1157		const manifest_file = self.manifest_file.?;
1158		if (self.manifest_dirty) {
1159		self.manifest_dirty = false;
1160
1161		var buffer: [4000]u8 = undefined;
1162		var fw = manifest_file.writer(io, &buffer);
1163		writeDirtyManifestToStream(self, &fw) catch \|err\| switch (err) {
1164		error.WriteFailed => return fw.err.?,
1165		else => \|e\| return e,
1166		};
1167		}
	1122	pub fn writeManifest(m: *Manifest) !void {
	1123	assert(m.have_exclusive_lock);
	1124	const io = m.cache.io;
	1125	const manifest_file = m.manifest_file.?;
	1126	if (m.manifest_dirty) {
	1127
	1128	m.contents.appendAssumeCapacity(0);
	1129	defer _ = m.contents.pop().?;
	1130
	1131	try manifest_file.setLength(io, m.contents.items.len);
	1132	try manifest_file.writePositionalAll(io, m.contents.items, 0);
1168	1133
1169		if (self.want_shared_lock) {
1170		try self.downgradeToSharedLock();
	1134	m.manifest_dirty = false;
1171	1135	}
1172		}
1173	1136
1174		fn writeDirtyManifestToStream(self: Manifest, fw: Io.File.Writer) !void {
1175		try fw.interface.writeAll(manifest_header ++ "\n");
1176		for (self.files.keys()) \|file\| {
1177		try fw.interface.print("{d} {d} {d} {x} {d} {s}\n", .{
1178		file.stat.size,
1179		file.stat.inode,
1180		file.stat.mtime,
1181		&file.bin_digest,
1182		file.prefixed_path.prefix,
1183		file.prefixed_path.sub_path,
1184		});
	1137	if (m.want_shared_lock) {
	1138	try m.downgradeToSharedLock();
1185	1139	}
1186		try fw.end();
1187	1140	}
1188	1141
1189	1142	fn downgradeToSharedLock(self: *Manifest) !void {
...	...	@@ -1275,33 +1228,40 @@ pub const Manifest = struct {
1275	1228
1276	1229	pub fn populateOtherManifest(man: Manifest, other: Manifest, prefix_map: [5]u8) Allocator.Error!void {
1277	1230	const gpa = other.cache.gpa;
	1231	assert(other.manifest_file != null);
1278	1232	assert(@typeInfo(std.zig.Server.Message.PathPrefix).@"enum".field_names.len == man.cache.prefixes_len);
1279	1233	assert(man.cache.prefixes_len == 5);
1280		for (man.files.keys()) \|file\| {
1281		const prefixed_path: PrefixedPath = .{
1282		.prefix = prefix_map[file.prefixed_path.prefix],
1283		.sub_path = try gpa.dupe(u8, file.prefixed_path.sub_path),
1284		};
1285		errdefer gpa.free(prefixed_path.sub_path);
1286	1234
1287		const gop = try other.files.getOrPutAdapted(gpa, prefixed_path, FilesAdapter{});
1288		errdefer _ = other.files.pop();
	1235	const orig_files_len = other.files.count();
	1236	const orig_contents_len = other.contents.items.len;
	1237	errdefer {
	1238	other.files.shrinkRetainingCapacity(orig_files_len);
	1239	other.contents.shrinkRetainingCapacity(orig_contents_len);
	1240	}
	1241
	1242	for (man.files.keys(), 0..) \|off, file_index\| {
	1243	try other.files.ensureUnusedCapacity(gpa, 1);
	1244
	1245	const next_off = if (file_index < man.files.count())
	1246	@backingInt(man.files.keys()[file_index + 1])
	1247	else
	1248	man.contents.items.len;
	1249
	1250	const copy_bytes = man.contents.items[@backingInt(off)..next_off];
	1251	const prev_contents_len = other.contents.items.len;
	1252	try other.contents.appendSlice(gpa, copy_bytes);
	1253
	1254	const gop = other.files.getOrPutAssumeCapacity(@fromBackingInt(prev_contents_len), .{
	1255	.manifest = other,
	1256	});
1289	1257
1290	1258	if (gop.found_existing) {
1291		gpa.free(prefixed_path.sub_path);
	1259	other.contents.shrinkRetainingCapacity(prev_contents_len);
1292	1260	continue;
1293	1261	}
1294	1262
1295		gop.key_ptr.* = .{
1296		.prefixed_path = prefixed_path,
1297		.max_file_size = file.max_file_size,
1298		.handle = file.handle,
1299		.stat = file.stat,
1300		.bin_digest = file.bin_digest,
1301		.contents = null,
1302		};
1303
1304		other.hash.hasher.update(&gop.key_ptr.bin_digest);
	1263	const other_file = File.get(@fromBackingInt(prev_contents_len));
	1264	other_file.prefix = prefix_map[other_file.prefix];
1305	1265	}
1306	1266	}
1307	1267	};

lib/std/Build/Configuration.zig+2-3

...	...	@@ -1872,12 +1872,11 @@ pub const PathDep = extern struct {
1872	1872	pkg: Package.OptionalIndex,
1873	1873
1874	1874	pub const Flags = packed struct(u32) {
1875		mode: Mode,
	1875	is_directory: bool,
	1876	metadata_only: bool,
1876	1877	base: LazyPath.Relative.Base,
1877	1878	_: u16 = 0,
1878	1879	};
1879
1880		pub const Mode = enum(u8) { directory, contents, metadata };
1881	1880	};
1882	1881
1883	1882	pub const InstallDestDir = enum(u32) {

lib/std/fs/path.zig+7

...	...	@@ -1178,6 +1178,13 @@ pub fn resolvePosix(gpa: Allocator, paths: []const []const u8) Allocator.Error![
1178	1178	}
1179	1179	}
1180	1180
	1181	pub fn resolvePosix2(gpa: Allocator, al: *std.ArrayList(u8), paths: []const []const u8) Allocator.Error!void {
	1182	_ = gpa;
	1183	_ = al;
	1184	_ = paths;
	1185	@panic("TODO");
	1186	}
	1187
1181	1188	test resolve {
1182	1189	try testResolveWindows(&[_][]const u8{ "a", "..\\..\\.." }, "..\\..");
1183	1190	try testResolveWindows(&[_][]const u8{ "..", "", "..\\..\\foo" }, "..\\..\\..\\foo");

src/Zcu/PerThread.zig+5-6

...	...	@@ -346,11 +346,8 @@ pub fn update(
346	346	}
347	347	}
348	348	fn workerUpdateBuiltinFile(comp: Compilation, file: Zcu.File) void {
349		Builtin.updateFileOnDisk(file, comp) catch \|err\| comp.lockAndSetMiscFailure(
350		.write_builtin_zig,
351		"unable to write '{f}': {s}",
352		.{ file.path.fmt(comp), @errorName(err) },
353		);
	349	Builtin.updateFileOnDisk(file, comp) catch \|err\|
	350	comp.lockAndSetMiscFailure(.write_builtin_zig, "unable to write {qf}: {t}", .{ file.path.fmt(comp), err });
354	351	}
355	352	fn workerUpdateFile(
356	353	comp: *Compilation,
...	...	@@ -369,7 +366,9 @@ fn workerUpdateFile(
369	366	const active = comp.zcu.?.activate(tid);
370	367	defer active.deactivate();
371	368	active.pt.updateFile(file_index, file) catch \|err\| {
372		active.pt.reportRetryableFileError(file_index, "unable to load '{s}': {s}", .{ std.fs.path.basename(file.path.sub_path), @errorName(err) }) catch \|oom\| switch (oom) {
	369	active.pt.reportRetryableFileError(file_index, "unable to load {q}: {t}", .{
	370	std.fs.path.basename(file.path.sub_path), err,
	371	}) catch \|oom\| switch (oom) {
373	372	error.OutOfMemory => {
374	373	comp.mutex.lockUncancelable(io);
375	374	defer comp.mutex.unlock(io);

test/src/Cases.zig+1-1

...	...	@@ -319,7 +319,7 @@ pub fn addCompile(
319	319	pub fn addFromDir(ctx: Cases, dir: Io.Dir, path_from_root: []const u8, b: std.Build) void {
320	320	var current_file: []const u8 = "none";
321	321	ctx.addFromDirInner(dir, path_from_root, &current_file, b) catch \|err\| {
322		std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}\n", .{
	322	std.debug.panicExtra(@returnAddress(), "test harness failed to process file {q}: {t}", .{
323	323	current_file, err,
324	324	});
325	325	};