Commit 9cb52ca6ce7043ba0ce08d5650ac542075f10685 | zig

author	Andrew Kelley <andrew@ziglang.org> 2023-02-05 19:39:04-07:00
committer	Andrew Kelley <andrew@ziglang.org> 2023-02-13 06:42:25-07:00
log	9cb52ca6ce7043ba0ce08d5650ac542075f10685
tree	272ce33129d65e8af6f068fb620ac5a3d33370fc
parent	2654d0c66860d32714e33404554482cbc0cbabf5

move the cache system from compiler to std lib

18 files changed, 2404 insertions(+), 2398 deletions(-)

CMakeLists.txt
lib
- std
  - Build.zig
  - Build
    Cache.zig
    Cache
    DepTokenizer.zig
src
- Cache.zig
- Compilation.zig
- DepTokenizer.zig
- Module.zig
- Package.zig
- glibc.zig
- link.zig
- link
  - Coff
    lld.zig
  - Elf.zig
  - MachO.zig
  - MachO
    zld.zig
  - Wasm.zig
- main.zig
- mingw.zig

CMakeLists.txt+3-2

...	...	@@ -216,6 +216,9 @@ set(ZIG_STAGE2_SOURCES
216	216	"${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig"
217	217	"${CMAKE_SOURCE_DIR}/lib/std/base64.zig"
218	218	"${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig"
	219	"${CMAKE_SOURCE_DIR}/lib/std/Build.zig"
	220	"${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig"
	221	"${CMAKE_SOURCE_DIR}/lib/std/Build/Cache/DepTokenizer.zig"
219	222	"${CMAKE_SOURCE_DIR}/lib/std/builtin.zig"
220	223	"${CMAKE_SOURCE_DIR}/lib/std/c.zig"
221	224	"${CMAKE_SOURCE_DIR}/lib/std/c/linux.zig"
...	...	@@ -523,9 +526,7 @@ set(ZIG_STAGE2_SOURCES
523	526	"${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig"
524	527	"${CMAKE_SOURCE_DIR}/src/Air.zig"
525	528	"${CMAKE_SOURCE_DIR}/src/AstGen.zig"
526		"${CMAKE_SOURCE_DIR}/src/Cache.zig"
527	529	"${CMAKE_SOURCE_DIR}/src/Compilation.zig"
528		"${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig"
529	530	"${CMAKE_SOURCE_DIR}/src/Liveness.zig"
530	531	"${CMAKE_SOURCE_DIR}/src/Module.zig"
531	532	"${CMAKE_SOURCE_DIR}/src/Package.zig"

lib/std/Build.zig+2

...	...	@@ -19,6 +19,8 @@ const NativeTargetInfo = std.zig.system.NativeTargetInfo;
19	19	const Sha256 = std.crypto.hash.sha2.Sha256;
20	20	const Build = @This();
21	21
	22	pub const Cache = @import("Build/Cache.zig");
	23
22	24	/// deprecated: use `CompileStep`.
23	25	pub const LibExeObjStep = CompileStep;
24	26	/// deprecated: use `Build`.

lib/std/Build/Cache.zig created+1276

...	...	@@ -0,0 +1,1276 @@
	1	//! Manages `zig-cache` directories.
	2	//! This is not a general-purpose cache. It is designed to be fast and simple,
	3	//! not to withstand attacks using specially-crafted input.
	4
	5	pub const Directory = struct {
	6	/// This field is redundant for operations that can act on the open directory handle
	7	/// directly, but it is needed when passing the directory to a child process.
	8	/// `null` means cwd.
	9	path: ?[]const u8,
	10	handle: std.fs.Dir,
	11
	12	pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
	13	if (self.path) \|p\| {
	14	// TODO clean way to do this with only 1 allocation
	15	const part2 = try std.fs.path.join(allocator, paths);
	16	defer allocator.free(part2);
	17	return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
	18	} else {
	19	return std.fs.path.join(allocator, paths);
	20	}
	21	}
	22
	23	pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
	24	if (self.path) \|p\| {
	25	// TODO clean way to do this with only 1 allocation
	26	const part2 = try std.fs.path.join(allocator, paths);
	27	defer allocator.free(part2);
	28	return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
	29	} else {
	30	return std.fs.path.joinZ(allocator, paths);
	31	}
	32	}
	33
	34	/// Whether or not the handle should be closed, or the path should be freed
	35	/// is determined by usage, however this function is provided for convenience
	36	/// if it happens to be what the caller needs.
	37	pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
	38	self.handle.close();
	39	if (self.path) \|p\| gpa.free(p);
	40	self.* = undefined;
	41	}
	42	};
	43
	44	gpa: Allocator,
	45	manifest_dir: fs.Dir,
	46	hash: HashHelper = .{},
	47	/// This value is accessed from multiple threads, protected by mutex.
	48	recent_problematic_timestamp: i128 = 0,
	49	mutex: std.Thread.Mutex = .{},
	50
	51	/// A set of strings such as the zig library directory or project source root, which
	52	/// are stripped from the file paths before putting into the cache. They
	53	/// are replaced with single-character indicators. This is not to save
	54	/// space but to eliminate absolute file paths. This improves portability
	55	/// and usefulness of the cache for advanced use cases.
	56	prefixes_buffer: [3]Directory = undefined,
	57	prefixes_len: usize = 0,
	58
	59	pub const DepTokenizer = @import("Cache/DepTokenizer.zig");
	60
	61	const Cache = @This();
	62	const std = @import("std");
	63	const builtin = @import("builtin");
	64	const crypto = std.crypto;
	65	const fs = std.fs;
	66	const assert = std.debug.assert;
	67	const testing = std.testing;
	68	const mem = std.mem;
	69	const fmt = std.fmt;
	70	const Allocator = std.mem.Allocator;
	71	const log = std.log.scoped(.cache);
	72
	73	pub fn addPrefix(cache: *Cache, directory: Directory) void {
	74	if (directory.path) \|p\| {
	75	log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p });
	76	}
	77	cache.prefixes_buffer[cache.prefixes_len] = directory;
	78	cache.prefixes_len += 1;
	79	}
	80
	81	/// Be sure to call `Manifest.deinit` after successful initialization.
	82	pub fn obtain(cache: *Cache) Manifest {
	83	return Manifest{
	84	.cache = cache,
	85	.hash = cache.hash,
	86	.manifest_file = null,
	87	.manifest_dirty = false,
	88	.hex_digest = undefined,
	89	};
	90	}
	91
	92	pub fn prefixes(cache: *const Cache) []const Directory {
	93	return cache.prefixes_buffer[0..cache.prefixes_len];
	94	}
	95
	96	const PrefixedPath = struct {
	97	prefix: u8,
	98	sub_path: []u8,
	99	};
	100
	101	fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
	102	const gpa = cache.gpa;
	103	const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
	104	errdefer gpa.free(resolved_path);
	105	return findPrefixResolved(cache, resolved_path);
	106	}
	107
	108	/// Takes ownership of `resolved_path` on success.
	109	fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
	110	const gpa = cache.gpa;
	111	const prefixes_slice = cache.prefixes();
	112	var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
	113	while (i < prefixes_slice.len) : (i += 1) {
	114	const p = prefixes_slice[i].path.?;
	115	if (mem.startsWith(u8, resolved_path, p)) {
	116	// +1 to skip over the path separator here
	117	const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
	118	gpa.free(resolved_path);
	119	return PrefixedPath{
	120	.prefix = @intCast(u8, i),
	121	.sub_path = sub_path,
	122	};
	123	} else {
	124	log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p });
	125	}
	126	}
	127
	128	return PrefixedPath{
	129	.prefix = 0,
	130	.sub_path = resolved_path,
	131	};
	132	}
	133
	134	/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
	135	pub const bin_digest_len = 16;
	136	pub const hex_digest_len = bin_digest_len * 2;
	137	pub const BinDigest = [bin_digest_len]u8;
	138
	139	const manifest_file_size_max = 50 * 1024 * 1024;
	140
	141	/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
	142	/// provides enough collision resistance for the Manifest use cases, while being one of our
	143	/// fastest options right now.
	144	pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
	145
	146	/// Initial state, that can be copied.
	147	pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
	148
	149	pub const File = struct {
	150	prefixed_path: ?PrefixedPath,
	151	max_file_size: ?usize,
	152	stat: Stat,
	153	bin_digest: BinDigest,
	154	contents: ?[]const u8,
	155
	156	pub const Stat = struct {
	157	inode: fs.File.INode,
	158	size: u64,
	159	mtime: i128,
	160	};
	161
	162	pub fn deinit(self: *File, gpa: Allocator) void {
	163	if (self.prefixed_path) \|pp\| {
	164	gpa.free(pp.sub_path);
	165	self.prefixed_path = null;
	166	}
	167	if (self.contents) \|contents\| {
	168	gpa.free(contents);
	169	self.contents = null;
	170	}
	171	self.* = undefined;
	172	}
	173	};
	174
	175	pub const HashHelper = struct {
	176	hasher: Hasher = hasher_init,
	177
	178	/// Record a slice of bytes as an dependency of the process being cached
	179	pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
	180	hh.hasher.update(mem.asBytes(&bytes.len));
	181	hh.hasher.update(bytes);
	182	}
	183
	184	pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
	185	hh.add(optional_bytes != null);
	186	hh.addBytes(optional_bytes orelse return);
	187	}
	188
	189	pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
	190	hh.add(list_of_bytes.len);
	191	for (list_of_bytes) \|bytes\| hh.addBytes(bytes);
	192	}
	193
	194	/// Convert the input value into bytes and record it as a dependency of the process being cached.
	195	pub fn add(hh: *HashHelper, x: anytype) void {
	196	switch (@TypeOf(x)) {
	197	std.builtin.Version => {
	198	hh.add(x.major);
	199	hh.add(x.minor);
	200	hh.add(x.patch);
	201	},
	202	std.Target.Os.TaggedVersionRange => {
	203	switch (x) {
	204	.linux => \|linux\| {
	205	hh.add(linux.range.min);
	206	hh.add(linux.range.max);
	207	hh.add(linux.glibc);
	208	},
	209	.windows => \|windows\| {
	210	hh.add(windows.min);
	211	hh.add(windows.max);
	212	},
	213	.semver => \|semver\| {
	214	hh.add(semver.min);
	215	hh.add(semver.max);
	216	},
	217	.none => {},
	218	}
	219	},
	220	else => switch (@typeInfo(@TypeOf(x))) {
	221	.Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
	222	else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
	223	},
	224	}
	225	}
	226
	227	pub fn addOptional(hh: *HashHelper, optional: anytype) void {
	228	hh.add(optional != null);
	229	hh.add(optional orelse return);
	230	}
	231
	232	/// Returns a hex encoded hash of the inputs, without modifying state.
	233	pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
	234	var copy = hh;
	235	return copy.final();
	236	}
	237
	238	pub fn peekBin(hh: HashHelper) BinDigest {
	239	var copy = hh;
	240	var bin_digest: BinDigest = undefined;
	241	copy.hasher.final(&bin_digest);
	242	return bin_digest;
	243	}
	244
	245	/// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
	246	pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
	247	var bin_digest: BinDigest = undefined;
	248	hh.hasher.final(&bin_digest);
	249
	250	var out_digest: [hex_digest_len]u8 = undefined;
	251	_ = std.fmt.bufPrint(
	252	&out_digest,
	253	"{s}",
	254	.{std.fmt.fmtSliceHexLower(&bin_digest)},
	255	) catch unreachable;
	256	return out_digest;
	257	}
	258	};
	259
	260	pub const Lock = struct {
	261	manifest_file: fs.File,
	262
	263	pub fn release(lock: *Lock) void {
	264	if (builtin.os.tag == .windows) {
	265	// Windows does not guarantee that locks are immediately unlocked when
	266	// the file handle is closed. See LockFileEx documentation.
	267	lock.manifest_file.unlock();
	268	}
	269
	270	lock.manifest_file.close();
	271	lock.* = undefined;
	272	}
	273	};
	274
	275	pub const Manifest = struct {
	276	cache: *Cache,
	277	/// Current state for incremental hashing.
	278	hash: HashHelper,
	279	manifest_file: ?fs.File,
	280	manifest_dirty: bool,
	281	/// Set this flag to true before calling hit() in order to indicate that
	282	/// upon a cache hit, the code using the cache will not modify the files
	283	/// within the cache directory. This allows multiple processes to utilize
	284	/// the same cache directory at the same time.
	285	want_shared_lock: bool = true,
	286	have_exclusive_lock: bool = false,
	287	// Indicate that we want isProblematicTimestamp to perform a filesystem write in
	288	// order to obtain a problematic timestamp for the next call. Calls after that
	289	// will then use the same timestamp, to avoid unnecessary filesystem writes.
	290	want_refresh_timestamp: bool = true,
	291	files: std.ArrayListUnmanaged(File) = .{},
	292	hex_digest: [hex_digest_len]u8,
	293	/// Populated when hit() returns an error because of one
	294	/// of the files listed in the manifest.
	295	failed_file_index: ?usize = null,
	296	/// Keeps track of the last time we performed a file system write to observe
	297	/// what time the file system thinks it is, according to its own granularity.
	298	recent_problematic_timestamp: i128 = 0,
	299
	300	/// Add a file as a dependency of process being cached. When `hit` is
	301	/// called, the file's contents will be checked to ensure that it matches
	302	/// the contents from previous times.
	303	///
	304	/// Max file size will be used to determine the amount of space the file contents
	305	/// are allowed to take up in memory. If max_file_size is null, then the contents
	306	/// will not be loaded into memory.
	307	///
	308	/// Returns the index of the entry in the `files` array list. You can use it
	309	/// to access the contents of the file after calling `hit()` like so:
	310	///
	311	/// ```
	312	/// var file_contents = cache_hash.files.items[file_index].contents.?;
	313	/// ```
	314	pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
	315	assert(self.manifest_file == null);
	316
	317	const gpa = self.cache.gpa;
	318	try self.files.ensureUnusedCapacity(gpa, 1);
	319	const prefixed_path = try self.cache.findPrefix(file_path);
	320	errdefer gpa.free(prefixed_path.sub_path);
	321
	322	log.debug("Manifest.addFile {s} -> {d} {s}", .{
	323	file_path, prefixed_path.prefix, prefixed_path.sub_path,
	324	});
	325
	326	self.files.addOneAssumeCapacity().* = .{
	327	.prefixed_path = prefixed_path,
	328	.contents = null,
	329	.max_file_size = max_file_size,
	330	.stat = undefined,
	331	.bin_digest = undefined,
	332	};
	333
	334	self.hash.add(prefixed_path.prefix);
	335	self.hash.addBytes(prefixed_path.sub_path);
	336
	337	return self.files.items.len - 1;
	338	}
	339
	340	pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
	341	self.hash.add(optional_file_path != null);
	342	const file_path = optional_file_path orelse return;
	343	_ = try self.addFile(file_path, null);
	344	}
	345
	346	pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
	347	self.hash.add(list_of_files.len);
	348	for (list_of_files) \|file_path\| {
	349	_ = try self.addFile(file_path, null);
	350	}
	351	}
	352
	353	/// Check the cache to see if the input exists in it. If it exists, returns `true`.
	354	/// A hex encoding of its hash is available by calling `final`.
	355	///
	356	/// This function will also acquire an exclusive lock to the manifest file. This means
	357	/// that a process holding a Manifest will block any other process attempting to
	358	/// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
	359	/// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
	360	/// file to be locked in exclusive mode.
	361	///
	362	/// The lock on the manifest file is released when `deinit` is called. As another
	363	/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
	364	/// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
	365	pub fn hit(self: *Manifest) !bool {
	366	const gpa = self.cache.gpa;
	367	assert(self.manifest_file == null);
	368
	369	self.failed_file_index = null;
	370
	371	const ext = ".txt";
	372	var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
	373
	374	var bin_digest: BinDigest = undefined;
	375	self.hash.hasher.final(&bin_digest);
	376
	377	_ = std.fmt.bufPrint(
	378	&self.hex_digest,
	379	"{s}",
	380	.{std.fmt.fmtSliceHexLower(&bin_digest)},
	381	) catch unreachable;
	382
	383	self.hash.hasher = hasher_init;
	384	self.hash.hasher.update(&bin_digest);
	385
	386	mem.copy(u8, &manifest_file_path, &self.hex_digest);
	387	manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
	388
	389	if (self.files.items.len == 0) {
	390	// If there are no file inputs, we check if the manifest file exists instead of
	391	// comparing the hashes on the files used for the cached item
	392	while (true) {
	393	if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
	394	.mode = .read_write,
	395	.lock = .Exclusive,
	396	.lock_nonblocking = self.want_shared_lock,
	397	})) \|manifest_file\| {
	398	self.manifest_file = manifest_file;
	399	self.have_exclusive_lock = true;
	400	break;
	401	} else \|open_err\| switch (open_err) {
	402	error.WouldBlock => {
	403	self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
	404	.lock = .Shared,
	405	});
	406	break;
	407	},
	408	error.FileNotFound => {
	409	if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
	410	.read = true,
	411	.truncate = false,
	412	.lock = .Exclusive,
	413	.lock_nonblocking = self.want_shared_lock,
	414	})) \|manifest_file\| {
	415	self.manifest_file = manifest_file;
	416	self.manifest_dirty = true;
	417	self.have_exclusive_lock = true;
	418	return false; // cache miss; exclusive lock already held
	419	} else \|err\| switch (err) {
	420	error.WouldBlock => continue,
	421	else => \|e\| return e,
	422	}
	423	},
	424	else => \|e\| return e,
	425	}
	426	}
	427	} else {
	428	if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
	429	.read = true,
	430	.truncate = false,
	431	.lock = .Exclusive,
	432	.lock_nonblocking = self.want_shared_lock,
	433	})) \|manifest_file\| {
	434	self.manifest_file = manifest_file;
	435	self.have_exclusive_lock = true;
	436	} else \|err\| switch (err) {
	437	error.WouldBlock => {
	438	self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
	439	.lock = .Shared,
	440	});
	441	},
	442	else => \|e\| return e,
	443	}
	444	}
	445
	446	self.want_refresh_timestamp = true;
	447
	448	const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
	449	defer gpa.free(file_contents);
	450
	451	const input_file_count = self.files.items.len;
	452	var any_file_changed = false;
	453	var line_iter = mem.tokenize(u8, file_contents, "\n");
	454	var idx: usize = 0;
	455	while (line_iter.next()) \|line\| {
	456	defer idx += 1;
	457
	458	const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
	459	const new = try self.files.addOne(gpa);
	460	new.* = .{
	461	.prefixed_path = null,
	462	.contents = null,
	463	.max_file_size = null,
	464	.stat = undefined,
	465	.bin_digest = undefined,
	466	};
	467	break :blk new;
	468	};
	469
	470	var iter = mem.tokenize(u8, line, " ");
	471	const size = iter.next() orelse return error.InvalidFormat;
	472	const inode = iter.next() orelse return error.InvalidFormat;
	473	const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
	474	const digest_str = iter.next() orelse return error.InvalidFormat;
	475	const prefix_str = iter.next() orelse return error.InvalidFormat;
	476	const file_path = iter.rest();
	477
	478	cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
	479	cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
	480	cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
	481	_ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
	482	const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
	483	if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
	484
	485	if (file_path.len == 0) {
	486	return error.InvalidFormat;
	487	}
	488	if (cache_hash_file.prefixed_path) \|pp\| {
	489	if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
	490	return error.InvalidFormat;
	491	}
	492	}
	493
	494	if (cache_hash_file.prefixed_path == null) {
	495	cache_hash_file.prefixed_path = .{
	496	.prefix = prefix,
	497	.sub_path = try gpa.dupe(u8, file_path),
	498	};
	499	}
	500
	501	const pp = cache_hash_file.prefixed_path.?;
	502	const dir = self.cache.prefixes()[pp.prefix].handle;
	503	const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch \|err\| switch (err) {
	504	error.FileNotFound => {
	505	try self.upgradeToExclusiveLock();
	506	return false;
	507	},
	508	else => return error.CacheUnavailable,
	509	};
	510	defer this_file.close();
	511
	512	const actual_stat = this_file.stat() catch \|err\| {
	513	self.failed_file_index = idx;
	514	return err;
	515	};
	516	const size_match = actual_stat.size == cache_hash_file.stat.size;
	517	const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
	518	const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
	519
	520	if (!size_match or !mtime_match or !inode_match) {
	521	self.manifest_dirty = true;
	522
	523	cache_hash_file.stat = .{
	524	.size = actual_stat.size,
	525	.mtime = actual_stat.mtime,
	526	.inode = actual_stat.inode,
	527	};
	528
	529	if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
	530	// The actual file has an unreliable timestamp, force it to be hashed
	531	cache_hash_file.stat.mtime = 0;
	532	cache_hash_file.stat.inode = 0;
	533	}
	534
	535	var actual_digest: BinDigest = undefined;
	536	hashFile(this_file, &actual_digest) catch \|err\| {
	537	self.failed_file_index = idx;
	538	return err;
	539	};
	540
	541	if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
	542	cache_hash_file.bin_digest = actual_digest;
	543	// keep going until we have the input file digests
	544	any_file_changed = true;
	545	}
	546	}
	547
	548	if (!any_file_changed) {
	549	self.hash.hasher.update(&cache_hash_file.bin_digest);
	550	}
	551	}
	552
	553	if (any_file_changed) {
	554	// cache miss
	555	// keep the manifest file open
	556	self.unhit(bin_digest, input_file_count);
	557	try self.upgradeToExclusiveLock();
	558	return false;
	559	}
	560
	561	if (idx < input_file_count) {
	562	self.manifest_dirty = true;
	563	while (idx < input_file_count) : (idx += 1) {
	564	const ch_file = &self.files.items[idx];
	565	self.populateFileHash(ch_file) catch \|err\| {
	566	self.failed_file_index = idx;
	567	return err;
	568	};
	569	}
	570	try self.upgradeToExclusiveLock();
	571	return false;
	572	}
	573
	574	if (self.want_shared_lock) {
	575	try self.downgradeToSharedLock();
	576	}
	577
	578	return true;
	579	}
	580
	581	pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
	582	// Reset the hash.
	583	self.hash.hasher = hasher_init;
	584	self.hash.hasher.update(&bin_digest);
	585
	586	// Remove files not in the initial hash.
	587	for (self.files.items[input_file_count..]) \|*file\| {
	588	file.deinit(self.cache.gpa);
	589	}
	590	self.files.shrinkRetainingCapacity(input_file_count);
	591
	592	for (self.files.items) \|file\| {
	593	self.hash.hasher.update(&file.bin_digest);
	594	}
	595	}
	596
	597	fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
	598	// If the file_time is prior to the most recent problematic timestamp
	599	// then we don't need to access the filesystem.
	600	if (file_time < man.recent_problematic_timestamp)
	601	return false;
	602
	603	// Next we will check the globally shared Cache timestamp, which is accessed
	604	// from multiple threads.
	605	man.cache.mutex.lock();
	606	defer man.cache.mutex.unlock();
	607
	608	// Save the global one to our local one to avoid locking next time.
	609	man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
	610	if (file_time < man.recent_problematic_timestamp)
	611	return false;
	612
	613	// This flag prevents multiple filesystem writes for the same hit() call.
	614	if (man.want_refresh_timestamp) {
	615	man.want_refresh_timestamp = false;
	616
	617	var file = man.cache.manifest_dir.createFile("timestamp", .{
	618	.read = true,
	619	.truncate = true,
	620	}) catch return true;
	621	defer file.close();
	622
	623	// Save locally and also save globally (we still hold the global lock).
	624	man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
	625	man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
	626	}
	627
	628	return file_time >= man.recent_problematic_timestamp;
	629	}
	630
	631	fn populateFileHash(self: Manifest, ch_file: File) !void {
	632	const pp = ch_file.prefixed_path.?;
	633	const dir = self.cache.prefixes()[pp.prefix].handle;
	634	const file = try dir.openFile(pp.sub_path, .{});
	635	defer file.close();
	636
	637	const actual_stat = try file.stat();
	638	ch_file.stat = .{
	639	.size = actual_stat.size,
	640	.mtime = actual_stat.mtime,
	641	.inode = actual_stat.inode,
	642	};
	643
	644	if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
	645	// The actual file has an unreliable timestamp, force it to be hashed
	646	ch_file.stat.mtime = 0;
	647	ch_file.stat.inode = 0;
	648	}
	649
	650	if (ch_file.max_file_size) \|max_file_size\| {
	651	if (ch_file.stat.size > max_file_size) {
	652	return error.FileTooBig;
	653	}
	654
	655	const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
	656	errdefer self.cache.gpa.free(contents);
	657
	658	// Hash while reading from disk, to keep the contents in the cpu cache while
	659	// doing hashing.
	660	var hasher = hasher_init;
	661	var off: usize = 0;
	662	while (true) {
	663	// give me everything you've got, captain
	664	const bytes_read = try file.read(contents[off..]);
	665	if (bytes_read == 0) break;
	666	hasher.update(contents[off..][0..bytes_read]);
	667	off += bytes_read;
	668	}
	669	hasher.final(&ch_file.bin_digest);
	670
	671	ch_file.contents = contents;
	672	} else {
	673	try hashFile(file, &ch_file.bin_digest);
	674	}
	675
	676	self.hash.hasher.update(&ch_file.bin_digest);
	677	}
	678
	679	/// Add a file as a dependency of process being cached, after the initial hash has been
	680	/// calculated. This is useful for processes that don't know all the files that
	681	/// are depended on ahead of time. For example, a source file that can import other files
	682	/// will need to be recompiled if the imported file is changed.
	683	pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
	684	assert(self.manifest_file != null);
	685
	686	const gpa = self.cache.gpa;
	687	const prefixed_path = try self.cache.findPrefix(file_path);
	688	errdefer gpa.free(prefixed_path.sub_path);
	689
	690	log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{
	691	file_path, prefixed_path.prefix, prefixed_path.sub_path,
	692	});
	693
	694	const new_ch_file = try self.files.addOne(gpa);
	695	new_ch_file.* = .{
	696	.prefixed_path = prefixed_path,
	697	.max_file_size = max_file_size,
	698	.stat = undefined,
	699	.bin_digest = undefined,
	700	.contents = null,
	701	};
	702	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
	703
	704	try self.populateFileHash(new_ch_file);
	705
	706	return new_ch_file.contents.?;
	707	}
	708
	709	/// Add a file as a dependency of process being cached, after the initial hash has been
	710	/// calculated. This is useful for processes that don't know the all the files that
	711	/// are depended on ahead of time. For example, a source file that can import other files
	712	/// will need to be recompiled if the imported file is changed.
	713	pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
	714	assert(self.manifest_file != null);
	715
	716	const gpa = self.cache.gpa;
	717	const prefixed_path = try self.cache.findPrefix(file_path);
	718	errdefer gpa.free(prefixed_path.sub_path);
	719
	720	log.debug("Manifest.addFilePost {s} -> {d} {s}", .{
	721	file_path, prefixed_path.prefix, prefixed_path.sub_path,
	722	});
	723
	724	const new_ch_file = try self.files.addOne(gpa);
	725	new_ch_file.* = .{
	726	.prefixed_path = prefixed_path,
	727	.max_file_size = null,
	728	.stat = undefined,
	729	.bin_digest = undefined,
	730	.contents = null,
	731	};
	732	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
	733
	734	try self.populateFileHash(new_ch_file);
	735	}
	736
	737	/// Like `addFilePost` but when the file contents have already been loaded from disk.
	738	/// On success, cache takes ownership of `resolved_path`.
	739	pub fn addFilePostContents(
	740	self: *Manifest,
	741	resolved_path: []u8,
	742	bytes: []const u8,
	743	stat: File.Stat,
	744	) error{OutOfMemory}!void {
	745	assert(self.manifest_file != null);
	746	const gpa = self.cache.gpa;
	747
	748	const ch_file = try self.files.addOne(gpa);
	749	errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
	750
	751	log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path});
	752
	753	const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
	754	errdefer gpa.free(prefixed_path.sub_path);
	755
	756	log.debug("Manifest.addFilePostContents -> {d} {s}", .{
	757	prefixed_path.prefix, prefixed_path.sub_path,
	758	});
	759
	760	ch_file.* = .{
	761	.prefixed_path = prefixed_path,
	762	.max_file_size = null,
	763	.stat = stat,
	764	.bin_digest = undefined,
	765	.contents = null,
	766	};
	767
	768	if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
	769	// The actual file has an unreliable timestamp, force it to be hashed
	770	ch_file.stat.mtime = 0;
	771	ch_file.stat.inode = 0;
	772	}
	773
	774	{
	775	var hasher = hasher_init;
	776	hasher.update(bytes);
	777	hasher.final(&ch_file.bin_digest);
	778	}
	779
	780	self.hash.hasher.update(&ch_file.bin_digest);
	781	}
	782
	783	pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
	784	assert(self.manifest_file != null);
	785
	786	const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
	787	defer self.cache.gpa.free(dep_file_contents);
	788
	789	var error_buf = std.ArrayList(u8).init(self.cache.gpa);
	790	defer error_buf.deinit();
	791
	792	var it: DepTokenizer = .{ .bytes = dep_file_contents };
	793
	794	// Skip first token: target.
	795	switch (it.next() orelse return) { // Empty dep file OK.
	796	.target, .target_must_resolve, .prereq => {},
	797	else => \|err\| {
	798	try err.printError(error_buf.writer());
	799	log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
	800	return error.InvalidDepFile;
	801	},
	802	}
	803	// Process 0+ preqreqs.
	804	// Clang is invoked in single-source mode so we never get more targets.
	805	while (true) {
	806	switch (it.next() orelse return) {
	807	.target, .target_must_resolve => return,
	808	.prereq => \|file_path\| try self.addFilePost(file_path),
	809	else => \|err\| {
	810	try err.printError(error_buf.writer());
	811	log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
	812	return error.InvalidDepFile;
	813	},
	814	}
	815	}
	816	}
	817
	818	/// Returns a hex encoded hash of the inputs.
	819	pub fn final(self: *Manifest) [hex_digest_len]u8 {
	820	assert(self.manifest_file != null);
	821
	822	// We don't close the manifest file yet, because we want to
	823	// keep it locked until the API user is done using it.
	824	// We also don't write out the manifest yet, because until
	825	// cache_release is called we still might be working on creating
	826	// the artifacts to cache.
	827
	828	var bin_digest: BinDigest = undefined;
	829	self.hash.hasher.final(&bin_digest);
	830
	831	var out_digest: [hex_digest_len]u8 = undefined;
	832	_ = std.fmt.bufPrint(
	833	&out_digest,
	834	"{s}",
	835	.{std.fmt.fmtSliceHexLower(&bin_digest)},
	836	) catch unreachable;
	837
	838	return out_digest;
	839	}
	840
	841	/// If `want_shared_lock` is true, this function automatically downgrades the
	842	/// lock from exclusive to shared.
	843	pub fn writeManifest(self: *Manifest) !void {
	844	assert(self.have_exclusive_lock);
	845
	846	const manifest_file = self.manifest_file.?;
	847	if (self.manifest_dirty) {
	848	self.manifest_dirty = false;
	849
	850	var contents = std.ArrayList(u8).init(self.cache.gpa);
	851	defer contents.deinit();
	852
	853	const writer = contents.writer();
	854	var encoded_digest: [hex_digest_len]u8 = undefined;
	855
	856	for (self.files.items) \|file\| {
	857	_ = std.fmt.bufPrint(
	858	&encoded_digest,
	859	"{s}",
	860	.{std.fmt.fmtSliceHexLower(&file.bin_digest)},
	861	) catch unreachable;
	862	try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
	863	file.stat.size,
	864	file.stat.inode,
	865	file.stat.mtime,
	866	&encoded_digest,
	867	file.prefixed_path.?.prefix,
	868	file.prefixed_path.?.sub_path,
	869	});
	870	}
	871
	872	try manifest_file.setEndPos(contents.items.len);
	873	try manifest_file.pwriteAll(contents.items, 0);
	874	}
	875
	876	if (self.want_shared_lock) {
	877	try self.downgradeToSharedLock();
	878	}
	879	}
	880
	881	fn downgradeToSharedLock(self: *Manifest) !void {
	882	if (!self.have_exclusive_lock) return;
	883
	884	// WASI does not currently support flock, so we bypass it here.
	885	// TODO: If/when flock is supported on WASI, this check should be removed.
	886	// See https://github.com/WebAssembly/wasi-filesystem/issues/2
	887	if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
	888	const manifest_file = self.manifest_file.?;
	889	try manifest_file.downgradeLock();
	890	}
	891
	892	self.have_exclusive_lock = false;
	893	}
	894
	895	fn upgradeToExclusiveLock(self: *Manifest) !void {
	896	if (self.have_exclusive_lock) return;
	897	assert(self.manifest_file != null);
	898
	899	// WASI does not currently support flock, so we bypass it here.
	900	// TODO: If/when flock is supported on WASI, this check should be removed.
	901	// See https://github.com/WebAssembly/wasi-filesystem/issues/2
	902	if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
	903	const manifest_file = self.manifest_file.?;
	904	// Here we intentionally have a period where the lock is released, in case there are
	905	// other processes holding a shared lock.
	906	manifest_file.unlock();
	907	try manifest_file.lock(.Exclusive);
	908	}
	909	self.have_exclusive_lock = true;
	910	}
	911
	912	/// Obtain only the data needed to maintain a lock on the manifest file.
	913	/// The `Manifest` remains safe to deinit.
	914	/// Don't forget to call `writeManifest` before this!
	915	pub fn toOwnedLock(self: *Manifest) Lock {
	916	const lock: Lock = .{
	917	.manifest_file = self.manifest_file.?,
	918	};
	919
	920	self.manifest_file = null;
	921	return lock;
	922	}
	923
	924	/// Releases the manifest file and frees any memory the Manifest was using.
	925	/// `Manifest.hit` must be called first.
	926	/// Don't forget to call `writeManifest` before this!
	927	pub fn deinit(self: *Manifest) void {
	928	if (self.manifest_file) \|file\| {
	929	if (builtin.os.tag == .windows) {
	930	// See Lock.release for why this is required on Windows
	931	file.unlock();
	932	}
	933
	934	file.close();
	935	}
	936	for (self.files.items) \|*file\| {
	937	file.deinit(self.cache.gpa);
	938	}
	939	self.files.deinit(self.cache.gpa);
	940	}
	941	};
	942
	943	/// On operating systems that support symlinks, does a readlink. On other operating systems,
	944	/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
	945	/// it is treated as not supporting symlinks.
	946	pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
	947	if (builtin.os.tag == .windows) {
	948	return dir.readFile(sub_path, buffer);
	949	} else {
	950	return dir.readLink(sub_path, buffer);
	951	}
	952	}
	953
	954	/// On operating systems that support symlinks, does a symlink. On other operating systems,
	955	/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
	956	/// it is treated as not supporting symlinks.
	957	/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
	958	pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
	959	assert(data.len <= 255);
	960	if (builtin.os.tag == .windows) {
	961	return dir.writeFile(sub_path, data);
	962	} else {
	963	return dir.symLink(data, sub_path, .{});
	964	}
	965	}
	966
	967	fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
	968	var buf: [1024]u8 = undefined;
	969
	970	var hasher = hasher_init;
	971	while (true) {
	972	const bytes_read = try file.read(&buf);
	973	if (bytes_read == 0) break;
	974	hasher.update(buf[0..bytes_read]);
	975	}
	976
	977	hasher.final(bin_digest);
	978	}
	979
	980	// Create/Write a file, close it, then grab its stat.mtime timestamp.
	981	fn testGetCurrentFileTimestamp() !i128 {
	982	var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
	983	.read = true,
	984	.truncate = true,
	985	});
	986	defer file.close();
	987
	988	return (try file.stat()).mtime;
	989	}
	990
	991	test "cache file and then recall it" {
	992	if (builtin.os.tag == .wasi) {
	993	// https://github.com/ziglang/zig/issues/5437
	994	return error.SkipZigTest;
	995	}
	996
	997	const cwd = fs.cwd();
	998
	999	const temp_file = "test.txt";
	1000	const temp_manifest_dir = "temp_manifest_dir";
	1001
	1002	try cwd.writeFile(temp_file, "Hello, world!\n");
	1003
	1004	// Wait for file timestamps to tick
	1005	const initial_time = try testGetCurrentFileTimestamp();
	1006	while ((try testGetCurrentFileTimestamp()) == initial_time) {
	1007	std.time.sleep(1);
	1008	}
	1009
	1010	var digest1: [hex_digest_len]u8 = undefined;
	1011	var digest2: [hex_digest_len]u8 = undefined;
	1012
	1013	{
	1014	var cache = Cache{
	1015	.gpa = testing.allocator,
	1016	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	1017	};
	1018	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
	1019	defer cache.manifest_dir.close();
	1020
	1021	{
	1022	var ch = cache.obtain();
	1023	defer ch.deinit();
	1024
	1025	ch.hash.add(true);
	1026	ch.hash.add(@as(u16, 1234));
	1027	ch.hash.addBytes("1234");
	1028	_ = try ch.addFile(temp_file, null);
	1029
	1030	// There should be nothing in the cache
	1031	try testing.expectEqual(false, try ch.hit());
	1032
	1033	digest1 = ch.final();
	1034	try ch.writeManifest();
	1035	}
	1036	{
	1037	var ch = cache.obtain();
	1038	defer ch.deinit();
	1039
	1040	ch.hash.add(true);
	1041	ch.hash.add(@as(u16, 1234));
	1042	ch.hash.addBytes("1234");
	1043	_ = try ch.addFile(temp_file, null);
	1044
	1045	// Cache hit! We just "built" the same file
	1046	try testing.expect(try ch.hit());
	1047	digest2 = ch.final();
	1048
	1049	try testing.expectEqual(false, ch.have_exclusive_lock);
	1050	}
	1051
	1052	try testing.expectEqual(digest1, digest2);
	1053	}
	1054
	1055	try cwd.deleteTree(temp_manifest_dir);
	1056	try cwd.deleteFile(temp_file);
	1057	}
	1058
	1059	test "check that changing a file makes cache fail" {
	1060	if (builtin.os.tag == .wasi) {
	1061	// https://github.com/ziglang/zig/issues/5437
	1062	return error.SkipZigTest;
	1063	}
	1064	const cwd = fs.cwd();
	1065
	1066	const temp_file = "cache_hash_change_file_test.txt";
	1067	const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
	1068	const original_temp_file_contents = "Hello, world!\n";
	1069	const updated_temp_file_contents = "Hello, world; but updated!\n";
	1070
	1071	try cwd.deleteTree(temp_manifest_dir);
	1072	try cwd.deleteTree(temp_file);
	1073
	1074	try cwd.writeFile(temp_file, original_temp_file_contents);
	1075
	1076	// Wait for file timestamps to tick
	1077	const initial_time = try testGetCurrentFileTimestamp();
	1078	while ((try testGetCurrentFileTimestamp()) == initial_time) {
	1079	std.time.sleep(1);
	1080	}
	1081
	1082	var digest1: [hex_digest_len]u8 = undefined;
	1083	var digest2: [hex_digest_len]u8 = undefined;
	1084
	1085	{
	1086	var cache = Cache{
	1087	.gpa = testing.allocator,
	1088	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	1089	};
	1090	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
	1091	defer cache.manifest_dir.close();
	1092
	1093	{
	1094	var ch = cache.obtain();
	1095	defer ch.deinit();
	1096
	1097	ch.hash.addBytes("1234");
	1098	const temp_file_idx = try ch.addFile(temp_file, 100);
	1099
	1100	// There should be nothing in the cache
	1101	try testing.expectEqual(false, try ch.hit());
	1102
	1103	try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
	1104
	1105	digest1 = ch.final();
	1106
	1107	try ch.writeManifest();
	1108	}
	1109
	1110	try cwd.writeFile(temp_file, updated_temp_file_contents);
	1111
	1112	{
	1113	var ch = cache.obtain();
	1114	defer ch.deinit();
	1115
	1116	ch.hash.addBytes("1234");
	1117	const temp_file_idx = try ch.addFile(temp_file, 100);
	1118
	1119	// A file that we depend on has been updated, so the cache should not contain an entry for it
	1120	try testing.expectEqual(false, try ch.hit());
	1121
	1122	// The cache system does not keep the contents of re-hashed input files.
	1123	try testing.expect(ch.files.items[temp_file_idx].contents == null);
	1124
	1125	digest2 = ch.final();
	1126
	1127	try ch.writeManifest();
	1128	}
	1129
	1130	try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
	1131	}
	1132
	1133	try cwd.deleteTree(temp_manifest_dir);
	1134	try cwd.deleteTree(temp_file);
	1135	}
	1136
	1137	test "no file inputs" {
	1138	if (builtin.os.tag == .wasi) {
	1139	// https://github.com/ziglang/zig/issues/5437
	1140	return error.SkipZigTest;
	1141	}
	1142	const cwd = fs.cwd();
	1143	const temp_manifest_dir = "no_file_inputs_manifest_dir";
	1144	defer cwd.deleteTree(temp_manifest_dir) catch {};
	1145
	1146	var digest1: [hex_digest_len]u8 = undefined;
	1147	var digest2: [hex_digest_len]u8 = undefined;
	1148
	1149	var cache = Cache{
	1150	.gpa = testing.allocator,
	1151	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	1152	};
	1153	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
	1154	defer cache.manifest_dir.close();
	1155
	1156	{
	1157	var man = cache.obtain();
	1158	defer man.deinit();
	1159
	1160	man.hash.addBytes("1234");
	1161
	1162	// There should be nothing in the cache
	1163	try testing.expectEqual(false, try man.hit());
	1164
	1165	digest1 = man.final();
	1166
	1167	try man.writeManifest();
	1168	}
	1169	{
	1170	var man = cache.obtain();
	1171	defer man.deinit();
	1172
	1173	man.hash.addBytes("1234");
	1174
	1175	try testing.expect(try man.hit());
	1176	digest2 = man.final();
	1177	try testing.expectEqual(false, man.have_exclusive_lock);
	1178	}
	1179
	1180	try testing.expectEqual(digest1, digest2);
	1181	}
	1182
	1183	test "Manifest with files added after initial hash work" {
	1184	if (builtin.os.tag == .wasi) {
	1185	// https://github.com/ziglang/zig/issues/5437
	1186	return error.SkipZigTest;
	1187	}
	1188	const cwd = fs.cwd();
	1189
	1190	const temp_file1 = "cache_hash_post_file_test1.txt";
	1191	const temp_file2 = "cache_hash_post_file_test2.txt";
	1192	const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
	1193
	1194	try cwd.writeFile(temp_file1, "Hello, world!\n");
	1195	try cwd.writeFile(temp_file2, "Hello world the second!\n");
	1196
	1197	// Wait for file timestamps to tick
	1198	const initial_time = try testGetCurrentFileTimestamp();
	1199	while ((try testGetCurrentFileTimestamp()) == initial_time) {
	1200	std.time.sleep(1);
	1201	}
	1202
	1203	var digest1: [hex_digest_len]u8 = undefined;
	1204	var digest2: [hex_digest_len]u8 = undefined;
	1205	var digest3: [hex_digest_len]u8 = undefined;
	1206
	1207	{
	1208	var cache = Cache{
	1209	.gpa = testing.allocator,
	1210	.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
	1211	};
	1212	cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
	1213	defer cache.manifest_dir.close();
	1214
	1215	{
	1216	var ch = cache.obtain();
	1217	defer ch.deinit();
	1218
	1219	ch.hash.addBytes("1234");
	1220	_ = try ch.addFile(temp_file1, null);
	1221
	1222	// There should be nothing in the cache
	1223	try testing.expectEqual(false, try ch.hit());
	1224
	1225	_ = try ch.addFilePost(temp_file2);
	1226
	1227	digest1 = ch.final();
	1228	try ch.writeManifest();
	1229	}
	1230	{
	1231	var ch = cache.obtain();
	1232	defer ch.deinit();
	1233
	1234	ch.hash.addBytes("1234");
	1235	_ = try ch.addFile(temp_file1, null);
	1236
	1237	try testing.expect(try ch.hit());
	1238	digest2 = ch.final();
	1239
	1240	try testing.expectEqual(false, ch.have_exclusive_lock);
	1241	}
	1242	try testing.expect(mem.eql(u8, &digest1, &digest2));
	1243
	1244	// Modify the file added after initial hash
	1245	try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
	1246
	1247	// Wait for file timestamps to tick
	1248	const initial_time2 = try testGetCurrentFileTimestamp();
	1249	while ((try testGetCurrentFileTimestamp()) == initial_time2) {
	1250	std.time.sleep(1);
	1251	}
	1252
	1253	{
	1254	var ch = cache.obtain();
	1255	defer ch.deinit();
	1256
	1257	ch.hash.addBytes("1234");
	1258	_ = try ch.addFile(temp_file1, null);
	1259
	1260	// A file that we depend on has been updated, so the cache should not contain an entry for it
	1261	try testing.expectEqual(false, try ch.hit());
	1262
	1263	_ = try ch.addFilePost(temp_file2);
	1264
	1265	digest3 = ch.final();
	1266
	1267	try ch.writeManifest();
	1268	}
	1269
	1270	try testing.expect(!mem.eql(u8, &digest1, &digest3));
	1271	}
	1272
	1273	try cwd.deleteTree(temp_manifest_dir);
	1274	try cwd.deleteFile(temp_file1);
	1275	try cwd.deleteFile(temp_file2);
	1276	}

lib/std/Build/Cache/DepTokenizer.zig created+1069

...	...	@@ -0,0 +1,1069 @@
	1	const Tokenizer = @This();
	2
	3	index: usize = 0,
	4	bytes: []const u8,
	5	state: State = .lhs,
	6
	7	const std = @import("std");
	8	const testing = std.testing;
	9	const assert = std.debug.assert;
	10
	11	pub fn next(self: *Tokenizer) ?Token {
	12	var start = self.index;
	13	var must_resolve = false;
	14	while (self.index < self.bytes.len) {
	15	const char = self.bytes[self.index];
	16	switch (self.state) {
	17	.lhs => switch (char) {
	18	'\t', '\n', '\r', ' ' => {
	19	// silently ignore whitespace
	20	self.index += 1;
	21	},
	22	else => {
	23	start = self.index;
	24	self.state = .target;
	25	},
	26	},
	27	.target => switch (char) {
	28	'\t', '\n', '\r', ' ' => {
	29	return errorIllegalChar(.invalid_target, self.index, char);
	30	},
	31	'$' => {
	32	self.state = .target_dollar_sign;
	33	self.index += 1;
	34	},
	35	'\\' => {
	36	self.state = .target_reverse_solidus;
	37	self.index += 1;
	38	},
	39	':' => {
	40	self.state = .target_colon;
	41	self.index += 1;
	42	},
	43	else => {
	44	self.index += 1;
	45	},
	46	},
	47	.target_reverse_solidus => switch (char) {
	48	'\t', '\n', '\r' => {
	49	return errorIllegalChar(.bad_target_escape, self.index, char);
	50	},
	51	' ', '#', '\\' => {
	52	must_resolve = true;
	53	self.state = .target;
	54	self.index += 1;
	55	},
	56	'$' => {
	57	self.state = .target_dollar_sign;
	58	self.index += 1;
	59	},
	60	else => {
	61	self.state = .target;
	62	self.index += 1;
	63	},
	64	},
	65	.target_dollar_sign => switch (char) {
	66	'$' => {
	67	must_resolve = true;
	68	self.state = .target;
	69	self.index += 1;
	70	},
	71	else => {
	72	return errorIllegalChar(.expected_dollar_sign, self.index, char);
	73	},
	74	},
	75	.target_colon => switch (char) {
	76	'\n', '\r' => {
	77	const bytes = self.bytes[start .. self.index - 1];
	78	if (bytes.len != 0) {
	79	self.state = .lhs;
	80	return finishTarget(must_resolve, bytes);
	81	}
	82	// silently ignore null target
	83	self.state = .lhs;
	84	},
	85	'/', '\\' => {
	86	self.state = .target_colon_reverse_solidus;
	87	self.index += 1;
	88	},
	89	else => {
	90	const bytes = self.bytes[start .. self.index - 1];
	91	if (bytes.len != 0) {
	92	self.state = .rhs;
	93	return finishTarget(must_resolve, bytes);
	94	}
	95	// silently ignore null target
	96	self.state = .lhs;
	97	},
	98	},
	99	.target_colon_reverse_solidus => switch (char) {
	100	'\n', '\r' => {
	101	const bytes = self.bytes[start .. self.index - 2];
	102	if (bytes.len != 0) {
	103	self.state = .lhs;
	104	return finishTarget(must_resolve, bytes);
	105	}
	106	// silently ignore null target
	107	self.state = .lhs;
	108	},
	109	else => {
	110	self.state = .target;
	111	},
	112	},
	113	.rhs => switch (char) {
	114	'\t', ' ' => {
	115	// silently ignore horizontal whitespace
	116	self.index += 1;
	117	},
	118	'\n', '\r' => {
	119	self.state = .lhs;
	120	},
	121	'\\' => {
	122	self.state = .rhs_continuation;
	123	self.index += 1;
	124	},
	125	'"' => {
	126	self.state = .prereq_quote;
	127	self.index += 1;
	128	start = self.index;
	129	},
	130	else => {
	131	start = self.index;
	132	self.state = .prereq;
	133	},
	134	},
	135	.rhs_continuation => switch (char) {
	136	'\n' => {
	137	self.state = .rhs;
	138	self.index += 1;
	139	},
	140	'\r' => {
	141	self.state = .rhs_continuation_linefeed;
	142	self.index += 1;
	143	},
	144	else => {
	145	return errorIllegalChar(.continuation_eol, self.index, char);
	146	},
	147	},
	148	.rhs_continuation_linefeed => switch (char) {
	149	'\n' => {
	150	self.state = .rhs;
	151	self.index += 1;
	152	},
	153	else => {
	154	return errorIllegalChar(.continuation_eol, self.index, char);
	155	},
	156	},
	157	.prereq_quote => switch (char) {
	158	'"' => {
	159	self.index += 1;
	160	self.state = .rhs;
	161	return Token{ .prereq = self.bytes[start .. self.index - 1] };
	162	},
	163	else => {
	164	self.index += 1;
	165	},
	166	},
	167	.prereq => switch (char) {
	168	'\t', ' ' => {
	169	self.state = .rhs;
	170	return Token{ .prereq = self.bytes[start..self.index] };
	171	},
	172	'\n', '\r' => {
	173	self.state = .lhs;
	174	return Token{ .prereq = self.bytes[start..self.index] };
	175	},
	176	'\\' => {
	177	self.state = .prereq_continuation;
	178	self.index += 1;
	179	},
	180	else => {
	181	self.index += 1;
	182	},
	183	},
	184	.prereq_continuation => switch (char) {
	185	'\n' => {
	186	self.index += 1;
	187	self.state = .rhs;
	188	return Token{ .prereq = self.bytes[start .. self.index - 2] };
	189	},
	190	'\r' => {
	191	self.state = .prereq_continuation_linefeed;
	192	self.index += 1;
	193	},
	194	else => {
	195	// not continuation
	196	self.state = .prereq;
	197	self.index += 1;
	198	},
	199	},
	200	.prereq_continuation_linefeed => switch (char) {
	201	'\n' => {
	202	self.index += 1;
	203	self.state = .rhs;
	204	return Token{ .prereq = self.bytes[start .. self.index - 1] };
	205	},
	206	else => {
	207	return errorIllegalChar(.continuation_eol, self.index, char);
	208	},
	209	},
	210	}
	211	} else {
	212	switch (self.state) {
	213	.lhs,
	214	.rhs,
	215	.rhs_continuation,
	216	.rhs_continuation_linefeed,
	217	=> return null,
	218	.target => {
	219	return errorPosition(.incomplete_target, start, self.bytes[start..]);
	220	},
	221	.target_reverse_solidus,
	222	.target_dollar_sign,
	223	=> {
	224	const idx = self.index - 1;
	225	return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
	226	},
	227	.target_colon => {
	228	const bytes = self.bytes[start .. self.index - 1];
	229	if (bytes.len != 0) {
	230	self.index += 1;
	231	self.state = .rhs;
	232	return finishTarget(must_resolve, bytes);
	233	}
	234	// silently ignore null target
	235	self.state = .lhs;
	236	return null;
	237	},
	238	.target_colon_reverse_solidus => {
	239	const bytes = self.bytes[start .. self.index - 2];
	240	if (bytes.len != 0) {
	241	self.index += 1;
	242	self.state = .rhs;
	243	return finishTarget(must_resolve, bytes);
	244	}
	245	// silently ignore null target
	246	self.state = .lhs;
	247	return null;
	248	},
	249	.prereq_quote => {
	250	return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
	251	},
	252	.prereq => {
	253	self.state = .lhs;
	254	return Token{ .prereq = self.bytes[start..] };
	255	},
	256	.prereq_continuation => {
	257	self.state = .lhs;
	258	return Token{ .prereq = self.bytes[start .. self.index - 1] };
	259	},
	260	.prereq_continuation_linefeed => {
	261	self.state = .lhs;
	262	return Token{ .prereq = self.bytes[start .. self.index - 2] };
	263	},
	264	}
	265	}
	266	unreachable;
	267	}
	268
	269	fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token {
	270	return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
	271	}
	272
	273	fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token {
	274	return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
	275	}
	276
	277	fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
	278	return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
	279	}
	280
	281	const State = enum {
	282	lhs,
	283	target,
	284	target_reverse_solidus,
	285	target_dollar_sign,
	286	target_colon,
	287	target_colon_reverse_solidus,
	288	rhs,
	289	rhs_continuation,
	290	rhs_continuation_linefeed,
	291	prereq_quote,
	292	prereq,
	293	prereq_continuation,
	294	prereq_continuation_linefeed,
	295	};
	296
	297	pub const Token = union(enum) {
	298	target: []const u8,
	299	target_must_resolve: []const u8,
	300	prereq: []const u8,
	301
	302	incomplete_quoted_prerequisite: IndexAndBytes,
	303	incomplete_target: IndexAndBytes,
	304
	305	invalid_target: IndexAndChar,
	306	bad_target_escape: IndexAndChar,
	307	expected_dollar_sign: IndexAndChar,
	308	continuation_eol: IndexAndChar,
	309	incomplete_escape: IndexAndChar,
	310
	311	pub const IndexAndChar = struct {
	312	index: usize,
	313	char: u8,
	314	};
	315
	316	pub const IndexAndBytes = struct {
	317	index: usize,
	318	bytes: []const u8,
	319	};
	320
	321	/// Resolve escapes in target. Only valid with .target_must_resolve.
	322	pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
	323	const bytes = self.target_must_resolve; // resolve called on incorrect token
	324
	325	var state: enum { start, escape, dollar } = .start;
	326	for (bytes) \|c\| {
	327	switch (state) {
	328	.start => {
	329	switch (c) {
	330	'\\' => state = .escape,
	331	'$' => state = .dollar,
	332	else => try writer.writeByte(c),
	333	}
	334	},
	335	.escape => {
	336	switch (c) {
	337	' ', '#', '\\' => {},
	338	'$' => {
	339	try writer.writeByte('\\');
	340	state = .dollar;
	341	continue;
	342	},
	343	else => try writer.writeByte('\\'),
	344	}
	345	try writer.writeByte(c);
	346	state = .start;
	347	},
	348	.dollar => {
	349	try writer.writeByte('$');
	350	switch (c) {
	351	'$' => {},
	352	else => try writer.writeByte(c),
	353	}
	354	state = .start;
	355	},
	356	}
	357	}
	358	}
	359
	360	pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
	361	switch (self) {
	362	.target, .target_must_resolve, .prereq => unreachable, // not an error
	363	.incomplete_quoted_prerequisite,
	364	.incomplete_target,
	365	=> \|index_and_bytes\| {
	366	try writer.print("{s} '", .{self.errStr()});
	367	if (self == .incomplete_target) {
	368	const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
	369	try tmp.resolve(writer);
	370	} else {
	371	try printCharValues(writer, index_and_bytes.bytes);
	372	}
	373	try writer.print("' at position {d}", .{index_and_bytes.index});
	374	},
	375	.invalid_target,
	376	.bad_target_escape,
	377	.expected_dollar_sign,
	378	.continuation_eol,
	379	.incomplete_escape,
	380	=> \|index_and_char\| {
	381	try writer.writeAll("illegal char ");
	382	try printUnderstandableChar(writer, index_and_char.char);
	383	try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() });
	384	},
	385	}
	386	}
	387
	388	fn errStr(self: Token) []const u8 {
	389	return switch (self) {
	390	.target, .target_must_resolve, .prereq => unreachable, // not an error
	391	.incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
	392	.incomplete_target => "incomplete target",
	393	.invalid_target => "invalid target",
	394	.bad_target_escape => "bad target escape",
	395	.expected_dollar_sign => "expecting '$'",
	396	.continuation_eol => "continuation expecting end-of-line",
	397	.incomplete_escape => "incomplete escape",
	398	};
	399	}
	400	};
	401
	402	test "empty file" {
	403	try depTokenizer("", "");
	404	}
	405
	406	test "empty whitespace" {
	407	try depTokenizer("\n", "");
	408	try depTokenizer("\r", "");
	409	try depTokenizer("\r\n", "");
	410	try depTokenizer(" ", "");
	411	}
	412
	413	test "empty colon" {
	414	try depTokenizer(":", "");
	415	try depTokenizer("\n:", "");
	416	try depTokenizer("\r:", "");
	417	try depTokenizer("\r\n:", "");
	418	try depTokenizer(" :", "");
	419	}
	420
	421	test "empty target" {
	422	try depTokenizer("foo.o:", "target = {foo.o}");
	423	try depTokenizer(
	424	\\foo.o:
	425	\\bar.o:
	426	\\abcd.o:
	427	,
	428	\\target = {foo.o}
	429	\\target = {bar.o}
	430	\\target = {abcd.o}
	431	);
	432	}
	433
	434	test "whitespace empty target" {
	435	try depTokenizer("\nfoo.o:", "target = {foo.o}");
	436	try depTokenizer("\rfoo.o:", "target = {foo.o}");
	437	try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
	438	try depTokenizer(" foo.o:", "target = {foo.o}");
	439	}
	440
	441	test "escape empty target" {
	442	try depTokenizer("\\ foo.o:", "target = { foo.o}");
	443	try depTokenizer("\\#foo.o:", "target = {#foo.o}");
	444	try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
	445	try depTokenizer("$$foo.o:", "target = {$foo.o}");
	446	}
	447
	448	test "empty target linefeeds" {
	449	try depTokenizer("\n", "");
	450	try depTokenizer("\r\n", "");
	451
	452	const expect = "target = {foo.o}";
	453	try depTokenizer(
	454	\\foo.o:
	455	, expect);
	456	try depTokenizer(
	457	\\foo.o:
	458	\\
	459	, expect);
	460	try depTokenizer(
	461	\\foo.o:
	462	, expect);
	463	try depTokenizer(
	464	\\foo.o:
	465	\\
	466	, expect);
	467	}
	468
	469	test "empty target linefeeds + continuations" {
	470	const expect = "target = {foo.o}";
	471	try depTokenizer(
	472	\\foo.o:\
	473	, expect);
	474	try depTokenizer(
	475	\\foo.o:\
	476	\\
	477	, expect);
	478	try depTokenizer(
	479	\\foo.o:\
	480	, expect);
	481	try depTokenizer(
	482	\\foo.o:\
	483	\\
	484	, expect);
	485	}
	486
	487	test "empty target linefeeds + hspace + continuations" {
	488	const expect = "target = {foo.o}";
	489	try depTokenizer(
	490	\\foo.o: \
	491	, expect);
	492	try depTokenizer(
	493	\\foo.o: \
	494	\\
	495	, expect);
	496	try depTokenizer(
	497	\\foo.o: \
	498	, expect);
	499	try depTokenizer(
	500	\\foo.o: \
	501	\\
	502	, expect);
	503	}
	504
	505	test "prereq" {
	506	const expect =
	507	\\target = {foo.o}
	508	\\prereq = {foo.c}
	509	;
	510	try depTokenizer("foo.o: foo.c", expect);
	511	try depTokenizer(
	512	\\foo.o: \
	513	\\foo.c
	514	, expect);
	515	try depTokenizer(
	516	\\foo.o: \
	517	\\ foo.c
	518	, expect);
	519	try depTokenizer(
	520	\\foo.o: \
	521	\\ foo.c
	522	, expect);
	523	}
	524
	525	test "prereq continuation" {
	526	const expect =
	527	\\target = {foo.o}
	528	\\prereq = {foo.h}
	529	\\prereq = {bar.h}
	530	;
	531	try depTokenizer(
	532	\\foo.o: foo.h\
	533	\\bar.h
	534	, expect);
	535	try depTokenizer(
	536	\\foo.o: foo.h\
	537	\\bar.h
	538	, expect);
	539	}
	540
	541	test "multiple prereqs" {
	542	const expect =
	543	\\target = {foo.o}
	544	\\prereq = {foo.c}
	545	\\prereq = {foo.h}
	546	\\prereq = {bar.h}
	547	;
	548	try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
	549	try depTokenizer(
	550	\\foo.o: \
	551	\\foo.c foo.h bar.h
	552	, expect);
	553	try depTokenizer(
	554	\\foo.o: foo.c foo.h bar.h\
	555	, expect);
	556	try depTokenizer(
	557	\\foo.o: foo.c foo.h bar.h\
	558	\\
	559	, expect);
	560	try depTokenizer(
	561	\\foo.o: \
	562	\\foo.c \
	563	\\ foo.h\
	564	\\bar.h
	565	\\
	566	, expect);
	567	try depTokenizer(
	568	\\foo.o: \
	569	\\foo.c \
	570	\\ foo.h\
	571	\\bar.h\
	572	\\
	573	, expect);
	574	try depTokenizer(
	575	\\foo.o: \
	576	\\foo.c \
	577	\\ foo.h\
	578	\\bar.h\
	579	, expect);
	580	}
	581
	582	test "multiple targets and prereqs" {
	583	try depTokenizer(
	584	\\foo.o: foo.c
	585	\\bar.o: bar.c a.h b.h c.h
	586	\\abc.o: abc.c \
	587	\\ one.h two.h \
	588	\\ three.h four.h
	589	,
	590	\\target = {foo.o}
	591	\\prereq = {foo.c}
	592	\\target = {bar.o}
	593	\\prereq = {bar.c}
	594	\\prereq = {a.h}
	595	\\prereq = {b.h}
	596	\\prereq = {c.h}
	597	\\target = {abc.o}
	598	\\prereq = {abc.c}
	599	\\prereq = {one.h}
	600	\\prereq = {two.h}
	601	\\prereq = {three.h}
	602	\\prereq = {four.h}
	603	);
	604	try depTokenizer(
	605	\\ascii.o: ascii.c
	606	\\base64.o: base64.c stdio.h
	607	\\elf.o: elf.c a.h b.h c.h
	608	\\macho.o: \
	609	\\ macho.c\
	610	\\ a.h b.h c.h
	611	,
	612	\\target = {ascii.o}
	613	\\prereq = {ascii.c}
	614	\\target = {base64.o}
	615	\\prereq = {base64.c}
	616	\\prereq = {stdio.h}
	617	\\target = {elf.o}
	618	\\prereq = {elf.c}
	619	\\prereq = {a.h}
	620	\\prereq = {b.h}
	621	\\prereq = {c.h}
	622	\\target = {macho.o}
	623	\\prereq = {macho.c}
	624	\\prereq = {a.h}
	625	\\prereq = {b.h}
	626	\\prereq = {c.h}
	627	);
	628	try depTokenizer(
	629	\\a$$scii.o: ascii.c
	630	\\\\base64.o: "\base64.c" "s t#dio.h"
	631	\\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
	632	\\macho.o: \
	633	\\ "macho!.c" \
	634	\\ a.h b.h c.h
	635	,
	636	\\target = {a$scii.o}
	637	\\prereq = {ascii.c}
	638	\\target = {\base64.o}
	639	\\prereq = {\base64.c}
	640	\\prereq = {s t#dio.h}
	641	\\target = {e\lf.o}
	642	\\prereq = {e\lf.c}
	643	\\prereq = {a.h$$}
	644	\\prereq = {$$b.h c.h$$}
	645	\\target = {macho.o}
	646	\\prereq = {macho!.c}
	647	\\prereq = {a.h}
	648	\\prereq = {b.h}
	649	\\prereq = {c.h}
	650	);
	651	}
	652
	653	test "windows quoted prereqs" {
	654	try depTokenizer(
	655	\\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
	656	\\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
	657	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
	658	\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
	659	,
	660	\\target = {c:\foo.o}
	661	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
	662	\\target = {c:\foo2.o}
	663	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
	664	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
	665	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
	666	);
	667	}
	668
	669	test "windows mixed prereqs" {
	670	try depTokenizer(
	671	\\cimport.o: \
	672	\\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
	673	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
	674	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
	675	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
	676	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
	677	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
	678	\\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
	679	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
	680	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
	681	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
	682	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
	683	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
	684	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
	685	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
	686	\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
	687	\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
	688	,
	689	\\target = {cimport.o}
	690	\\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
	691	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
	692	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
	693	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
	694	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
	695	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
	696	\\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
	697	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
	698	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
	699	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
	700	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
	701	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
	702	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
	703	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
	704	\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
	705	\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
	706	);
	707	}
	708
	709	test "windows funky targets" {
	710	try depTokenizer(
	711	\\C:\Users\anon\foo.o:
	712	\\C:\Users\anon\foo\ .o:
	713	\\C:\Users\anon\foo\#.o:
	714	\\C:\Users\anon\foo$$.o:
	715	\\C:\Users\anon\\\ foo.o:
	716	\\C:\Users\anon\\#foo.o:
	717	\\C:\Users\anon\$$foo.o:
	718	\\C:\Users\anon\\\ \ \ \ \ foo.o:
	719	,
	720	\\target = {C:\Users\anon\foo.o}
	721	\\target = {C:\Users\anon\foo .o}
	722	\\target = {C:\Users\anon\foo#.o}
	723	\\target = {C:\Users\anon\foo$.o}
	724	\\target = {C:\Users\anon\ foo.o}
	725	\\target = {C:\Users\anon\#foo.o}
	726	\\target = {C:\Users\anon\$foo.o}
	727	\\target = {C:\Users\anon\ foo.o}
	728	);
	729	}
	730
	731	test "windows drive and forward slashes" {
	732	try depTokenizer(
	733	\\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
	734	\\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c
	735	,
	736	\\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj}
	737	\\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c}
	738	);
	739	}
	740
	741	test "error incomplete escape - reverse_solidus" {
	742	try depTokenizer("\\",
	743	\\ERROR: illegal char '\' at position 0: incomplete escape
	744	);
	745	try depTokenizer("\t\\",
	746	\\ERROR: illegal char '\' at position 1: incomplete escape
	747	);
	748	try depTokenizer("\n\\",
	749	\\ERROR: illegal char '\' at position 1: incomplete escape
	750	);
	751	try depTokenizer("\r\\",
	752	\\ERROR: illegal char '\' at position 1: incomplete escape
	753	);
	754	try depTokenizer("\r\n\\",
	755	\\ERROR: illegal char '\' at position 2: incomplete escape
	756	);
	757	try depTokenizer(" \\",
	758	\\ERROR: illegal char '\' at position 1: incomplete escape
	759	);
	760	}
	761
	762	test "error incomplete escape - dollar_sign" {
	763	try depTokenizer("$",
	764	\\ERROR: illegal char '$' at position 0: incomplete escape
	765	);
	766	try depTokenizer("\t$",
	767	\\ERROR: illegal char '$' at position 1: incomplete escape
	768	);
	769	try depTokenizer("\n$",
	770	\\ERROR: illegal char '$' at position 1: incomplete escape
	771	);
	772	try depTokenizer("\r$",
	773	\\ERROR: illegal char '$' at position 1: incomplete escape
	774	);
	775	try depTokenizer("\r\n$",
	776	\\ERROR: illegal char '$' at position 2: incomplete escape
	777	);
	778	try depTokenizer(" $",
	779	\\ERROR: illegal char '$' at position 1: incomplete escape
	780	);
	781	}
	782
	783	test "error incomplete target" {
	784	try depTokenizer("foo.o",
	785	\\ERROR: incomplete target 'foo.o' at position 0
	786	);
	787	try depTokenizer("\tfoo.o",
	788	\\ERROR: incomplete target 'foo.o' at position 1
	789	);
	790	try depTokenizer("\nfoo.o",
	791	\\ERROR: incomplete target 'foo.o' at position 1
	792	);
	793	try depTokenizer("\rfoo.o",
	794	\\ERROR: incomplete target 'foo.o' at position 1
	795	);
	796	try depTokenizer("\r\nfoo.o",
	797	\\ERROR: incomplete target 'foo.o' at position 2
	798	);
	799	try depTokenizer(" foo.o",
	800	\\ERROR: incomplete target 'foo.o' at position 1
	801	);
	802
	803	try depTokenizer("\\ foo.o",
	804	\\ERROR: incomplete target ' foo.o' at position 0
	805	);
	806	try depTokenizer("\\#foo.o",
	807	\\ERROR: incomplete target '#foo.o' at position 0
	808	);
	809	try depTokenizer("\\\\foo.o",
	810	\\ERROR: incomplete target '\foo.o' at position 0
	811	);
	812	try depTokenizer("$$foo.o",
	813	\\ERROR: incomplete target '$foo.o' at position 0
	814	);
	815	}
	816
	817	test "error illegal char at position - bad target escape" {
	818	try depTokenizer("\\\t",
	819	\\ERROR: illegal char \x09 at position 1: bad target escape
	820	);
	821	try depTokenizer("\\\n",
	822	\\ERROR: illegal char \x0A at position 1: bad target escape
	823	);
	824	try depTokenizer("\\\r",
	825	\\ERROR: illegal char \x0D at position 1: bad target escape
	826	);
	827	try depTokenizer("\\\r\n",
	828	\\ERROR: illegal char \x0D at position 1: bad target escape
	829	);
	830	}
	831
	832	test "error illegal char at position - execting dollar_sign" {
	833	try depTokenizer("$\t",
	834	\\ERROR: illegal char \x09 at position 1: expecting '$'
	835	);
	836	try depTokenizer("$\n",
	837	\\ERROR: illegal char \x0A at position 1: expecting '$'
	838	);
	839	try depTokenizer("$\r",
	840	\\ERROR: illegal char \x0D at position 1: expecting '$'
	841	);
	842	try depTokenizer("$\r\n",
	843	\\ERROR: illegal char \x0D at position 1: expecting '$'
	844	);
	845	}
	846
	847	test "error illegal char at position - invalid target" {
	848	try depTokenizer("foo\t.o",
	849	\\ERROR: illegal char \x09 at position 3: invalid target
	850	);
	851	try depTokenizer("foo\n.o",
	852	\\ERROR: illegal char \x0A at position 3: invalid target
	853	);
	854	try depTokenizer("foo\r.o",
	855	\\ERROR: illegal char \x0D at position 3: invalid target
	856	);
	857	try depTokenizer("foo\r\n.o",
	858	\\ERROR: illegal char \x0D at position 3: invalid target
	859	);
	860	}
	861
	862	test "error target - continuation expecting end-of-line" {
	863	try depTokenizer("foo.o: \\\t",
	864	\\target = {foo.o}
	865	\\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
	866	);
	867	try depTokenizer("foo.o: \\ ",
	868	\\target = {foo.o}
	869	\\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line
	870	);
	871	try depTokenizer("foo.o: \\x",
	872	\\target = {foo.o}
	873	\\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
	874	);
	875	try depTokenizer("foo.o: \\\x0dx",
	876	\\target = {foo.o}
	877	\\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
	878	);
	879	}
	880
	881	test "error prereq - continuation expecting end-of-line" {
	882	try depTokenizer("foo.o: foo.h\\\x0dx",
	883	\\target = {foo.o}
	884	\\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
	885	);
	886	}
	887
	888	// - tokenize input, emit textual representation, and compare to expect
	889	fn depTokenizer(input: []const u8, expect: []const u8) !void {
	890	var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
	891	const arena = arena_allocator.allocator();
	892	defer arena_allocator.deinit();
	893
	894	var it: Tokenizer = .{ .bytes = input };
	895	var buffer = std.ArrayList(u8).init(arena);
	896	var resolve_buf = std.ArrayList(u8).init(arena);
	897	var i: usize = 0;
	898	while (it.next()) \|token\| {
	899	if (i != 0) try buffer.appendSlice("\n");
	900	switch (token) {
	901	.target, .prereq => \|bytes\| {
	902	try buffer.appendSlice(@tagName(token));
	903	try buffer.appendSlice(" = {");
	904	for (bytes) \|b\| {
	905	try buffer.append(printable_char_tab[b]);
	906	}
	907	try buffer.appendSlice("}");
	908	},
	909	.target_must_resolve => {
	910	try buffer.appendSlice("target = {");
	911	try token.resolve(resolve_buf.writer());
	912	for (resolve_buf.items) \|b\| {
	913	try buffer.append(printable_char_tab[b]);
	914	}
	915	resolve_buf.items.len = 0;
	916	try buffer.appendSlice("}");
	917	},
	918	else => {
	919	try buffer.appendSlice("ERROR: ");
	920	try token.printError(buffer.writer());
	921	break;
	922	},
	923	}
	924	i += 1;
	925	}
	926
	927	if (std.mem.eql(u8, expect, buffer.items)) {
	928	try testing.expect(true);
	929	return;
	930	}
	931
	932	const out = std.io.getStdErr().writer();
	933
	934	try out.writeAll("\n");
	935	try printSection(out, "<<<< input", input);
	936	try printSection(out, "==== expect", expect);
	937	try printSection(out, ">>>> got", buffer.items);
	938	try printRuler(out);
	939
	940	try testing.expect(false);
	941	}
	942
	943	fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
	944	try printLabel(out, label, bytes);
	945	try hexDump(out, bytes);
	946	try printRuler(out);
	947	try out.writeAll(bytes);
	948	try out.writeAll("\n");
	949	}
	950
	951	fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
	952	var buf: [80]u8 = undefined;
	953	var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len });
	954	try out.writeAll(text);
	955	var i: usize = text.len;
	956	const end = 79;
	957	while (i < end) : (i += 1) {
	958	try out.writeAll(&[_]u8{label[0]});
	959	}
	960	try out.writeAll("\n");
	961	}
	962
	963	fn printRuler(out: anytype) !void {
	964	var i: usize = 0;
	965	const end = 79;
	966	while (i < end) : (i += 1) {
	967	try out.writeAll("-");
	968	}
	969	try out.writeAll("\n");
	970	}
	971
	972	fn hexDump(out: anytype, bytes: []const u8) !void {
	973	const n16 = bytes.len >> 4;
	974	var line: usize = 0;
	975	var offset: usize = 0;
	976	while (line < n16) : (line += 1) {
	977	try hexDump16(out, offset, bytes[offset .. offset + 16]);
	978	offset += 16;
	979	}
	980
	981	const n = bytes.len & 0x0f;
	982	if (n > 0) {
	983	try printDecValue(out, offset, 8);
	984	try out.writeAll(":");
	985	try out.writeAll(" ");
	986	var end1 = std.math.min(offset + n, offset + 8);
	987	for (bytes[offset..end1]) \|b\| {
	988	try out.writeAll(" ");
	989	try printHexValue(out, b, 2);
	990	}
	991	var end2 = offset + n;
	992	if (end2 > end1) {
	993	try out.writeAll(" ");
	994	for (bytes[end1..end2]) \|b\| {
	995	try out.writeAll(" ");
	996	try printHexValue(out, b, 2);
	997	}
	998	}
	999	const short = 16 - n;
	1000	var i: usize = 0;
	1001	while (i < short) : (i += 1) {
	1002	try out.writeAll(" ");
	1003	}
	1004	if (end2 > end1) {
	1005	try out.writeAll(" \|");
	1006	} else {
	1007	try out.writeAll(" \|");
	1008	}
	1009	try printCharValues(out, bytes[offset..end2]);
	1010	try out.writeAll("\|\n");
	1011	offset += n;
	1012	}
	1013
	1014	try printDecValue(out, offset, 8);
	1015	try out.writeAll(":");
	1016	try out.writeAll("\n");
	1017	}
	1018
	1019	fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
	1020	try printDecValue(out, offset, 8);
	1021	try out.writeAll(":");
	1022	try out.writeAll(" ");
	1023	for (bytes[0..8]) \|b\| {
	1024	try out.writeAll(" ");
	1025	try printHexValue(out, b, 2);
	1026	}
	1027	try out.writeAll(" ");
	1028	for (bytes[8..16]) \|b\| {
	1029	try out.writeAll(" ");
	1030	try printHexValue(out, b, 2);
	1031	}
	1032	try out.writeAll(" \|");
	1033	try printCharValues(out, bytes);
	1034	try out.writeAll("\|\n");
	1035	}
	1036
	1037	fn printDecValue(out: anytype, value: u64, width: u8) !void {
	1038	var buffer: [20]u8 = undefined;
	1039	const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' });
	1040	try out.writeAll(buffer[0..len]);
	1041	}
	1042
	1043	fn printHexValue(out: anytype, value: u64, width: u8) !void {
	1044	var buffer: [16]u8 = undefined;
	1045	const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' });
	1046	try out.writeAll(buffer[0..len]);
	1047	}
	1048
	1049	fn printCharValues(out: anytype, bytes: []const u8) !void {
	1050	for (bytes) \|b\| {
	1051	try out.writeAll(&[_]u8{printable_char_tab[b]});
	1052	}
	1053	}
	1054
	1055	fn printUnderstandableChar(out: anytype, char: u8) !void {
	1056	if (std.ascii.isPrint(char)) {
	1057	try out.print("'{c}'", .{char});
	1058	} else {
	1059	try out.print("\\x{X:0>2}", .{char});
	1060	}
	1061	}
	1062
	1063	// zig fmt: off
	1064	const printable_char_tab: [256]u8 = (
	1065	"................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
	1066	"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{\|}~." ++
	1067	"................................................................" ++
	1068	"................................................................"
	1069	).*;

src/Cache.zig deleted-1265

...	...	@@ -1,1265 +0,0 @@
1		//! Manages `zig-cache` directories.
2		//! This is not a general-purpose cache. It is designed to be fast and simple,
3		//! not to withstand attacks using specially-crafted input.
4
5		gpa: Allocator,
6		manifest_dir: fs.Dir,
7		hash: HashHelper = .{},
8		/// This value is accessed from multiple threads, protected by mutex.
9		recent_problematic_timestamp: i128 = 0,
10		mutex: std.Thread.Mutex = .{},
11
12		/// A set of strings such as the zig library directory or project source root, which
13		/// are stripped from the file paths before putting into the cache. They
14		/// are replaced with single-character indicators. This is not to save
15		/// space but to eliminate absolute file paths. This improves portability
16		/// and usefulness of the cache for advanced use cases.
17		prefixes_buffer: [3]Compilation.Directory = undefined,
18		prefixes_len: usize = 0,
19
20		const Cache = @This();
21		const std = @import("std");
22		const builtin = @import("builtin");
23		const crypto = std.crypto;
24		const fs = std.fs;
25		const assert = std.debug.assert;
26		const testing = std.testing;
27		const mem = std.mem;
28		const fmt = std.fmt;
29		const Allocator = std.mem.Allocator;
30		const Compilation = @import("Compilation.zig");
31		const log = std.log.scoped(.cache);
32
33		pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void {
34		if (directory.path) \|p\| {
35		log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p });
36		}
37		cache.prefixes_buffer[cache.prefixes_len] = directory;
38		cache.prefixes_len += 1;
39		}
40
41		/// Be sure to call `Manifest.deinit` after successful initialization.
42		pub fn obtain(cache: *Cache) Manifest {
43		return Manifest{
44		.cache = cache,
45		.hash = cache.hash,
46		.manifest_file = null,
47		.manifest_dirty = false,
48		.hex_digest = undefined,
49		};
50		}
51
52		pub fn prefixes(cache: *const Cache) []const Compilation.Directory {
53		return cache.prefixes_buffer[0..cache.prefixes_len];
54		}
55
56		const PrefixedPath = struct {
57		prefix: u8,
58		sub_path: []u8,
59		};
60
61		fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
62		const gpa = cache.gpa;
63		const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
64		errdefer gpa.free(resolved_path);
65		return findPrefixResolved(cache, resolved_path);
66		}
67
68		/// Takes ownership of `resolved_path` on success.
69		fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
70		const gpa = cache.gpa;
71		const prefixes_slice = cache.prefixes();
72		var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
73		while (i < prefixes_slice.len) : (i += 1) {
74		const p = prefixes_slice[i].path.?;
75		if (mem.startsWith(u8, resolved_path, p)) {
76		// +1 to skip over the path separator here
77		const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
78		gpa.free(resolved_path);
79		return PrefixedPath{
80		.prefix = @intCast(u8, i),
81		.sub_path = sub_path,
82		};
83		} else {
84		log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p });
85		}
86		}
87
88		return PrefixedPath{
89		.prefix = 0,
90		.sub_path = resolved_path,
91		};
92		}
93
94		/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
95		pub const bin_digest_len = 16;
96		pub const hex_digest_len = bin_digest_len * 2;
97		pub const BinDigest = [bin_digest_len]u8;
98
99		const manifest_file_size_max = 50 * 1024 * 1024;
100
101		/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it
102		/// provides enough collision resistance for the Manifest use cases, while being one of our
103		/// fastest options right now.
104		pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
105
106		/// Initial state, that can be copied.
107		pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
108
109		pub const File = struct {
110		prefixed_path: ?PrefixedPath,
111		max_file_size: ?usize,
112		stat: Stat,
113		bin_digest: BinDigest,
114		contents: ?[]const u8,
115
116		pub const Stat = struct {
117		inode: fs.File.INode,
118		size: u64,
119		mtime: i128,
120		};
121
122		pub fn deinit(self: *File, gpa: Allocator) void {
123		if (self.prefixed_path) \|pp\| {
124		gpa.free(pp.sub_path);
125		self.prefixed_path = null;
126		}
127		if (self.contents) \|contents\| {
128		gpa.free(contents);
129		self.contents = null;
130		}
131		self.* = undefined;
132		}
133		};
134
135		pub const HashHelper = struct {
136		hasher: Hasher = hasher_init,
137
138		const EmitLoc = Compilation.EmitLoc;
139
140		/// Record a slice of bytes as an dependency of the process being cached
141		pub fn addBytes(hh: *HashHelper, bytes: []const u8) void {
142		hh.hasher.update(mem.asBytes(&bytes.len));
143		hh.hasher.update(bytes);
144		}
145
146		pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void {
147		hh.add(optional_bytes != null);
148		hh.addBytes(optional_bytes orelse return);
149		}
150
151		pub fn addEmitLoc(hh: *HashHelper, emit_loc: EmitLoc) void {
152		hh.addBytes(emit_loc.basename);
153		}
154
155		pub fn addOptionalEmitLoc(hh: *HashHelper, optional_emit_loc: ?EmitLoc) void {
156		hh.add(optional_emit_loc != null);
157		hh.addEmitLoc(optional_emit_loc orelse return);
158		}
159
160		pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void {
161		hh.add(list_of_bytes.len);
162		for (list_of_bytes) \|bytes\| hh.addBytes(bytes);
163		}
164
165		/// Convert the input value into bytes and record it as a dependency of the process being cached.
166		pub fn add(hh: *HashHelper, x: anytype) void {
167		switch (@TypeOf(x)) {
168		std.builtin.Version => {
169		hh.add(x.major);
170		hh.add(x.minor);
171		hh.add(x.patch);
172		},
173		std.Target.Os.TaggedVersionRange => {
174		switch (x) {
175		.linux => \|linux\| {
176		hh.add(linux.range.min);
177		hh.add(linux.range.max);
178		hh.add(linux.glibc);
179		},
180		.windows => \|windows\| {
181		hh.add(windows.min);
182		hh.add(windows.max);
183		},
184		.semver => \|semver\| {
185		hh.add(semver.min);
186		hh.add(semver.max);
187		},
188		.none => {},
189		}
190		},
191		else => switch (@typeInfo(@TypeOf(x))) {
192		.Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)),
193		else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))),
194		},
195		}
196		}
197
198		pub fn addOptional(hh: *HashHelper, optional: anytype) void {
199		hh.add(optional != null);
200		hh.add(optional orelse return);
201		}
202
203		/// Returns a hex encoded hash of the inputs, without modifying state.
204		pub fn peek(hh: HashHelper) [hex_digest_len]u8 {
205		var copy = hh;
206		return copy.final();
207		}
208
209		pub fn peekBin(hh: HashHelper) BinDigest {
210		var copy = hh;
211		var bin_digest: BinDigest = undefined;
212		copy.hasher.final(&bin_digest);
213		return bin_digest;
214		}
215
216		/// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
217		pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
218		var bin_digest: BinDigest = undefined;
219		hh.hasher.final(&bin_digest);
220
221		var out_digest: [hex_digest_len]u8 = undefined;
222		_ = std.fmt.bufPrint(
223		&out_digest,
224		"{s}",
225		.{std.fmt.fmtSliceHexLower(&bin_digest)},
226		) catch unreachable;
227		return out_digest;
228		}
229		};
230
231		pub const Lock = struct {
232		manifest_file: fs.File,
233
234		pub fn release(lock: *Lock) void {
235		if (builtin.os.tag == .windows) {
236		// Windows does not guarantee that locks are immediately unlocked when
237		// the file handle is closed. See LockFileEx documentation.
238		lock.manifest_file.unlock();
239		}
240
241		lock.manifest_file.close();
242		lock.* = undefined;
243		}
244		};
245
246		pub const Manifest = struct {
247		cache: *Cache,
248		/// Current state for incremental hashing.
249		hash: HashHelper,
250		manifest_file: ?fs.File,
251		manifest_dirty: bool,
252		/// Set this flag to true before calling hit() in order to indicate that
253		/// upon a cache hit, the code using the cache will not modify the files
254		/// within the cache directory. This allows multiple processes to utilize
255		/// the same cache directory at the same time.
256		want_shared_lock: bool = true,
257		have_exclusive_lock: bool = false,
258		// Indicate that we want isProblematicTimestamp to perform a filesystem write in
259		// order to obtain a problematic timestamp for the next call. Calls after that
260		// will then use the same timestamp, to avoid unnecessary filesystem writes.
261		want_refresh_timestamp: bool = true,
262		files: std.ArrayListUnmanaged(File) = .{},
263		hex_digest: [hex_digest_len]u8,
264		/// Populated when hit() returns an error because of one
265		/// of the files listed in the manifest.
266		failed_file_index: ?usize = null,
267		/// Keeps track of the last time we performed a file system write to observe
268		/// what time the file system thinks it is, according to its own granularity.
269		recent_problematic_timestamp: i128 = 0,
270
271		/// Add a file as a dependency of process being cached. When `hit` is
272		/// called, the file's contents will be checked to ensure that it matches
273		/// the contents from previous times.
274		///
275		/// Max file size will be used to determine the amount of space the file contents
276		/// are allowed to take up in memory. If max_file_size is null, then the contents
277		/// will not be loaded into memory.
278		///
279		/// Returns the index of the entry in the `files` array list. You can use it
280		/// to access the contents of the file after calling `hit()` like so:
281		///
282		/// ```
283		/// var file_contents = cache_hash.files.items[file_index].contents.?;
284		/// ```
285		pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
286		assert(self.manifest_file == null);
287
288		const gpa = self.cache.gpa;
289		try self.files.ensureUnusedCapacity(gpa, 1);
290		const prefixed_path = try self.cache.findPrefix(file_path);
291		errdefer gpa.free(prefixed_path.sub_path);
292
293		log.debug("Manifest.addFile {s} -> {d} {s}", .{
294		file_path, prefixed_path.prefix, prefixed_path.sub_path,
295		});
296
297		self.files.addOneAssumeCapacity().* = .{
298		.prefixed_path = prefixed_path,
299		.contents = null,
300		.max_file_size = max_file_size,
301		.stat = undefined,
302		.bin_digest = undefined,
303		};
304
305		self.hash.add(prefixed_path.prefix);
306		self.hash.addBytes(prefixed_path.sub_path);
307
308		return self.files.items.len - 1;
309		}
310
311		pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void {
312		_ = try self.addFile(c_source.src_path, null);
313		// Hash the extra flags, with special care to call addFile for file parameters.
314		// TODO this logic can likely be improved by utilizing clang_options_data.zig.
315		const file_args = [_][]const u8{"-include"};
316		var arg_i: usize = 0;
317		while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
318		const arg = c_source.extra_flags[arg_i];
319		self.hash.addBytes(arg);
320		for (file_args) \|file_arg\| {
321		if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
322		arg_i += 1;
323		_ = try self.addFile(c_source.extra_flags[arg_i], null);
324		}
325		}
326		}
327		}
328
329		pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void {
330		self.hash.add(optional_file_path != null);
331		const file_path = optional_file_path orelse return;
332		_ = try self.addFile(file_path, null);
333		}
334
335		pub fn addListOfFiles(self: *Manifest, list_of_files: []const []const u8) !void {
336		self.hash.add(list_of_files.len);
337		for (list_of_files) \|file_path\| {
338		_ = try self.addFile(file_path, null);
339		}
340		}
341
342		/// Check the cache to see if the input exists in it. If it exists, returns `true`.
343		/// A hex encoding of its hash is available by calling `final`.
344		///
345		/// This function will also acquire an exclusive lock to the manifest file. This means
346		/// that a process holding a Manifest will block any other process attempting to
347		/// acquire the lock. If `want_shared_lock` is `true`, a cache hit guarantees the
348		/// manifest file to be locked in shared mode, and a cache miss guarantees the manifest
349		/// file to be locked in exclusive mode.
350		///
351		/// The lock on the manifest file is released when `deinit` is called. As another
352		/// option, one may call `toOwnedLock` to obtain a smaller object which can represent
353		/// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
354		pub fn hit(self: *Manifest) !bool {
355		const gpa = self.cache.gpa;
356		assert(self.manifest_file == null);
357
358		self.failed_file_index = null;
359
360		const ext = ".txt";
361		var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
362
363		var bin_digest: BinDigest = undefined;
364		self.hash.hasher.final(&bin_digest);
365
366		_ = std.fmt.bufPrint(
367		&self.hex_digest,
368		"{s}",
369		.{std.fmt.fmtSliceHexLower(&bin_digest)},
370		) catch unreachable;
371
372		self.hash.hasher = hasher_init;
373		self.hash.hasher.update(&bin_digest);
374
375		mem.copy(u8, &manifest_file_path, &self.hex_digest);
376		manifest_file_path[self.hex_digest.len..][0..ext.len].* = ext.*;
377
378		if (self.files.items.len == 0) {
379		// If there are no file inputs, we check if the manifest file exists instead of
380		// comparing the hashes on the files used for the cached item
381		while (true) {
382		if (self.cache.manifest_dir.openFile(&manifest_file_path, .{
383		.mode = .read_write,
384		.lock = .Exclusive,
385		.lock_nonblocking = self.want_shared_lock,
386		})) \|manifest_file\| {
387		self.manifest_file = manifest_file;
388		self.have_exclusive_lock = true;
389		break;
390		} else \|open_err\| switch (open_err) {
391		error.WouldBlock => {
392		self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
393		.lock = .Shared,
394		});
395		break;
396		},
397		error.FileNotFound => {
398		if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
399		.read = true,
400		.truncate = false,
401		.lock = .Exclusive,
402		.lock_nonblocking = self.want_shared_lock,
403		})) \|manifest_file\| {
404		self.manifest_file = manifest_file;
405		self.manifest_dirty = true;
406		self.have_exclusive_lock = true;
407		return false; // cache miss; exclusive lock already held
408		} else \|err\| switch (err) {
409		error.WouldBlock => continue,
410		else => \|e\| return e,
411		}
412		},
413		else => \|e\| return e,
414		}
415		}
416		} else {
417		if (self.cache.manifest_dir.createFile(&manifest_file_path, .{
418		.read = true,
419		.truncate = false,
420		.lock = .Exclusive,
421		.lock_nonblocking = self.want_shared_lock,
422		})) \|manifest_file\| {
423		self.manifest_file = manifest_file;
424		self.have_exclusive_lock = true;
425		} else \|err\| switch (err) {
426		error.WouldBlock => {
427		self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
428		.lock = .Shared,
429		});
430		},
431		else => \|e\| return e,
432		}
433		}
434
435		self.want_refresh_timestamp = true;
436
437		const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
438		defer gpa.free(file_contents);
439
440		const input_file_count = self.files.items.len;
441		var any_file_changed = false;
442		var line_iter = mem.tokenize(u8, file_contents, "\n");
443		var idx: usize = 0;
444		while (line_iter.next()) \|line\| {
445		defer idx += 1;
446
447		const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
448		const new = try self.files.addOne(gpa);
449		new.* = .{
450		.prefixed_path = null,
451		.contents = null,
452		.max_file_size = null,
453		.stat = undefined,
454		.bin_digest = undefined,
455		};
456		break :blk new;
457		};
458
459		var iter = mem.tokenize(u8, line, " ");
460		const size = iter.next() orelse return error.InvalidFormat;
461		const inode = iter.next() orelse return error.InvalidFormat;
462		const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
463		const digest_str = iter.next() orelse return error.InvalidFormat;
464		const prefix_str = iter.next() orelse return error.InvalidFormat;
465		const file_path = iter.rest();
466
467		cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
468		cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
469		cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
470		_ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
471		const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
472		if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
473
474		if (file_path.len == 0) {
475		return error.InvalidFormat;
476		}
477		if (cache_hash_file.prefixed_path) \|pp\| {
478		if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
479		return error.InvalidFormat;
480		}
481		}
482
483		if (cache_hash_file.prefixed_path == null) {
484		cache_hash_file.prefixed_path = .{
485		.prefix = prefix,
486		.sub_path = try gpa.dupe(u8, file_path),
487		};
488		}
489
490		const pp = cache_hash_file.prefixed_path.?;
491		const dir = self.cache.prefixes()[pp.prefix].handle;
492		const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch \|err\| switch (err) {
493		error.FileNotFound => {
494		try self.upgradeToExclusiveLock();
495		return false;
496		},
497		else => return error.CacheUnavailable,
498		};
499		defer this_file.close();
500
501		const actual_stat = this_file.stat() catch \|err\| {
502		self.failed_file_index = idx;
503		return err;
504		};
505		const size_match = actual_stat.size == cache_hash_file.stat.size;
506		const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
507		const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
508
509		if (!size_match or !mtime_match or !inode_match) {
510		self.manifest_dirty = true;
511
512		cache_hash_file.stat = .{
513		.size = actual_stat.size,
514		.mtime = actual_stat.mtime,
515		.inode = actual_stat.inode,
516		};
517
518		if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
519		// The actual file has an unreliable timestamp, force it to be hashed
520		cache_hash_file.stat.mtime = 0;
521		cache_hash_file.stat.inode = 0;
522		}
523
524		var actual_digest: BinDigest = undefined;
525		hashFile(this_file, &actual_digest) catch \|err\| {
526		self.failed_file_index = idx;
527		return err;
528		};
529
530		if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
531		cache_hash_file.bin_digest = actual_digest;
532		// keep going until we have the input file digests
533		any_file_changed = true;
534		}
535		}
536
537		if (!any_file_changed) {
538		self.hash.hasher.update(&cache_hash_file.bin_digest);
539		}
540		}
541
542		if (any_file_changed) {
543		// cache miss
544		// keep the manifest file open
545		self.unhit(bin_digest, input_file_count);
546		try self.upgradeToExclusiveLock();
547		return false;
548		}
549
550		if (idx < input_file_count) {
551		self.manifest_dirty = true;
552		while (idx < input_file_count) : (idx += 1) {
553		const ch_file = &self.files.items[idx];
554		self.populateFileHash(ch_file) catch \|err\| {
555		self.failed_file_index = idx;
556		return err;
557		};
558		}
559		try self.upgradeToExclusiveLock();
560		return false;
561		}
562
563		if (self.want_shared_lock) {
564		try self.downgradeToSharedLock();
565		}
566
567		return true;
568		}
569
570		pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
571		// Reset the hash.
572		self.hash.hasher = hasher_init;
573		self.hash.hasher.update(&bin_digest);
574
575		// Remove files not in the initial hash.
576		for (self.files.items[input_file_count..]) \|*file\| {
577		file.deinit(self.cache.gpa);
578		}
579		self.files.shrinkRetainingCapacity(input_file_count);
580
581		for (self.files.items) \|file\| {
582		self.hash.hasher.update(&file.bin_digest);
583		}
584		}
585
586		fn isProblematicTimestamp(man: *Manifest, file_time: i128) bool {
587		// If the file_time is prior to the most recent problematic timestamp
588		// then we don't need to access the filesystem.
589		if (file_time < man.recent_problematic_timestamp)
590		return false;
591
592		// Next we will check the globally shared Cache timestamp, which is accessed
593		// from multiple threads.
594		man.cache.mutex.lock();
595		defer man.cache.mutex.unlock();
596
597		// Save the global one to our local one to avoid locking next time.
598		man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
599		if (file_time < man.recent_problematic_timestamp)
600		return false;
601
602		// This flag prevents multiple filesystem writes for the same hit() call.
603		if (man.want_refresh_timestamp) {
604		man.want_refresh_timestamp = false;
605
606		var file = man.cache.manifest_dir.createFile("timestamp", .{
607		.read = true,
608		.truncate = true,
609		}) catch return true;
610		defer file.close();
611
612		// Save locally and also save globally (we still hold the global lock).
613		man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
614		man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
615		}
616
617		return file_time >= man.recent_problematic_timestamp;
618		}
619
620		fn populateFileHash(self: Manifest, ch_file: File) !void {
621		const pp = ch_file.prefixed_path.?;
622		const dir = self.cache.prefixes()[pp.prefix].handle;
623		const file = try dir.openFile(pp.sub_path, .{});
624		defer file.close();
625
626		const actual_stat = try file.stat();
627		ch_file.stat = .{
628		.size = actual_stat.size,
629		.mtime = actual_stat.mtime,
630		.inode = actual_stat.inode,
631		};
632
633		if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
634		// The actual file has an unreliable timestamp, force it to be hashed
635		ch_file.stat.mtime = 0;
636		ch_file.stat.inode = 0;
637		}
638
639		if (ch_file.max_file_size) \|max_file_size\| {
640		if (ch_file.stat.size > max_file_size) {
641		return error.FileTooBig;
642		}
643
644		const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size));
645		errdefer self.cache.gpa.free(contents);
646
647		// Hash while reading from disk, to keep the contents in the cpu cache while
648		// doing hashing.
649		var hasher = hasher_init;
650		var off: usize = 0;
651		while (true) {
652		// give me everything you've got, captain
653		const bytes_read = try file.read(contents[off..]);
654		if (bytes_read == 0) break;
655		hasher.update(contents[off..][0..bytes_read]);
656		off += bytes_read;
657		}
658		hasher.final(&ch_file.bin_digest);
659
660		ch_file.contents = contents;
661		} else {
662		try hashFile(file, &ch_file.bin_digest);
663		}
664
665		self.hash.hasher.update(&ch_file.bin_digest);
666		}
667
668		/// Add a file as a dependency of process being cached, after the initial hash has been
669		/// calculated. This is useful for processes that don't know all the files that
670		/// are depended on ahead of time. For example, a source file that can import other files
671		/// will need to be recompiled if the imported file is changed.
672		pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
673		assert(self.manifest_file != null);
674
675		const gpa = self.cache.gpa;
676		const prefixed_path = try self.cache.findPrefix(file_path);
677		errdefer gpa.free(prefixed_path.sub_path);
678
679		log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{
680		file_path, prefixed_path.prefix, prefixed_path.sub_path,
681		});
682
683		const new_ch_file = try self.files.addOne(gpa);
684		new_ch_file.* = .{
685		.prefixed_path = prefixed_path,
686		.max_file_size = max_file_size,
687		.stat = undefined,
688		.bin_digest = undefined,
689		.contents = null,
690		};
691		errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
692
693		try self.populateFileHash(new_ch_file);
694
695		return new_ch_file.contents.?;
696		}
697
698		/// Add a file as a dependency of process being cached, after the initial hash has been
699		/// calculated. This is useful for processes that don't know the all the files that
700		/// are depended on ahead of time. For example, a source file that can import other files
701		/// will need to be recompiled if the imported file is changed.
702		pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
703		assert(self.manifest_file != null);
704
705		const gpa = self.cache.gpa;
706		const prefixed_path = try self.cache.findPrefix(file_path);
707		errdefer gpa.free(prefixed_path.sub_path);
708
709		log.debug("Manifest.addFilePost {s} -> {d} {s}", .{
710		file_path, prefixed_path.prefix, prefixed_path.sub_path,
711		});
712
713		const new_ch_file = try self.files.addOne(gpa);
714		new_ch_file.* = .{
715		.prefixed_path = prefixed_path,
716		.max_file_size = null,
717		.stat = undefined,
718		.bin_digest = undefined,
719		.contents = null,
720		};
721		errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
722
723		try self.populateFileHash(new_ch_file);
724		}
725
726		/// Like `addFilePost` but when the file contents have already been loaded from disk.
727		/// On success, cache takes ownership of `resolved_path`.
728		pub fn addFilePostContents(
729		self: *Manifest,
730		resolved_path: []u8,
731		bytes: []const u8,
732		stat: File.Stat,
733		) error{OutOfMemory}!void {
734		assert(self.manifest_file != null);
735		const gpa = self.cache.gpa;
736
737		const ch_file = try self.files.addOne(gpa);
738		errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
739
740		log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path});
741
742		const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
743		errdefer gpa.free(prefixed_path.sub_path);
744
745		log.debug("Manifest.addFilePostContents -> {d} {s}", .{
746		prefixed_path.prefix, prefixed_path.sub_path,
747		});
748
749		ch_file.* = .{
750		.prefixed_path = prefixed_path,
751		.max_file_size = null,
752		.stat = stat,
753		.bin_digest = undefined,
754		.contents = null,
755		};
756
757		if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
758		// The actual file has an unreliable timestamp, force it to be hashed
759		ch_file.stat.mtime = 0;
760		ch_file.stat.inode = 0;
761		}
762
763		{
764		var hasher = hasher_init;
765		hasher.update(bytes);
766		hasher.final(&ch_file.bin_digest);
767		}
768
769		self.hash.hasher.update(&ch_file.bin_digest);
770		}
771
772		pub fn addDepFilePost(self: *Manifest, dir: fs.Dir, dep_file_basename: []const u8) !void {
773		assert(self.manifest_file != null);
774
775		const dep_file_contents = try dir.readFileAlloc(self.cache.gpa, dep_file_basename, manifest_file_size_max);
776		defer self.cache.gpa.free(dep_file_contents);
777
778		var error_buf = std.ArrayList(u8).init(self.cache.gpa);
779		defer error_buf.deinit();
780
781		var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents };
782
783		// Skip first token: target.
784		switch (it.next() orelse return) { // Empty dep file OK.
785		.target, .target_must_resolve, .prereq => {},
786		else => \|err\| {
787		try err.printError(error_buf.writer());
788		log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
789		return error.InvalidDepFile;
790		},
791		}
792		// Process 0+ preqreqs.
793		// Clang is invoked in single-source mode so we never get more targets.
794		while (true) {
795		switch (it.next() orelse return) {
796		.target, .target_must_resolve => return,
797		.prereq => \|file_path\| try self.addFilePost(file_path),
798		else => \|err\| {
799		try err.printError(error_buf.writer());
800		log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
801		return error.InvalidDepFile;
802		},
803		}
804		}
805		}
806
807		/// Returns a hex encoded hash of the inputs.
808		pub fn final(self: *Manifest) [hex_digest_len]u8 {
809		assert(self.manifest_file != null);
810
811		// We don't close the manifest file yet, because we want to
812		// keep it locked until the API user is done using it.
813		// We also don't write out the manifest yet, because until
814		// cache_release is called we still might be working on creating
815		// the artifacts to cache.
816
817		var bin_digest: BinDigest = undefined;
818		self.hash.hasher.final(&bin_digest);
819
820		var out_digest: [hex_digest_len]u8 = undefined;
821		_ = std.fmt.bufPrint(
822		&out_digest,
823		"{s}",
824		.{std.fmt.fmtSliceHexLower(&bin_digest)},
825		) catch unreachable;
826
827		return out_digest;
828		}
829
830		/// If `want_shared_lock` is true, this function automatically downgrades the
831		/// lock from exclusive to shared.
832		pub fn writeManifest(self: *Manifest) !void {
833		assert(self.have_exclusive_lock);
834
835		const manifest_file = self.manifest_file.?;
836		if (self.manifest_dirty) {
837		self.manifest_dirty = false;
838
839		var contents = std.ArrayList(u8).init(self.cache.gpa);
840		defer contents.deinit();
841
842		const writer = contents.writer();
843		var encoded_digest: [hex_digest_len]u8 = undefined;
844
845		for (self.files.items) \|file\| {
846		_ = std.fmt.bufPrint(
847		&encoded_digest,
848		"{s}",
849		.{std.fmt.fmtSliceHexLower(&file.bin_digest)},
850		) catch unreachable;
851		try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
852		file.stat.size,
853		file.stat.inode,
854		file.stat.mtime,
855		&encoded_digest,
856		file.prefixed_path.?.prefix,
857		file.prefixed_path.?.sub_path,
858		});
859		}
860
861		try manifest_file.setEndPos(contents.items.len);
862		try manifest_file.pwriteAll(contents.items, 0);
863		}
864
865		if (self.want_shared_lock) {
866		try self.downgradeToSharedLock();
867		}
868		}
869
870		fn downgradeToSharedLock(self: *Manifest) !void {
871		if (!self.have_exclusive_lock) return;
872
873		// WASI does not currently support flock, so we bypass it here.
874		// TODO: If/when flock is supported on WASI, this check should be removed.
875		// See https://github.com/WebAssembly/wasi-filesystem/issues/2
876		if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
877		const manifest_file = self.manifest_file.?;
878		try manifest_file.downgradeLock();
879		}
880
881		self.have_exclusive_lock = false;
882		}
883
884		fn upgradeToExclusiveLock(self: *Manifest) !void {
885		if (self.have_exclusive_lock) return;
886		assert(self.manifest_file != null);
887
888		// WASI does not currently support flock, so we bypass it here.
889		// TODO: If/when flock is supported on WASI, this check should be removed.
890		// See https://github.com/WebAssembly/wasi-filesystem/issues/2
891		if (builtin.os.tag != .wasi or std.process.can_spawn or !builtin.single_threaded) {
892		const manifest_file = self.manifest_file.?;
893		// Here we intentionally have a period where the lock is released, in case there are
894		// other processes holding a shared lock.
895		manifest_file.unlock();
896		try manifest_file.lock(.Exclusive);
897		}
898		self.have_exclusive_lock = true;
899		}
900
901		/// Obtain only the data needed to maintain a lock on the manifest file.
902		/// The `Manifest` remains safe to deinit.
903		/// Don't forget to call `writeManifest` before this!
904		pub fn toOwnedLock(self: *Manifest) Lock {
905		const lock: Lock = .{
906		.manifest_file = self.manifest_file.?,
907		};
908
909		self.manifest_file = null;
910		return lock;
911		}
912
913		/// Releases the manifest file and frees any memory the Manifest was using.
914		/// `Manifest.hit` must be called first.
915		/// Don't forget to call `writeManifest` before this!
916		pub fn deinit(self: *Manifest) void {
917		if (self.manifest_file) \|file\| {
918		if (builtin.os.tag == .windows) {
919		// See Lock.release for why this is required on Windows
920		file.unlock();
921		}
922
923		file.close();
924		}
925		for (self.files.items) \|*file\| {
926		file.deinit(self.cache.gpa);
927		}
928		self.files.deinit(self.cache.gpa);
929		}
930		};
931
932		/// On operating systems that support symlinks, does a readlink. On other operating systems,
933		/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
934		/// it is treated as not supporting symlinks.
935		pub fn readSmallFile(dir: fs.Dir, sub_path: []const u8, buffer: []u8) ![]u8 {
936		if (builtin.os.tag == .windows) {
937		return dir.readFile(sub_path, buffer);
938		} else {
939		return dir.readLink(sub_path, buffer);
940		}
941		}
942
943		/// On operating systems that support symlinks, does a symlink. On other operating systems,
944		/// uses the file contents. Windows supports symlinks but only with elevated privileges, so
945		/// it is treated as not supporting symlinks.
946		/// `data` must be a valid UTF-8 encoded file path and 255 bytes or fewer.
947		pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void {
948		assert(data.len <= 255);
949		if (builtin.os.tag == .windows) {
950		return dir.writeFile(sub_path, data);
951		} else {
952		return dir.symLink(data, sub_path, .{});
953		}
954		}
955
956		fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
957		var buf: [1024]u8 = undefined;
958
959		var hasher = hasher_init;
960		while (true) {
961		const bytes_read = try file.read(&buf);
962		if (bytes_read == 0) break;
963		hasher.update(buf[0..bytes_read]);
964		}
965
966		hasher.final(bin_digest);
967		}
968
969		// Create/Write a file, close it, then grab its stat.mtime timestamp.
970		fn testGetCurrentFileTimestamp() !i128 {
971		var file = try fs.cwd().createFile("test-filetimestamp.tmp", .{
972		.read = true,
973		.truncate = true,
974		});
975		defer file.close();
976
977		return (try file.stat()).mtime;
978		}
979
980		test "cache file and then recall it" {
981		if (builtin.os.tag == .wasi) {
982		// https://github.com/ziglang/zig/issues/5437
983		return error.SkipZigTest;
984		}
985
986		const cwd = fs.cwd();
987
988		const temp_file = "test.txt";
989		const temp_manifest_dir = "temp_manifest_dir";
990
991		try cwd.writeFile(temp_file, "Hello, world!\n");
992
993		// Wait for file timestamps to tick
994		const initial_time = try testGetCurrentFileTimestamp();
995		while ((try testGetCurrentFileTimestamp()) == initial_time) {
996		std.time.sleep(1);
997		}
998
999		var digest1: [hex_digest_len]u8 = undefined;
1000		var digest2: [hex_digest_len]u8 = undefined;
1001
1002		{
1003		var cache = Cache{
1004		.gpa = testing.allocator,
1005		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1006		};
1007		cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1008		defer cache.manifest_dir.close();
1009
1010		{
1011		var ch = cache.obtain();
1012		defer ch.deinit();
1013
1014		ch.hash.add(true);
1015		ch.hash.add(@as(u16, 1234));
1016		ch.hash.addBytes("1234");
1017		_ = try ch.addFile(temp_file, null);
1018
1019		// There should be nothing in the cache
1020		try testing.expectEqual(false, try ch.hit());
1021
1022		digest1 = ch.final();
1023		try ch.writeManifest();
1024		}
1025		{
1026		var ch = cache.obtain();
1027		defer ch.deinit();
1028
1029		ch.hash.add(true);
1030		ch.hash.add(@as(u16, 1234));
1031		ch.hash.addBytes("1234");
1032		_ = try ch.addFile(temp_file, null);
1033
1034		// Cache hit! We just "built" the same file
1035		try testing.expect(try ch.hit());
1036		digest2 = ch.final();
1037
1038		try testing.expectEqual(false, ch.have_exclusive_lock);
1039		}
1040
1041		try testing.expectEqual(digest1, digest2);
1042		}
1043
1044		try cwd.deleteTree(temp_manifest_dir);
1045		try cwd.deleteFile(temp_file);
1046		}
1047
1048		test "check that changing a file makes cache fail" {
1049		if (builtin.os.tag == .wasi) {
1050		// https://github.com/ziglang/zig/issues/5437
1051		return error.SkipZigTest;
1052		}
1053		const cwd = fs.cwd();
1054
1055		const temp_file = "cache_hash_change_file_test.txt";
1056		const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
1057		const original_temp_file_contents = "Hello, world!\n";
1058		const updated_temp_file_contents = "Hello, world; but updated!\n";
1059
1060		try cwd.deleteTree(temp_manifest_dir);
1061		try cwd.deleteTree(temp_file);
1062
1063		try cwd.writeFile(temp_file, original_temp_file_contents);
1064
1065		// Wait for file timestamps to tick
1066		const initial_time = try testGetCurrentFileTimestamp();
1067		while ((try testGetCurrentFileTimestamp()) == initial_time) {
1068		std.time.sleep(1);
1069		}
1070
1071		var digest1: [hex_digest_len]u8 = undefined;
1072		var digest2: [hex_digest_len]u8 = undefined;
1073
1074		{
1075		var cache = Cache{
1076		.gpa = testing.allocator,
1077		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1078		};
1079		cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1080		defer cache.manifest_dir.close();
1081
1082		{
1083		var ch = cache.obtain();
1084		defer ch.deinit();
1085
1086		ch.hash.addBytes("1234");
1087		const temp_file_idx = try ch.addFile(temp_file, 100);
1088
1089		// There should be nothing in the cache
1090		try testing.expectEqual(false, try ch.hit());
1091
1092		try testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
1093
1094		digest1 = ch.final();
1095
1096		try ch.writeManifest();
1097		}
1098
1099		try cwd.writeFile(temp_file, updated_temp_file_contents);
1100
1101		{
1102		var ch = cache.obtain();
1103		defer ch.deinit();
1104
1105		ch.hash.addBytes("1234");
1106		const temp_file_idx = try ch.addFile(temp_file, 100);
1107
1108		// A file that we depend on has been updated, so the cache should not contain an entry for it
1109		try testing.expectEqual(false, try ch.hit());
1110
1111		// The cache system does not keep the contents of re-hashed input files.
1112		try testing.expect(ch.files.items[temp_file_idx].contents == null);
1113
1114		digest2 = ch.final();
1115
1116		try ch.writeManifest();
1117		}
1118
1119		try testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
1120		}
1121
1122		try cwd.deleteTree(temp_manifest_dir);
1123		try cwd.deleteTree(temp_file);
1124		}
1125
1126		test "no file inputs" {
1127		if (builtin.os.tag == .wasi) {
1128		// https://github.com/ziglang/zig/issues/5437
1129		return error.SkipZigTest;
1130		}
1131		const cwd = fs.cwd();
1132		const temp_manifest_dir = "no_file_inputs_manifest_dir";
1133		defer cwd.deleteTree(temp_manifest_dir) catch {};
1134
1135		var digest1: [hex_digest_len]u8 = undefined;
1136		var digest2: [hex_digest_len]u8 = undefined;
1137
1138		var cache = Cache{
1139		.gpa = testing.allocator,
1140		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1141		};
1142		cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1143		defer cache.manifest_dir.close();
1144
1145		{
1146		var man = cache.obtain();
1147		defer man.deinit();
1148
1149		man.hash.addBytes("1234");
1150
1151		// There should be nothing in the cache
1152		try testing.expectEqual(false, try man.hit());
1153
1154		digest1 = man.final();
1155
1156		try man.writeManifest();
1157		}
1158		{
1159		var man = cache.obtain();
1160		defer man.deinit();
1161
1162		man.hash.addBytes("1234");
1163
1164		try testing.expect(try man.hit());
1165		digest2 = man.final();
1166		try testing.expectEqual(false, man.have_exclusive_lock);
1167		}
1168
1169		try testing.expectEqual(digest1, digest2);
1170		}
1171
1172		test "Manifest with files added after initial hash work" {
1173		if (builtin.os.tag == .wasi) {
1174		// https://github.com/ziglang/zig/issues/5437
1175		return error.SkipZigTest;
1176		}
1177		const cwd = fs.cwd();
1178
1179		const temp_file1 = "cache_hash_post_file_test1.txt";
1180		const temp_file2 = "cache_hash_post_file_test2.txt";
1181		const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
1182
1183		try cwd.writeFile(temp_file1, "Hello, world!\n");
1184		try cwd.writeFile(temp_file2, "Hello world the second!\n");
1185
1186		// Wait for file timestamps to tick
1187		const initial_time = try testGetCurrentFileTimestamp();
1188		while ((try testGetCurrentFileTimestamp()) == initial_time) {
1189		std.time.sleep(1);
1190		}
1191
1192		var digest1: [hex_digest_len]u8 = undefined;
1193		var digest2: [hex_digest_len]u8 = undefined;
1194		var digest3: [hex_digest_len]u8 = undefined;
1195
1196		{
1197		var cache = Cache{
1198		.gpa = testing.allocator,
1199		.manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
1200		};
1201		cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1202		defer cache.manifest_dir.close();
1203
1204		{
1205		var ch = cache.obtain();
1206		defer ch.deinit();
1207
1208		ch.hash.addBytes("1234");
1209		_ = try ch.addFile(temp_file1, null);
1210
1211		// There should be nothing in the cache
1212		try testing.expectEqual(false, try ch.hit());
1213
1214		_ = try ch.addFilePost(temp_file2);
1215
1216		digest1 = ch.final();
1217		try ch.writeManifest();
1218		}
1219		{
1220		var ch = cache.obtain();
1221		defer ch.deinit();
1222
1223		ch.hash.addBytes("1234");
1224		_ = try ch.addFile(temp_file1, null);
1225
1226		try testing.expect(try ch.hit());
1227		digest2 = ch.final();
1228
1229		try testing.expectEqual(false, ch.have_exclusive_lock);
1230		}
1231		try testing.expect(mem.eql(u8, &digest1, &digest2));
1232
1233		// Modify the file added after initial hash
1234		try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
1235
1236		// Wait for file timestamps to tick
1237		const initial_time2 = try testGetCurrentFileTimestamp();
1238		while ((try testGetCurrentFileTimestamp()) == initial_time2) {
1239		std.time.sleep(1);
1240		}
1241
1242		{
1243		var ch = cache.obtain();
1244		defer ch.deinit();
1245
1246		ch.hash.addBytes("1234");
1247		_ = try ch.addFile(temp_file1, null);
1248
1249		// A file that we depend on has been updated, so the cache should not contain an entry for it
1250		try testing.expectEqual(false, try ch.hit());
1251
1252		_ = try ch.addFilePost(temp_file2);
1253
1254		digest3 = ch.final();
1255
1256		try ch.writeManifest();
1257		}
1258
1259		try testing.expect(!mem.eql(u8, &digest1, &digest3));
1260		}
1261
1262		try cwd.deleteTree(temp_manifest_dir);
1263		try cwd.deleteFile(temp_file1);
1264		try cwd.deleteFile(temp_file2);
1265		}

src/Compilation.zig+42-50

...	...	@@ -26,7 +26,7 @@ const wasi_libc = @import("wasi_libc.zig");
26	26	const fatal = @import("main.zig").fatal;
27	27	const clangMain = @import("main.zig").clangMain;
28	28	const Module = @import("Module.zig");
29		const Cache = @import("Cache.zig");
	29	const Cache = std.Build.Cache;
30	30	const translate_c = @import("translate_c.zig");
31	31	const clang = @import("clang.zig");
32	32	const c_codegen = @import("codegen/c.zig");
...	...	@@ -807,44 +807,7 @@ pub const AllErrors = struct {
807	807	}
808	808	};
809	809
810		pub const Directory = struct {
811		/// This field is redundant for operations that can act on the open directory handle
812		/// directly, but it is needed when passing the directory to a child process.
813		/// `null` means cwd.
814		path: ?[]const u8,
815		handle: std.fs.Dir,
816
817		pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 {
818		if (self.path) \|p\| {
819		// TODO clean way to do this with only 1 allocation
820		const part2 = try std.fs.path.join(allocator, paths);
821		defer allocator.free(part2);
822		return std.fs.path.join(allocator, &[_][]const u8{ p, part2 });
823		} else {
824		return std.fs.path.join(allocator, paths);
825		}
826		}
827
828		pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 {
829		if (self.path) \|p\| {
830		// TODO clean way to do this with only 1 allocation
831		const part2 = try std.fs.path.join(allocator, paths);
832		defer allocator.free(part2);
833		return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 });
834		} else {
835		return std.fs.path.joinZ(allocator, paths);
836		}
837		}
838
839		/// Whether or not the handle should be closed, or the path should be freed
840		/// is determined by usage, however this function is provided for convenience
841		/// if it happens to be what the caller needs.
842		pub fn closeAndFree(self: *Directory, gpa: Allocator) void {
843		self.handle.close();
844		if (self.path) \|p\| gpa.free(p);
845		self.* = undefined;
846		}
847		};
	810	pub const Directory = Cache.Directory;
848	811
849	812	pub const EmitLoc = struct {
850	813	/// If this is `null` it means the file will be output to the cache directory.
...	...	@@ -854,6 +817,35 @@ pub const EmitLoc = struct {
854	817	basename: []const u8,
855	818	};
856	819
	820	pub const cache_helpers = struct {
	821	pub fn addEmitLoc(hh: *Cache.HashHelper, emit_loc: EmitLoc) void {
	822	hh.addBytes(emit_loc.basename);
	823	}
	824
	825	pub fn addOptionalEmitLoc(hh: *Cache.HashHelper, optional_emit_loc: ?EmitLoc) void {
	826	hh.add(optional_emit_loc != null);
	827	addEmitLoc(hh, optional_emit_loc orelse return);
	828	}
	829
	830	pub fn hashCSource(self: *Cache.Manifest, c_source: Compilation.CSourceFile) !void {
	831	_ = try self.addFile(c_source.src_path, null);
	832	// Hash the extra flags, with special care to call addFile for file parameters.
	833	// TODO this logic can likely be improved by utilizing clang_options_data.zig.
	834	const file_args = [_][]const u8{"-include"};
	835	var arg_i: usize = 0;
	836	while (arg_i < c_source.extra_flags.len) : (arg_i += 1) {
	837	const arg = c_source.extra_flags[arg_i];
	838	self.hash.addBytes(arg);
	839	for (file_args) \|file_arg\| {
	840	if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) {
	841	arg_i += 1;
	842	_ = try self.addFile(c_source.extra_flags[arg_i], null);
	843	}
	844	}
	845	}
	846	}
	847	};
	848
857	849	pub const ClangPreprocessorMode = enum {
858	850	no,
859	851	/// This means we are doing `zig cc -E -o <path>`.
...	...	@@ -1522,8 +1514,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
1522	1514	cache.hash.add(link_libunwind);
1523	1515	cache.hash.add(options.output_mode);
1524	1516	cache.hash.add(options.machine_code_model);
1525		cache.hash.addOptionalEmitLoc(options.emit_bin);
1526		cache.hash.addOptionalEmitLoc(options.emit_implib);
	1517	cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_bin);
	1518	cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_implib);
1527	1519	cache.hash.addBytes(options.root_name);
1528	1520	if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model);
1529	1521	// TODO audit this and make sure everything is in it
...	...	@@ -2636,11 +2628,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: Compilation, man: Cache.Manifes
2636	2628	man.hash.addListOfBytes(key.src.extra_flags);
2637	2629	}
2638	2630
2639		man.hash.addOptionalEmitLoc(comp.emit_asm);
2640		man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
2641		man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
2642		man.hash.addOptionalEmitLoc(comp.emit_analysis);
2643		man.hash.addOptionalEmitLoc(comp.emit_docs);
	2631	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
	2632	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
	2633	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
	2634	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_analysis);
	2635	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_docs);
2644	2636
2645	2637	man.hash.addListOfBytes(comp.clang_argv);
2646	2638
...	...	@@ -3959,11 +3951,11 @@ fn updateCObject(comp: Compilation, c_object: CObject, c_obj_prog_node: *std.P
3959	3951	defer man.deinit();
3960	3952
3961	3953	man.hash.add(comp.clang_preprocessor_mode);
3962		man.hash.addOptionalEmitLoc(comp.emit_asm);
3963		man.hash.addOptionalEmitLoc(comp.emit_llvm_ir);
3964		man.hash.addOptionalEmitLoc(comp.emit_llvm_bc);
	3954	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm);
	3955	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir);
	3956	cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc);
3965	3957
3966		try man.hashCSource(c_object.src);
	3958	try cache_helpers.hashCSource(&man, c_object.src);
3967	3959
3968	3960	var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
3969	3961	defer arena_allocator.deinit();

src/DepTokenizer.zig deleted-1069

...	...	@@ -1,1069 +0,0 @@
1		const Tokenizer = @This();
2
3		index: usize = 0,
4		bytes: []const u8,
5		state: State = .lhs,
6
7		const std = @import("std");
8		const testing = std.testing;
9		const assert = std.debug.assert;
10
11		pub fn next(self: *Tokenizer) ?Token {
12		var start = self.index;
13		var must_resolve = false;
14		while (self.index < self.bytes.len) {
15		const char = self.bytes[self.index];
16		switch (self.state) {
17		.lhs => switch (char) {
18		'\t', '\n', '\r', ' ' => {
19		// silently ignore whitespace
20		self.index += 1;
21		},
22		else => {
23		start = self.index;
24		self.state = .target;
25		},
26		},
27		.target => switch (char) {
28		'\t', '\n', '\r', ' ' => {
29		return errorIllegalChar(.invalid_target, self.index, char);
30		},
31		'$' => {
32		self.state = .target_dollar_sign;
33		self.index += 1;
34		},
35		'\\' => {
36		self.state = .target_reverse_solidus;
37		self.index += 1;
38		},
39		':' => {
40		self.state = .target_colon;
41		self.index += 1;
42		},
43		else => {
44		self.index += 1;
45		},
46		},
47		.target_reverse_solidus => switch (char) {
48		'\t', '\n', '\r' => {
49		return errorIllegalChar(.bad_target_escape, self.index, char);
50		},
51		' ', '#', '\\' => {
52		must_resolve = true;
53		self.state = .target;
54		self.index += 1;
55		},
56		'$' => {
57		self.state = .target_dollar_sign;
58		self.index += 1;
59		},
60		else => {
61		self.state = .target;
62		self.index += 1;
63		},
64		},
65		.target_dollar_sign => switch (char) {
66		'$' => {
67		must_resolve = true;
68		self.state = .target;
69		self.index += 1;
70		},
71		else => {
72		return errorIllegalChar(.expected_dollar_sign, self.index, char);
73		},
74		},
75		.target_colon => switch (char) {
76		'\n', '\r' => {
77		const bytes = self.bytes[start .. self.index - 1];
78		if (bytes.len != 0) {
79		self.state = .lhs;
80		return finishTarget(must_resolve, bytes);
81		}
82		// silently ignore null target
83		self.state = .lhs;
84		},
85		'/', '\\' => {
86		self.state = .target_colon_reverse_solidus;
87		self.index += 1;
88		},
89		else => {
90		const bytes = self.bytes[start .. self.index - 1];
91		if (bytes.len != 0) {
92		self.state = .rhs;
93		return finishTarget(must_resolve, bytes);
94		}
95		// silently ignore null target
96		self.state = .lhs;
97		},
98		},
99		.target_colon_reverse_solidus => switch (char) {
100		'\n', '\r' => {
101		const bytes = self.bytes[start .. self.index - 2];
102		if (bytes.len != 0) {
103		self.state = .lhs;
104		return finishTarget(must_resolve, bytes);
105		}
106		// silently ignore null target
107		self.state = .lhs;
108		},
109		else => {
110		self.state = .target;
111		},
112		},
113		.rhs => switch (char) {
114		'\t', ' ' => {
115		// silently ignore horizontal whitespace
116		self.index += 1;
117		},
118		'\n', '\r' => {
119		self.state = .lhs;
120		},
121		'\\' => {
122		self.state = .rhs_continuation;
123		self.index += 1;
124		},
125		'"' => {
126		self.state = .prereq_quote;
127		self.index += 1;
128		start = self.index;
129		},
130		else => {
131		start = self.index;
132		self.state = .prereq;
133		},
134		},
135		.rhs_continuation => switch (char) {
136		'\n' => {
137		self.state = .rhs;
138		self.index += 1;
139		},
140		'\r' => {
141		self.state = .rhs_continuation_linefeed;
142		self.index += 1;
143		},
144		else => {
145		return errorIllegalChar(.continuation_eol, self.index, char);
146		},
147		},
148		.rhs_continuation_linefeed => switch (char) {
149		'\n' => {
150		self.state = .rhs;
151		self.index += 1;
152		},
153		else => {
154		return errorIllegalChar(.continuation_eol, self.index, char);
155		},
156		},
157		.prereq_quote => switch (char) {
158		'"' => {
159		self.index += 1;
160		self.state = .rhs;
161		return Token{ .prereq = self.bytes[start .. self.index - 1] };
162		},
163		else => {
164		self.index += 1;
165		},
166		},
167		.prereq => switch (char) {
168		'\t', ' ' => {
169		self.state = .rhs;
170		return Token{ .prereq = self.bytes[start..self.index] };
171		},
172		'\n', '\r' => {
173		self.state = .lhs;
174		return Token{ .prereq = self.bytes[start..self.index] };
175		},
176		'\\' => {
177		self.state = .prereq_continuation;
178		self.index += 1;
179		},
180		else => {
181		self.index += 1;
182		},
183		},
184		.prereq_continuation => switch (char) {
185		'\n' => {
186		self.index += 1;
187		self.state = .rhs;
188		return Token{ .prereq = self.bytes[start .. self.index - 2] };
189		},
190		'\r' => {
191		self.state = .prereq_continuation_linefeed;
192		self.index += 1;
193		},
194		else => {
195		// not continuation
196		self.state = .prereq;
197		self.index += 1;
198		},
199		},
200		.prereq_continuation_linefeed => switch (char) {
201		'\n' => {
202		self.index += 1;
203		self.state = .rhs;
204		return Token{ .prereq = self.bytes[start .. self.index - 1] };
205		},
206		else => {
207		return errorIllegalChar(.continuation_eol, self.index, char);
208		},
209		},
210		}
211		} else {
212		switch (self.state) {
213		.lhs,
214		.rhs,
215		.rhs_continuation,
216		.rhs_continuation_linefeed,
217		=> return null,
218		.target => {
219		return errorPosition(.incomplete_target, start, self.bytes[start..]);
220		},
221		.target_reverse_solidus,
222		.target_dollar_sign,
223		=> {
224		const idx = self.index - 1;
225		return errorIllegalChar(.incomplete_escape, idx, self.bytes[idx]);
226		},
227		.target_colon => {
228		const bytes = self.bytes[start .. self.index - 1];
229		if (bytes.len != 0) {
230		self.index += 1;
231		self.state = .rhs;
232		return finishTarget(must_resolve, bytes);
233		}
234		// silently ignore null target
235		self.state = .lhs;
236		return null;
237		},
238		.target_colon_reverse_solidus => {
239		const bytes = self.bytes[start .. self.index - 2];
240		if (bytes.len != 0) {
241		self.index += 1;
242		self.state = .rhs;
243		return finishTarget(must_resolve, bytes);
244		}
245		// silently ignore null target
246		self.state = .lhs;
247		return null;
248		},
249		.prereq_quote => {
250		return errorPosition(.incomplete_quoted_prerequisite, start, self.bytes[start..]);
251		},
252		.prereq => {
253		self.state = .lhs;
254		return Token{ .prereq = self.bytes[start..] };
255		},
256		.prereq_continuation => {
257		self.state = .lhs;
258		return Token{ .prereq = self.bytes[start .. self.index - 1] };
259		},
260		.prereq_continuation_linefeed => {
261		self.state = .lhs;
262		return Token{ .prereq = self.bytes[start .. self.index - 2] };
263		},
264		}
265		}
266		unreachable;
267		}
268
269		fn errorPosition(comptime id: std.meta.Tag(Token), index: usize, bytes: []const u8) Token {
270		return @unionInit(Token, @tagName(id), .{ .index = index, .bytes = bytes });
271		}
272
273		fn errorIllegalChar(comptime id: std.meta.Tag(Token), index: usize, char: u8) Token {
274		return @unionInit(Token, @tagName(id), .{ .index = index, .char = char });
275		}
276
277		fn finishTarget(must_resolve: bool, bytes: []const u8) Token {
278		return if (must_resolve) .{ .target_must_resolve = bytes } else .{ .target = bytes };
279		}
280
281		const State = enum {
282		lhs,
283		target,
284		target_reverse_solidus,
285		target_dollar_sign,
286		target_colon,
287		target_colon_reverse_solidus,
288		rhs,
289		rhs_continuation,
290		rhs_continuation_linefeed,
291		prereq_quote,
292		prereq,
293		prereq_continuation,
294		prereq_continuation_linefeed,
295		};
296
297		pub const Token = union(enum) {
298		target: []const u8,
299		target_must_resolve: []const u8,
300		prereq: []const u8,
301
302		incomplete_quoted_prerequisite: IndexAndBytes,
303		incomplete_target: IndexAndBytes,
304
305		invalid_target: IndexAndChar,
306		bad_target_escape: IndexAndChar,
307		expected_dollar_sign: IndexAndChar,
308		continuation_eol: IndexAndChar,
309		incomplete_escape: IndexAndChar,
310
311		pub const IndexAndChar = struct {
312		index: usize,
313		char: u8,
314		};
315
316		pub const IndexAndBytes = struct {
317		index: usize,
318		bytes: []const u8,
319		};
320
321		/// Resolve escapes in target. Only valid with .target_must_resolve.
322		pub fn resolve(self: Token, writer: anytype) @TypeOf(writer).Error!void {
323		const bytes = self.target_must_resolve; // resolve called on incorrect token
324
325		var state: enum { start, escape, dollar } = .start;
326		for (bytes) \|c\| {
327		switch (state) {
328		.start => {
329		switch (c) {
330		'\\' => state = .escape,
331		'$' => state = .dollar,
332		else => try writer.writeByte(c),
333		}
334		},
335		.escape => {
336		switch (c) {
337		' ', '#', '\\' => {},
338		'$' => {
339		try writer.writeByte('\\');
340		state = .dollar;
341		continue;
342		},
343		else => try writer.writeByte('\\'),
344		}
345		try writer.writeByte(c);
346		state = .start;
347		},
348		.dollar => {
349		try writer.writeByte('$');
350		switch (c) {
351		'$' => {},
352		else => try writer.writeByte(c),
353		}
354		state = .start;
355		},
356		}
357		}
358		}
359
360		pub fn printError(self: Token, writer: anytype) @TypeOf(writer).Error!void {
361		switch (self) {
362		.target, .target_must_resolve, .prereq => unreachable, // not an error
363		.incomplete_quoted_prerequisite,
364		.incomplete_target,
365		=> \|index_and_bytes\| {
366		try writer.print("{s} '", .{self.errStr()});
367		if (self == .incomplete_target) {
368		const tmp = Token{ .target_must_resolve = index_and_bytes.bytes };
369		try tmp.resolve(writer);
370		} else {
371		try printCharValues(writer, index_and_bytes.bytes);
372		}
373		try writer.print("' at position {d}", .{index_and_bytes.index});
374		},
375		.invalid_target,
376		.bad_target_escape,
377		.expected_dollar_sign,
378		.continuation_eol,
379		.incomplete_escape,
380		=> \|index_and_char\| {
381		try writer.writeAll("illegal char ");
382		try printUnderstandableChar(writer, index_and_char.char);
383		try writer.print(" at position {d}: {s}", .{ index_and_char.index, self.errStr() });
384		},
385		}
386		}
387
388		fn errStr(self: Token) []const u8 {
389		return switch (self) {
390		.target, .target_must_resolve, .prereq => unreachable, // not an error
391		.incomplete_quoted_prerequisite => "incomplete quoted prerequisite",
392		.incomplete_target => "incomplete target",
393		.invalid_target => "invalid target",
394		.bad_target_escape => "bad target escape",
395		.expected_dollar_sign => "expecting '$'",
396		.continuation_eol => "continuation expecting end-of-line",
397		.incomplete_escape => "incomplete escape",
398		};
399		}
400		};
401
402		test "empty file" {
403		try depTokenizer("", "");
404		}
405
406		test "empty whitespace" {
407		try depTokenizer("\n", "");
408		try depTokenizer("\r", "");
409		try depTokenizer("\r\n", "");
410		try depTokenizer(" ", "");
411		}
412
413		test "empty colon" {
414		try depTokenizer(":", "");
415		try depTokenizer("\n:", "");
416		try depTokenizer("\r:", "");
417		try depTokenizer("\r\n:", "");
418		try depTokenizer(" :", "");
419		}
420
421		test "empty target" {
422		try depTokenizer("foo.o:", "target = {foo.o}");
423		try depTokenizer(
424		\\foo.o:
425		\\bar.o:
426		\\abcd.o:
427		,
428		\\target = {foo.o}
429		\\target = {bar.o}
430		\\target = {abcd.o}
431		);
432		}
433
434		test "whitespace empty target" {
435		try depTokenizer("\nfoo.o:", "target = {foo.o}");
436		try depTokenizer("\rfoo.o:", "target = {foo.o}");
437		try depTokenizer("\r\nfoo.o:", "target = {foo.o}");
438		try depTokenizer(" foo.o:", "target = {foo.o}");
439		}
440
441		test "escape empty target" {
442		try depTokenizer("\\ foo.o:", "target = { foo.o}");
443		try depTokenizer("\\#foo.o:", "target = {#foo.o}");
444		try depTokenizer("\\\\foo.o:", "target = {\\foo.o}");
445		try depTokenizer("$$foo.o:", "target = {$foo.o}");
446		}
447
448		test "empty target linefeeds" {
449		try depTokenizer("\n", "");
450		try depTokenizer("\r\n", "");
451
452		const expect = "target = {foo.o}";
453		try depTokenizer(
454		\\foo.o:
455		, expect);
456		try depTokenizer(
457		\\foo.o:
458		\\
459		, expect);
460		try depTokenizer(
461		\\foo.o:
462		, expect);
463		try depTokenizer(
464		\\foo.o:
465		\\
466		, expect);
467		}
468
469		test "empty target linefeeds + continuations" {
470		const expect = "target = {foo.o}";
471		try depTokenizer(
472		\\foo.o:\
473		, expect);
474		try depTokenizer(
475		\\foo.o:\
476		\\
477		, expect);
478		try depTokenizer(
479		\\foo.o:\
480		, expect);
481		try depTokenizer(
482		\\foo.o:\
483		\\
484		, expect);
485		}
486
487		test "empty target linefeeds + hspace + continuations" {
488		const expect = "target = {foo.o}";
489		try depTokenizer(
490		\\foo.o: \
491		, expect);
492		try depTokenizer(
493		\\foo.o: \
494		\\
495		, expect);
496		try depTokenizer(
497		\\foo.o: \
498		, expect);
499		try depTokenizer(
500		\\foo.o: \
501		\\
502		, expect);
503		}
504
505		test "prereq" {
506		const expect =
507		\\target = {foo.o}
508		\\prereq = {foo.c}
509		;
510		try depTokenizer("foo.o: foo.c", expect);
511		try depTokenizer(
512		\\foo.o: \
513		\\foo.c
514		, expect);
515		try depTokenizer(
516		\\foo.o: \
517		\\ foo.c
518		, expect);
519		try depTokenizer(
520		\\foo.o: \
521		\\ foo.c
522		, expect);
523		}
524
525		test "prereq continuation" {
526		const expect =
527		\\target = {foo.o}
528		\\prereq = {foo.h}
529		\\prereq = {bar.h}
530		;
531		try depTokenizer(
532		\\foo.o: foo.h\
533		\\bar.h
534		, expect);
535		try depTokenizer(
536		\\foo.o: foo.h\
537		\\bar.h
538		, expect);
539		}
540
541		test "multiple prereqs" {
542		const expect =
543		\\target = {foo.o}
544		\\prereq = {foo.c}
545		\\prereq = {foo.h}
546		\\prereq = {bar.h}
547		;
548		try depTokenizer("foo.o: foo.c foo.h bar.h", expect);
549		try depTokenizer(
550		\\foo.o: \
551		\\foo.c foo.h bar.h
552		, expect);
553		try depTokenizer(
554		\\foo.o: foo.c foo.h bar.h\
555		, expect);
556		try depTokenizer(
557		\\foo.o: foo.c foo.h bar.h\
558		\\
559		, expect);
560		try depTokenizer(
561		\\foo.o: \
562		\\foo.c \
563		\\ foo.h\
564		\\bar.h
565		\\
566		, expect);
567		try depTokenizer(
568		\\foo.o: \
569		\\foo.c \
570		\\ foo.h\
571		\\bar.h\
572		\\
573		, expect);
574		try depTokenizer(
575		\\foo.o: \
576		\\foo.c \
577		\\ foo.h\
578		\\bar.h\
579		, expect);
580		}
581
582		test "multiple targets and prereqs" {
583		try depTokenizer(
584		\\foo.o: foo.c
585		\\bar.o: bar.c a.h b.h c.h
586		\\abc.o: abc.c \
587		\\ one.h two.h \
588		\\ three.h four.h
589		,
590		\\target = {foo.o}
591		\\prereq = {foo.c}
592		\\target = {bar.o}
593		\\prereq = {bar.c}
594		\\prereq = {a.h}
595		\\prereq = {b.h}
596		\\prereq = {c.h}
597		\\target = {abc.o}
598		\\prereq = {abc.c}
599		\\prereq = {one.h}
600		\\prereq = {two.h}
601		\\prereq = {three.h}
602		\\prereq = {four.h}
603		);
604		try depTokenizer(
605		\\ascii.o: ascii.c
606		\\base64.o: base64.c stdio.h
607		\\elf.o: elf.c a.h b.h c.h
608		\\macho.o: \
609		\\ macho.c\
610		\\ a.h b.h c.h
611		,
612		\\target = {ascii.o}
613		\\prereq = {ascii.c}
614		\\target = {base64.o}
615		\\prereq = {base64.c}
616		\\prereq = {stdio.h}
617		\\target = {elf.o}
618		\\prereq = {elf.c}
619		\\prereq = {a.h}
620		\\prereq = {b.h}
621		\\prereq = {c.h}
622		\\target = {macho.o}
623		\\prereq = {macho.c}
624		\\prereq = {a.h}
625		\\prereq = {b.h}
626		\\prereq = {c.h}
627		);
628		try depTokenizer(
629		\\a$$scii.o: ascii.c
630		\\\\base64.o: "\base64.c" "s t#dio.h"
631		\\e\\lf.o: "e\lf.c" "a.h$$" "$$b.h c.h$$"
632		\\macho.o: \
633		\\ "macho!.c" \
634		\\ a.h b.h c.h
635		,
636		\\target = {a$scii.o}
637		\\prereq = {ascii.c}
638		\\target = {\base64.o}
639		\\prereq = {\base64.c}
640		\\prereq = {s t#dio.h}
641		\\target = {e\lf.o}
642		\\prereq = {e\lf.c}
643		\\prereq = {a.h$$}
644		\\prereq = {$$b.h c.h$$}
645		\\target = {macho.o}
646		\\prereq = {macho!.c}
647		\\prereq = {a.h}
648		\\prereq = {b.h}
649		\\prereq = {c.h}
650		);
651		}
652
653		test "windows quoted prereqs" {
654		try depTokenizer(
655		\\c:\foo.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo.c"
656		\\c:\foo2.o: "C:\Program Files (x86)\Microsoft Visual Studio\foo2.c" \
657		\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo1.h" \
658		\\ "C:\Program Files (x86)\Microsoft Visual Studio\foo2.h"
659		,
660		\\target = {c:\foo.o}
661		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo.c}
662		\\target = {c:\foo2.o}
663		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.c}
664		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo1.h}
665		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\foo2.h}
666		);
667		}
668
669		test "windows mixed prereqs" {
670		try depTokenizer(
671		\\cimport.o: \
672		\\ C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h \
673		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h" \
674		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h" \
675		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h" \
676		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h" \
677		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h" \
678		\\ C:\msys64\opt\zig\lib\zig\include\vadefs.h \
679		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h" \
680		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h" \
681		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h" \
682		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h" \
683		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h" \
684		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h" \
685		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h" \
686		\\ "C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h" \
687		\\ "C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h"
688		,
689		\\target = {cimport.o}
690		\\prereq = {C:\msys64\home\anon\project\zig\master\zig-cache\o\qhvhbUo7GU5iKyQ5mpA8TcQpncCYaQu0wwvr3ybiSTj_Dtqi1Nmcb70kfODJ2Qlg\cimport.h}
691		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\stdio.h}
692		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt.h}
693		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime.h}
694		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\sal.h}
695		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\concurrencysal.h}
696		\\prereq = {C:\msys64\opt\zig\lib\zig\include\vadefs.h}
697		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vadefs.h}
698		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstdio.h}
699		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_stdio_config.h}
700		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\string.h}
701		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memory.h}
702		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_memcpy_s.h}
703		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\errno.h}
704		\\prereq = {C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Tools\MSVC\14.21.27702\lib\x64\\..\..\include\vcruntime_string.h}
705		\\prereq = {C:\Program Files (x86)\Windows Kits\10\\Include\10.0.17763.0\ucrt\corecrt_wstring.h}
706		);
707		}
708
709		test "windows funky targets" {
710		try depTokenizer(
711		\\C:\Users\anon\foo.o:
712		\\C:\Users\anon\foo\ .o:
713		\\C:\Users\anon\foo\#.o:
714		\\C:\Users\anon\foo$$.o:
715		\\C:\Users\anon\\\ foo.o:
716		\\C:\Users\anon\\#foo.o:
717		\\C:\Users\anon\$$foo.o:
718		\\C:\Users\anon\\\ \ \ \ \ foo.o:
719		,
720		\\target = {C:\Users\anon\foo.o}
721		\\target = {C:\Users\anon\foo .o}
722		\\target = {C:\Users\anon\foo#.o}
723		\\target = {C:\Users\anon\foo$.o}
724		\\target = {C:\Users\anon\ foo.o}
725		\\target = {C:\Users\anon\#foo.o}
726		\\target = {C:\Users\anon\$foo.o}
727		\\target = {C:\Users\anon\ foo.o}
728		);
729		}
730
731		test "windows drive and forward slashes" {
732		try depTokenizer(
733		\\C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj: \
734		\\ C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c
735		,
736		\\target = {C:/msys64/what/zig-cache\tmp\48ac4d78dd531abd-cxa_thread_atexit.obj}
737		\\prereq = {C:/msys64/opt/zig3/lib/zig/libc/mingw/crt/cxa_thread_atexit.c}
738		);
739		}
740
741		test "error incomplete escape - reverse_solidus" {
742		try depTokenizer("\\",
743		\\ERROR: illegal char '\' at position 0: incomplete escape
744		);
745		try depTokenizer("\t\\",
746		\\ERROR: illegal char '\' at position 1: incomplete escape
747		);
748		try depTokenizer("\n\\",
749		\\ERROR: illegal char '\' at position 1: incomplete escape
750		);
751		try depTokenizer("\r\\",
752		\\ERROR: illegal char '\' at position 1: incomplete escape
753		);
754		try depTokenizer("\r\n\\",
755		\\ERROR: illegal char '\' at position 2: incomplete escape
756		);
757		try depTokenizer(" \\",
758		\\ERROR: illegal char '\' at position 1: incomplete escape
759		);
760		}
761
762		test "error incomplete escape - dollar_sign" {
763		try depTokenizer("$",
764		\\ERROR: illegal char '$' at position 0: incomplete escape
765		);
766		try depTokenizer("\t$",
767		\\ERROR: illegal char '$' at position 1: incomplete escape
768		);
769		try depTokenizer("\n$",
770		\\ERROR: illegal char '$' at position 1: incomplete escape
771		);
772		try depTokenizer("\r$",
773		\\ERROR: illegal char '$' at position 1: incomplete escape
774		);
775		try depTokenizer("\r\n$",
776		\\ERROR: illegal char '$' at position 2: incomplete escape
777		);
778		try depTokenizer(" $",
779		\\ERROR: illegal char '$' at position 1: incomplete escape
780		);
781		}
782
783		test "error incomplete target" {
784		try depTokenizer("foo.o",
785		\\ERROR: incomplete target 'foo.o' at position 0
786		);
787		try depTokenizer("\tfoo.o",
788		\\ERROR: incomplete target 'foo.o' at position 1
789		);
790		try depTokenizer("\nfoo.o",
791		\\ERROR: incomplete target 'foo.o' at position 1
792		);
793		try depTokenizer("\rfoo.o",
794		\\ERROR: incomplete target 'foo.o' at position 1
795		);
796		try depTokenizer("\r\nfoo.o",
797		\\ERROR: incomplete target 'foo.o' at position 2
798		);
799		try depTokenizer(" foo.o",
800		\\ERROR: incomplete target 'foo.o' at position 1
801		);
802
803		try depTokenizer("\\ foo.o",
804		\\ERROR: incomplete target ' foo.o' at position 0
805		);
806		try depTokenizer("\\#foo.o",
807		\\ERROR: incomplete target '#foo.o' at position 0
808		);
809		try depTokenizer("\\\\foo.o",
810		\\ERROR: incomplete target '\foo.o' at position 0
811		);
812		try depTokenizer("$$foo.o",
813		\\ERROR: incomplete target '$foo.o' at position 0
814		);
815		}
816
817		test "error illegal char at position - bad target escape" {
818		try depTokenizer("\\\t",
819		\\ERROR: illegal char \x09 at position 1: bad target escape
820		);
821		try depTokenizer("\\\n",
822		\\ERROR: illegal char \x0A at position 1: bad target escape
823		);
824		try depTokenizer("\\\r",
825		\\ERROR: illegal char \x0D at position 1: bad target escape
826		);
827		try depTokenizer("\\\r\n",
828		\\ERROR: illegal char \x0D at position 1: bad target escape
829		);
830		}
831
832		test "error illegal char at position - execting dollar_sign" {
833		try depTokenizer("$\t",
834		\\ERROR: illegal char \x09 at position 1: expecting '$'
835		);
836		try depTokenizer("$\n",
837		\\ERROR: illegal char \x0A at position 1: expecting '$'
838		);
839		try depTokenizer("$\r",
840		\\ERROR: illegal char \x0D at position 1: expecting '$'
841		);
842		try depTokenizer("$\r\n",
843		\\ERROR: illegal char \x0D at position 1: expecting '$'
844		);
845		}
846
847		test "error illegal char at position - invalid target" {
848		try depTokenizer("foo\t.o",
849		\\ERROR: illegal char \x09 at position 3: invalid target
850		);
851		try depTokenizer("foo\n.o",
852		\\ERROR: illegal char \x0A at position 3: invalid target
853		);
854		try depTokenizer("foo\r.o",
855		\\ERROR: illegal char \x0D at position 3: invalid target
856		);
857		try depTokenizer("foo\r\n.o",
858		\\ERROR: illegal char \x0D at position 3: invalid target
859		);
860		}
861
862		test "error target - continuation expecting end-of-line" {
863		try depTokenizer("foo.o: \\\t",
864		\\target = {foo.o}
865		\\ERROR: illegal char \x09 at position 8: continuation expecting end-of-line
866		);
867		try depTokenizer("foo.o: \\ ",
868		\\target = {foo.o}
869		\\ERROR: illegal char ' ' at position 8: continuation expecting end-of-line
870		);
871		try depTokenizer("foo.o: \\x",
872		\\target = {foo.o}
873		\\ERROR: illegal char 'x' at position 8: continuation expecting end-of-line
874		);
875		try depTokenizer("foo.o: \\\x0dx",
876		\\target = {foo.o}
877		\\ERROR: illegal char 'x' at position 9: continuation expecting end-of-line
878		);
879		}
880
881		test "error prereq - continuation expecting end-of-line" {
882		try depTokenizer("foo.o: foo.h\\\x0dx",
883		\\target = {foo.o}
884		\\ERROR: illegal char 'x' at position 14: continuation expecting end-of-line
885		);
886		}
887
888		// - tokenize input, emit textual representation, and compare to expect
889		fn depTokenizer(input: []const u8, expect: []const u8) !void {
890		var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator);
891		const arena = arena_allocator.allocator();
892		defer arena_allocator.deinit();
893
894		var it: Tokenizer = .{ .bytes = input };
895		var buffer = std.ArrayList(u8).init(arena);
896		var resolve_buf = std.ArrayList(u8).init(arena);
897		var i: usize = 0;
898		while (it.next()) \|token\| {
899		if (i != 0) try buffer.appendSlice("\n");
900		switch (token) {
901		.target, .prereq => \|bytes\| {
902		try buffer.appendSlice(@tagName(token));
903		try buffer.appendSlice(" = {");
904		for (bytes) \|b\| {
905		try buffer.append(printable_char_tab[b]);
906		}
907		try buffer.appendSlice("}");
908		},
909		.target_must_resolve => {
910		try buffer.appendSlice("target = {");
911		try token.resolve(resolve_buf.writer());
912		for (resolve_buf.items) \|b\| {
913		try buffer.append(printable_char_tab[b]);
914		}
915		resolve_buf.items.len = 0;
916		try buffer.appendSlice("}");
917		},
918		else => {
919		try buffer.appendSlice("ERROR: ");
920		try token.printError(buffer.writer());
921		break;
922		},
923		}
924		i += 1;
925		}
926
927		if (std.mem.eql(u8, expect, buffer.items)) {
928		try testing.expect(true);
929		return;
930		}
931
932		const out = std.io.getStdErr().writer();
933
934		try out.writeAll("\n");
935		try printSection(out, "<<<< input", input);
936		try printSection(out, "==== expect", expect);
937		try printSection(out, ">>>> got", buffer.items);
938		try printRuler(out);
939
940		try testing.expect(false);
941		}
942
943		fn printSection(out: anytype, label: []const u8, bytes: []const u8) !void {
944		try printLabel(out, label, bytes);
945		try hexDump(out, bytes);
946		try printRuler(out);
947		try out.writeAll(bytes);
948		try out.writeAll("\n");
949		}
950
951		fn printLabel(out: anytype, label: []const u8, bytes: []const u8) !void {
952		var buf: [80]u8 = undefined;
953		var text = try std.fmt.bufPrint(buf[0..], "{s} {d} bytes ", .{ label, bytes.len });
954		try out.writeAll(text);
955		var i: usize = text.len;
956		const end = 79;
957		while (i < end) : (i += 1) {
958		try out.writeAll(&[_]u8{label[0]});
959		}
960		try out.writeAll("\n");
961		}
962
963		fn printRuler(out: anytype) !void {
964		var i: usize = 0;
965		const end = 79;
966		while (i < end) : (i += 1) {
967		try out.writeAll("-");
968		}
969		try out.writeAll("\n");
970		}
971
972		fn hexDump(out: anytype, bytes: []const u8) !void {
973		const n16 = bytes.len >> 4;
974		var line: usize = 0;
975		var offset: usize = 0;
976		while (line < n16) : (line += 1) {
977		try hexDump16(out, offset, bytes[offset .. offset + 16]);
978		offset += 16;
979		}
980
981		const n = bytes.len & 0x0f;
982		if (n > 0) {
983		try printDecValue(out, offset, 8);
984		try out.writeAll(":");
985		try out.writeAll(" ");
986		var end1 = std.math.min(offset + n, offset + 8);
987		for (bytes[offset..end1]) \|b\| {
988		try out.writeAll(" ");
989		try printHexValue(out, b, 2);
990		}
991		var end2 = offset + n;
992		if (end2 > end1) {
993		try out.writeAll(" ");
994		for (bytes[end1..end2]) \|b\| {
995		try out.writeAll(" ");
996		try printHexValue(out, b, 2);
997		}
998		}
999		const short = 16 - n;
1000		var i: usize = 0;
1001		while (i < short) : (i += 1) {
1002		try out.writeAll(" ");
1003		}
1004		if (end2 > end1) {
1005		try out.writeAll(" \|");
1006		} else {
1007		try out.writeAll(" \|");
1008		}
1009		try printCharValues(out, bytes[offset..end2]);
1010		try out.writeAll("\|\n");
1011		offset += n;
1012		}
1013
1014		try printDecValue(out, offset, 8);
1015		try out.writeAll(":");
1016		try out.writeAll("\n");
1017		}
1018
1019		fn hexDump16(out: anytype, offset: usize, bytes: []const u8) !void {
1020		try printDecValue(out, offset, 8);
1021		try out.writeAll(":");
1022		try out.writeAll(" ");
1023		for (bytes[0..8]) \|b\| {
1024		try out.writeAll(" ");
1025		try printHexValue(out, b, 2);
1026		}
1027		try out.writeAll(" ");
1028		for (bytes[8..16]) \|b\| {
1029		try out.writeAll(" ");
1030		try printHexValue(out, b, 2);
1031		}
1032		try out.writeAll(" \|");
1033		try printCharValues(out, bytes);
1034		try out.writeAll("\|\n");
1035		}
1036
1037		fn printDecValue(out: anytype, value: u64, width: u8) !void {
1038		var buffer: [20]u8 = undefined;
1039		const len = std.fmt.formatIntBuf(buffer[0..], value, 10, .lower, .{ .width = width, .fill = '0' });
1040		try out.writeAll(buffer[0..len]);
1041		}
1042
1043		fn printHexValue(out: anytype, value: u64, width: u8) !void {
1044		var buffer: [16]u8 = undefined;
1045		const len = std.fmt.formatIntBuf(buffer[0..], value, 16, .lower, .{ .width = width, .fill = '0' });
1046		try out.writeAll(buffer[0..len]);
1047		}
1048
1049		fn printCharValues(out: anytype, bytes: []const u8) !void {
1050		for (bytes) \|b\| {
1051		try out.writeAll(&[_]u8{printable_char_tab[b]});
1052		}
1053		}
1054
1055		fn printUnderstandableChar(out: anytype, char: u8) !void {
1056		if (std.ascii.isPrint(char)) {
1057		try out.print("'{c}'", .{char});
1058		} else {
1059		try out.print("\\x{X:0>2}", .{char});
1060		}
1061		}
1062
1063		// zig fmt: off
1064		const printable_char_tab: [256]u8 = (
1065		"................................ !\"#$%&'()*+,-./0123456789:;<=>?" ++
1066		"@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{\|}~." ++
1067		"................................................................" ++
1068		"................................................................"
1069		).*;

src/Module.zig+1-1

...	...	@@ -16,7 +16,7 @@ const Ast = std.zig.Ast;
16	16
17	17	const Module = @This();
18	18	const Compilation = @import("Compilation.zig");
19		const Cache = @import("Cache.zig");
	19	const Cache = std.Build.Cache;
20	20	const Value = @import("value.zig").Value;
21	21	const Type = @import("type.zig").Type;
22	22	const TypedValue = @import("TypedValue.zig");

src/Package.zig+1-1

...	...	@@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig");
13	13	const Module = @import("Module.zig");
14	14	const ThreadPool = @import("ThreadPool.zig");
15	15	const WaitGroup = @import("WaitGroup.zig");
16		const Cache = @import("Cache.zig");
	16	const Cache = std.Build.Cache;
17	17	const build_options = @import("build_options");
18	18	const Manifest = @import("Manifest.zig");
19	19

src/glibc.zig+1-1

...	...	@@ -11,7 +11,7 @@ const target_util = @import("target.zig");
11	11	const Compilation = @import("Compilation.zig");
12	12	const build_options = @import("build_options");
13	13	const trace = @import("tracy.zig").trace;
14		const Cache = @import("Cache.zig");
	14	const Cache = std.Build.Cache;
15	15	const Package = @import("Package.zig");
16	16
17	17	pub const Lib = struct {

src/link.zig+1-1

...	...	@@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig");
10	10
11	11	const Air = @import("Air.zig");
12	12	const Allocator = std.mem.Allocator;
13		const Cache = @import("Cache.zig");
	13	const Cache = std.Build.Cache;
14	14	const Compilation = @import("Compilation.zig");
15	15	const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
16	16	const Liveness = @import("Liveness.zig");

src/link/Coff/lld.zig+1-1

...	...	@@ -5,6 +5,7 @@ const assert = std.debug.assert;
5	5	const fs = std.fs;
6	6	const log = std.log.scoped(.link);
7	7	const mem = std.mem;
	8	const Cache = std.Build.Cache;
8	9
9	10	const mingw = @import("../../mingw.zig");
10	11	const link = @import("../../link.zig");
...	...	@@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace;
13	14
14	15	const Allocator = mem.Allocator;
15	16
16		const Cache = @import("../../Cache.zig");
17	17	const Coff = @import("../Coff.zig");
18	18	const Compilation = @import("../../Compilation.zig");
19	19

src/link/Elf.zig+1-1

...	...	@@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace;
21	21	const Air = @import("../Air.zig");
22	22	const Allocator = std.mem.Allocator;
23	23	pub const Atom = @import("Elf/Atom.zig");
24		const Cache = @import("../Cache.zig");
	24	const Cache = std.Build.Cache;
25	25	const Compilation = @import("../Compilation.zig");
26	26	const Dwarf = @import("Dwarf.zig");
27	27	const File = link.File;

src/link/MachO.zig+1-1

...	...	@@ -28,7 +28,7 @@ const Air = @import("../Air.zig");
28	28	const Allocator = mem.Allocator;
29	29	const Archive = @import("MachO/Archive.zig");
30	30	pub const Atom = @import("MachO/Atom.zig");
31		const Cache = @import("../Cache.zig");
	31	const Cache = std.Build.Cache;
32	32	const CodeSignature = @import("MachO/CodeSignature.zig");
33	33	const Compilation = @import("../Compilation.zig");
34	34	const Dwarf = File.Dwarf;

src/link/MachO/zld.zig+1-1

...	...	@@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace;
20	20	const Allocator = mem.Allocator;
21	21	const Archive = @import("Archive.zig");
22	22	const Atom = @import("ZldAtom.zig");
23		const Cache = @import("../../Cache.zig");
	23	const Cache = std.Build.Cache;
24	24	const CodeSignature = @import("CodeSignature.zig");
25	25	const Compilation = @import("../../Compilation.zig");
26	26	const DwarfInfo = @import("DwarfInfo.zig");

src/link/Wasm.zig+1-1

...	...	@@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain;
20	20	const trace = @import("../tracy.zig").trace;
21	21	const build_options = @import("build_options");
22	22	const wasi_libc = @import("../wasi_libc.zig");
23		const Cache = @import("../Cache.zig");
	23	const Cache = std.Build.Cache;
24	24	const Type = @import("../type.zig").Type;
25	25	const TypedValue = @import("../TypedValue.zig");
26	26	const LlvmObject = @import("../codegen/llvm.zig").Object;

src/main.zig+2-2

...	...	@@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation;
20	20	const wasi_libc = @import("wasi_libc.zig");
21	21	const translate_c = @import("translate_c.zig");
22	22	const clang = @import("clang.zig");
23		const Cache = @import("Cache.zig");
	23	const Cache = std.Build.Cache;
24	24	const target_util = @import("target.zig");
25	25	const ThreadPool = @import("ThreadPool.zig");
26	26	const crash_report = @import("crash_report.zig");
...	...	@@ -3607,7 +3607,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void
3607	3607	defer if (enable_cache) man.deinit();
3608	3608
3609	3609	man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects
3610		man.hashCSource(c_source_file) catch \|err\| {
	3610	Compilation.cache_helpers.hashCSource(&man, c_source_file) catch \|err\| {
3611	3611	fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) });
3612	3612	};
3613	3613

src/mingw.zig+1-1

...	...	@@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw);
8	8	const builtin = @import("builtin");
9	9	const Compilation = @import("Compilation.zig");
10	10	const build_options = @import("build_options");
11		const Cache = @import("Cache.zig");
	11	const Cache = std.Build.Cache;
12	12
13	13	pub const CRTFile = enum {
14	14	crt2_o,