Commit cda102be020ef9c5c1425553ff611720f496f17e | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-05-25 19:29:03-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-05-25 19:29:03-04:00
log	cda102be020ef9c5c1425553ff611720f496f17e
tree	e43393b8686f5eb25fce441a4c41b4cb1190a0f7
parent	6d5ec184ab1a1b8c155714801f7d6cb7ea6f5b8f

improvements to self-hosted cache hash system

 * change miscellaneous things to more idiomatic zig style
 * change the digest length to 24 bytes instead of 48. This is
   still 70  more bits than UUIDs. For an analysis of probability of
   collisions, see:
   https://en.wikipedia.org/wiki/Universally_unique_identifier#Collisions
 * fix the API having the possibility of mismatched allocators
 * fix some error paths to behave properly
 * modify the guarantees about when file contents are loaded for input files
 * pwrite instead of seek + write
 * implement isProblematicTimestamp
 * fix tests with regards to a working isProblematicTimestamp function.
   this requires sleeping until the current timestamp becomes
   unproblematic.
 * introduce std.fs.File.INode, a cross platform type abstraction
   so that cache hash implementation does not need to reach into std.os.

2 files changed, 238 insertions(+), 162 deletions(-)

lib
- std
  - cache_hash.zig
  - fs
    file.zig

lib/std/cache_hash.zig+235-159

...	...	@@ -1,18 +1,19 @@
1		const Blake3 = @import("crypto.zig").Blake3;
2		const fs = @import("fs.zig");
3		const base64 = @import("base64.zig");
4		const ArrayList = @import("array_list.zig").ArrayList;
5		const debug = @import("debug.zig");
6		const testing = @import("testing.zig");
7		const mem = @import("mem.zig");
8		const fmt = @import("fmt.zig");
9		const Allocator = mem.Allocator;
10		const os = @import("os.zig");
11		const time = @import("time.zig");
	1	const std = @import("std.zig");
	2	const Blake3 = std.crypto.Blake3;
	3	const fs = std.fs;
	4	const base64 = std.base64;
	5	const ArrayList = std.ArrayList;
	6	const assert = std.debug.assert;
	7	const testing = std.testing;
	8	const mem = std.mem;
	9	const fmt = std.fmt;
	10	const Allocator = std.mem.Allocator;
12	11
13	12	const base64_encoder = fs.base64_encoder;
14	13	const base64_decoder = fs.base64_decoder;
15		const BIN_DIGEST_LEN = 48;
	14	/// This is 70 more bits than UUIDs. For an analysis of probability of collisions, see:
	15	/// https://en.wikipedia.org/wiki/Universally_unique_identifier#Collisions
	16	const BIN_DIGEST_LEN = 24;
16	17	const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
17	18
18	19	const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
...	...	@@ -22,22 +23,23 @@ pub const File = struct {
22	23	max_file_size: ?usize,
23	24	stat: fs.File.Stat,
24	25	bin_digest: [BIN_DIGEST_LEN]u8,
25		contents: ?[]const u8 = null,
	26	contents: ?[]const u8,
26	27
27		pub fn deinit(self: @This(), alloc: Allocator) void {
	28	pub fn deinit(self: File, allocator: Allocator) void {
28	29	if (self.path) \|owned_slice\| {
29		alloc.free(owned_slice);
	30	allocator.free(owned_slice);
30	31	self.path = null;
31	32	}
32	33	if (self.contents) \|contents\| {
33		alloc.free(contents);
	34	allocator.free(contents);
34	35	self.contents = null;
35	36	}
	37	self.* = undefined;
36	38	}
37	39	};
38	40
39	41	pub const CacheHash = struct {
40		alloc: *Allocator,
	42	allocator: *Allocator,
41	43	blake3: Blake3,
42	44	manifest_dir: fs.Dir,
43	45	manifest_file: ?fs.File,
...	...	@@ -45,24 +47,22 @@ pub const CacheHash = struct {
45	47	files: ArrayList(File),
46	48	b64_digest: [BASE64_DIGEST_LEN]u8,
47	49
48		pub fn init(alloc: *Allocator, manifest_dir_path: []const u8) !@This() {
49		try fs.cwd().makePath(manifest_dir_path);
50		const manifest_dir = try fs.cwd().openDir(manifest_dir_path, .{});
51
	50	/// Be sure to call release after successful initialization.
	51	pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
52	52	return CacheHash{
53		.alloc = alloc,
	53	.allocator = allocator,
54	54	.blake3 = Blake3.init(),
55		.manifest_dir = manifest_dir,
	55	.manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
56	56	.manifest_file = null,
57	57	.manifest_dirty = false,
58		.files = ArrayList(File).init(alloc),
	58	.files = ArrayList(File).init(allocator),
59	59	.b64_digest = undefined,
60	60	};
61	61	}
62	62
63	63	/// Record a slice of bytes as an dependency of the process being cached
64		pub fn addSlice(self: *@This(), val: []const u8) void {
65		debug.assert(self.manifest_file == null);
	64	pub fn addSlice(self: *CacheHash, val: []const u8) void {
	65	assert(self.manifest_file == null);
66	66
67	67	self.blake3.update(val);
68	68	self.blake3.update(&[_]u8{0});
...	...	@@ -70,8 +70,8 @@ pub const CacheHash = struct {
70	70
71	71	/// Convert the input value into bytes and record it as a dependency of the
72	72	/// process being cached
73		pub fn add(self: *@This(), val: var) void {
74		debug.assert(self.manifest_file == null);
	73	pub fn add(self: *CacheHash, val: var) void {
	74	assert(self.manifest_file == null);
75	75
76	76	const valPtr = switch (@typeInfo(@TypeOf(val))) {
77	77	.Int => &val,
...	...	@@ -96,16 +96,22 @@ pub const CacheHash = struct {
96	96	/// ```
97	97	/// var file_contents = cache_hash.files.items[file_index].contents.?;
98	98	/// ```
99		pub fn addFile(self: *@This(), file_path: []const u8, max_file_size: ?usize) !usize {
100		debug.assert(self.manifest_file == null);
	99	pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
	100	assert(self.manifest_file == null);
	101
	102	try self.files.ensureCapacity(self.files.items.len + 1);
	103	const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
101	104
102	105	const idx = self.files.items.len;
103		var cache_hash_file = try self.files.addOne();
104		cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
105		cache_hash_file.max_file_size = max_file_size;
106		cache_hash_file.contents = null;
	106	self.files.addOneAssumeCapacity().* = .{
	107	.path = resolved_path,
	108	.contents = null,
	109	.max_file_size = max_file_size,
	110	.stat = undefined,
	111	.bin_digest = undefined,
	112	};
107	113
108		self.addSlice(cache_hash_file.path.?);
	114	self.addSlice(resolved_path);
109	115
110	116	return idx;
111	117	}
...	...	@@ -118,8 +124,8 @@ pub const CacheHash = struct {
118	124	/// acquire the lock.
119	125	///
120	126	/// The lock on the manifest file is released when `CacheHash.release` is called.
121		pub fn hit(self: *@This()) !?[BASE64_DIGEST_LEN]u8 {
122		debug.assert(self.manifest_file == null);
	127	pub fn hit(self: *CacheHash) !?[BASE64_DIGEST_LEN]u8 {
	128	assert(self.manifest_file == null);
123	129
124	130	var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
125	131	self.blake3.final(&bin_digest);
...	...	@@ -129,8 +135,8 @@ pub const CacheHash = struct {
129	135	self.blake3 = Blake3.init();
130	136	self.blake3.update(&bin_digest);
131	137
132		const manifest_file_path = try fmt.allocPrint(self.alloc, "{}.txt", .{self.b64_digest});
133		defer self.alloc.free(manifest_file_path);
	138	const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});
	139	defer self.allocator.free(manifest_file_path);
134	140
135	141	if (self.files.items.len != 0) {
136	142	self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{
...	...	@@ -159,8 +165,8 @@ pub const CacheHash = struct {
159	165	};
160	166	}
161	167
162		const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.alloc, MANIFEST_FILE_SIZE_MAX);
163		defer self.alloc.free(file_contents);
	168	const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.allocator, MANIFEST_FILE_SIZE_MAX);
	169	defer self.allocator.free(file_contents);
164	170
165	171	const input_file_count = self.files.items.len;
166	172	var any_file_changed = false;
...	...	@@ -169,15 +175,17 @@ pub const CacheHash = struct {
169	175	while (line_iter.next()) \|line\| {
170	176	defer idx += 1;
171	177
172		var cache_hash_file: *File = undefined;
173		if (idx < input_file_count) {
174		cache_hash_file = &self.files.items[idx];
175		} else {
176		cache_hash_file = try self.files.addOne();
177		cache_hash_file.path = null;
178		cache_hash_file.max_file_size = null;
179		cache_hash_file.contents = null;
180		}
	178	const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
	179	const new = try self.files.addOne();
	180	new.* = .{
	181	.path = null,
	182	.contents = null,
	183	.max_file_size = null,
	184	.stat = undefined,
	185	.bin_digest = undefined,
	186	};
	187	break :blk new;
	188	};
181	189
182	190	var iter = mem.tokenize(line, " ");
183	191	const inode = iter.next() orelse return error.InvalidFormat;
...	...	@@ -185,7 +193,7 @@ pub const CacheHash = struct {
185	193	const digest_str = iter.next() orelse return error.InvalidFormat;
186	194	const file_path = iter.rest();
187	195
188		cache_hash_file.stat.inode = fmt.parseInt(os.ino_t, mtime_nsec_str, 10) catch return error.InvalidFormat;
	196	cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, mtime_nsec_str, 10) catch return error.InvalidFormat;
189	197	cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
190	198	base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
191	199
...	...	@@ -199,7 +207,7 @@ pub const CacheHash = struct {
199	207	}
200	208
201	209	if (cache_hash_file.path == null) {
202		cache_hash_file.path = try mem.dupe(self.alloc, u8, file_path);
	210	cache_hash_file.path = try mem.dupe(self.allocator, u8, file_path);
203	211	}
204	212
205	213	const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
...	...	@@ -216,16 +224,16 @@ pub const CacheHash = struct {
216	224
217	225	cache_hash_file.stat = actual_stat;
218	226
219		if (is_problematic_timestamp(cache_hash_file.stat.mtime)) {
	227	if (isProblematicTimestamp(cache_hash_file.stat.mtime)) {
220	228	cache_hash_file.stat.mtime = 0;
221	229	cache_hash_file.stat.inode = 0;
222	230	}
223	231
224	232	var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
225		cache_hash_file.contents = try hash_file(self.alloc, &actual_digest, &this_file, cache_hash_file.max_file_size);
	233	try hashFile(this_file, &actual_digest);
226	234
227	235	if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
228		mem.copy(u8, &cache_hash_file.bin_digest, &actual_digest);
	236	cache_hash_file.bin_digest = actual_digest;
229	237	// keep going until we have the input file digests
230	238	any_file_changed = true;
231	239	}
...	...	@@ -245,9 +253,9 @@ pub const CacheHash = struct {
245	253
246	254	// Remove files not in the initial hash
247	255	for (self.files.items[input_file_count..]) \|*file\| {
248		file.deinit(self.alloc);
	256	file.deinit(self.allocator);
249	257	}
250		try self.files.resize(input_file_count);
	258	self.files.shrink(input_file_count);
251	259
252	260	for (self.files.items) \|file\| {
253	261	self.blake3.update(&file.bin_digest);
...	...	@@ -258,10 +266,8 @@ pub const CacheHash = struct {
258	266	if (idx < input_file_count) {
259	267	self.manifest_dirty = true;
260	268	while (idx < input_file_count) : (idx += 1) {
261		var cache_hash_file = &self.files.items[idx];
262		const contents = self.populate_file_hash(cache_hash_file) catch \|err\| {
263		return error.CacheUnavailable;
264		};
	269	const ch_file = &self.files.items[idx];
	270	try self.populateFileHash(ch_file);
265	271	}
266	272	return null;
267	273	}
...	...	@@ -269,59 +275,97 @@ pub const CacheHash = struct {
269	275	return self.final();
270	276	}
271	277
272		fn populate_file_hash_fetch(self: @This(), otherAlloc: mem.Allocator, cache_hash_file: *File) !?[]u8 {
273		debug.assert(cache_hash_file.path != null);
274
275		const this_file = try fs.cwd().openFile(cache_hash_file.path.?, .{});
276		defer this_file.close();
	278	fn populateFileHash(self: CacheHash, ch_file: File) !void {
	279	const file = try fs.cwd().openFile(ch_file.path.?, .{});
	280	defer file.close();
277	281
278		cache_hash_file.stat = try this_file.stat();
	282	ch_file.stat = try file.stat();
279	283
280		if (is_problematic_timestamp(cache_hash_file.stat.mtime)) {
281		cache_hash_file.stat.mtime = 0;
282		cache_hash_file.stat.inode = 0;
	284	if (isProblematicTimestamp(ch_file.stat.mtime)) {
	285	ch_file.stat.mtime = 0;
	286	ch_file.stat.inode = 0;
283	287	}
284	288
285		const contents = try hash_file(otherAlloc, &cache_hash_file.bin_digest, &this_file, cache_hash_file.max_file_size);
286		self.blake3.update(&cache_hash_file.bin_digest);
	289	if (ch_file.max_file_size) \|max_file_size\| {
	290	if (ch_file.stat.size > max_file_size) {
	291	return error.FileTooBig;
	292	}
287	293
288		return contents;
289		}
	294	const contents = try self.allocator.alloc(u8, ch_file.stat.size);
	295	errdefer self.allocator.free(contents);
	296
	297	// Hash while reading from disk, to keep the contents in the cpu cache while
	298	// doing hashing.
	299	var blake3 = Blake3.init();
	300	var off: usize = 0;
	301	while (true) {
	302	// give me everything you've got, captain
	303	const bytes_read = try file.read(contents[off..]);
	304	if (bytes_read == 0) break;
	305	blake3.update(contents[off..][0..bytes_read]);
	306	off += bytes_read;
	307	}
	308	blake3.final(&ch_file.bin_digest);
290	309
291		fn populate_file_hash(self: @This(), cache_hash_file: File) !void {
292		cache_hash_file.contents = try self.populate_file_hash_fetch(self.alloc, cache_hash_file);
	310	ch_file.contents = contents;
	311	} else {
	312	try hashFile(file, &ch_file.bin_digest);
	313	}
	314
	315	self.blake3.update(&ch_file.bin_digest);
293	316	}
294	317
295	318	/// Add a file as a dependency of process being cached, after the initial hash has been
296	319	/// calculated. This is useful for processes that don't know the all the files that
297	320	/// are depended on ahead of time. For example, a source file that can import other files
298	321	/// will need to be recompiled if the imported file is changed.
299		///
300		/// Returns the contents of the file, allocated with the given allocator.
301		pub fn addFilePostFetch(self: @This(), otherAlloc: mem.Allocator, file_path: []const u8, max_file_size_opt: ?usize) !?[]u8 {
302		debug.assert(self.manifest_file != null);
303
304		var cache_hash_file = try self.files.addOne();
305		cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
306		cache_hash_file.max_file_size = max_file_size_opt;
307		cache_hash_file.contents = null;
	322	pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
	323	assert(self.manifest_file != null);
	324
	325	const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
	326	errdefer self.allocator.free(resolved_path);
	327
	328	const new_ch_file = try self.files.addOne();
	329	new_ch_file.* = .{
	330	.path = resolved_path,
	331	.max_file_size = max_file_size,
	332	.stat = undefined,
	333	.bin_digest = undefined,
	334	.contents = null,
	335	};
	336	errdefer self.files.shrink(self.files.items.len - 1);
308	337
309		const contents = try self.populate_file_hash_fetch(otherAlloc, cache_hash_file);
	338	try self.populateFileHash(new_ch_file);
310	339
311		return contents;
	340	return new_ch_file.contents.?;
312	341	}
313	342
314	343	/// Add a file as a dependency of process being cached, after the initial hash has been
315	344	/// calculated. This is useful for processes that don't know the all the files that
316	345	/// are depended on ahead of time. For example, a source file that can import other files
317	346	/// will need to be recompiled if the imported file is changed.
318		pub fn addFilePost(self: *@This(), file_path: []const u8) !void {
319		_ = try self.addFilePostFetch(self.alloc, file_path, null);
	347	pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
	348	assert(self.manifest_file != null);
	349
	350	const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
	351	errdefer self.allocator.free(resolved_path);
	352
	353	const new_ch_file = try self.files.addOne();
	354	new_ch_file.* = .{
	355	.path = resolved_path,
	356	.max_file_size = null,
	357	.stat = undefined,
	358	.bin_digest = undefined,
	359	.contents = null,
	360	};
	361	errdefer self.files.shrink(self.files.items.len - 1);
	362
	363	try self.populateFileHash(new_ch_file);
320	364	}
321	365
322	366	/// Returns a base64 encoded hash of the inputs.
323		pub fn final(self: *@This()) [BASE64_DIGEST_LEN]u8 {
324		debug.assert(self.manifest_file != null);
	367	pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
	368	assert(self.manifest_file != null);
325	369
326	370	// We don't close the manifest file yet, because we want to
327	371	// keep it locked until the API user is done using it.
...	...	@@ -338,11 +382,11 @@ pub const CacheHash = struct {
338	382	return out_digest;
339	383	}
340	384
341		pub fn write_manifest(self: *@This()) !void {
342		debug.assert(self.manifest_file != null);
	385	pub fn writeManifest(self: *CacheHash) !void {
	386	assert(self.manifest_file != null);
343	387
344	388	var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
345		var contents = ArrayList(u8).init(self.alloc);
	389	var contents = ArrayList(u8).init(self.allocator);
346	390	var outStream = contents.outStream();
347	391	defer contents.deinit();
348	392
...	...	@@ -351,68 +395,78 @@ pub const CacheHash = struct {
351	395	try outStream.print("{} {} {} {}\n", .{ file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path });
352	396	}
353	397
354		try self.manifest_file.?.seekTo(0);
355		try self.manifest_file.?.writeAll(contents.items);
	398	try self.manifest_file.?.pwriteAll(contents.items, 0);
	399	self.manifest_dirty = false;
356	400	}
357	401
358	402	/// Releases the manifest file and frees any memory the CacheHash was using.
359	403	/// `CacheHash.hit` must be called first.
360	404	///
361	405	/// Will also attempt to write to the manifest file if the manifest is dirty.
362		/// Writing to the manifest file is the only way that this file can return an
363		/// error.
364		pub fn release(self: *@This()) !void {
	406	/// Writing to the manifest file can fail, but this function ignores those errors.
	407	/// To detect failures from writing the manifest, one may explicitly call
	408	/// `writeManifest` before `release`.
	409	pub fn release(self: *CacheHash) void {
365	410	if (self.manifest_file) \|file\| {
366	411	if (self.manifest_dirty) {
367		try self.write_manifest();
	412	// To handle these errors, API users should call
	413	// writeManifest before release().
	414	self.writeManifest() catch {};
368	415	}
369	416
370	417	file.close();
371	418	}
372	419
373	420	for (self.files.items) \|*file\| {
374		file.deinit(self.alloc);
	421	file.deinit(self.allocator);
375	422	}
376	423	self.files.deinit();
377	424	self.manifest_dir.close();
378	425	}
379	426	};
380	427
381		/// Hash the file, and return the contents as an array
382		fn hash_file(alloc: Allocator, bin_digest: []u8, handle: const fs.File, max_file_size_opt: ?usize) !?[]u8 {
	428	fn hashFile(file: fs.File, bin_digest: []u8) !void {
383	429	var blake3 = Blake3.init();
384		var in_stream = handle.inStream();
385
386		if (max_file_size_opt) \|max_file_size\| {
387		const contents = try in_stream.readAllAlloc(alloc, max_file_size);
388
389		blake3.update(contents);
	430	var buf: [1024]u8 = undefined;
390	431
391		blake3.final(bin_digest);
392
393		return contents;
394		} else {
395		var buf: [1024]u8 = undefined;
396
397		while (true) {
398		const bytes_read = try in_stream.read(buf[0..]);
399		if (bytes_read == 0) break;
400		blake3.update(buf[0..bytes_read]);
401		}
402
403		blake3.final(bin_digest);
404		return null;
	432	while (true) {
	433	const bytes_read = try file.read(&buf);
	434	if (bytes_read == 0) break;
	435	blake3.update(buf[0..bytes_read]);
405	436	}
	437
	438	blake3.final(bin_digest);
406	439	}
407	440
408	441	/// If the wall clock time, rounded to the same precision as the
409	442	/// mtime, is equal to the mtime, then we cannot rely on this mtime
410	443	/// yet. We will instead save an mtime value that indicates the hash
411	444	/// must be unconditionally computed.
412		fn is_problematic_timestamp(file_mtime_ns: i64) bool {
413		const now_ms = time.milliTimestamp();
414		const file_mtime_ms = @divFloor(file_mtime_ns, time.millisecond);
415		return now_ms == file_mtime_ms;
	445	/// This function recognizes the precision of mtime by looking at trailing
	446	/// zero bits of the seconds and nanoseconds.
	447	fn isProblematicTimestamp(fs_clock: i128) bool {
	448	const wall_clock = std.time.nanoTimestamp();
	449
	450	// We have to break the nanoseconds into seconds and remainder nanoseconds
	451	// to detect precision of seconds, because looking at the zero bits in base
	452	// 2 would not detect precision of the seconds value.
	453	const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s));
	454	const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s));
	455	var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s));
	456	var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s));
	457
	458	// First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
	459	if (fs_nsec == 0) {
	460	wall_nsec = 0;
	461	if (fs_sec == 0) {
	462	wall_sec = 0;
	463	} else {
	464	wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec));
	465	}
	466	} else {
	467	wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec));
	468	}
	469	return wall_nsec == fs_nsec and wall_sec == fs_sec;
416	470	}
417	471
418	472	test "cache file and then recall it" {
...	...	@@ -423,12 +477,16 @@ test "cache file and then recall it" {
423	477
424	478	try cwd.writeFile(temp_file, "Hello, world!\n");
425	479
	480	while (isProblematicTimestamp(std.time.nanoTimestamp())) {
	481	std.time.sleep(1);
	482	}
	483
426	484	var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
427	485	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
428	486
429	487	{
430		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
431		defer ch.release() catch unreachable;
	488	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	489	defer ch.release();
432	490
433	491	ch.add(true);
434	492	ch.add(@as(u16, 1234));
...	...	@@ -436,13 +494,13 @@ test "cache file and then recall it" {
436	494	_ = try ch.addFile(temp_file, null);
437	495
438	496	// There should be nothing in the cache
439		testing.expectEqual(@as(?[64]u8, null), try ch.hit());
	497	testing.expectEqual(@as(?[32]u8, null), try ch.hit());
440	498
441	499	digest1 = ch.final();
442	500	}
443	501	{
444		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
445		defer ch.release() catch unreachable;
	502	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	503	defer ch.release();
446	504
447	505	ch.add(true);
448	506	ch.add(@as(u16, 1234));
...	...	@@ -460,13 +518,15 @@ test "cache file and then recall it" {
460	518	}
461	519
462	520	test "give problematic timestamp" {
463		const now_ns = @intCast(i64, time.milliTimestamp() * time.millisecond);
464		testing.expect(is_problematic_timestamp(now_ns));
	521	var fs_clock = std.time.nanoTimestamp();
	522	// to make it problematic, we make it only accurate to the second
	523	fs_clock = @divTrunc(fs_clock, std.time.ns_per_s);
	524	fs_clock *= std.time.ns_per_s;
	525	testing.expect(isProblematicTimestamp(fs_clock));
465	526	}
466	527
467	528	test "give nonproblematic timestamp" {
468		const now_ns = @intCast(i64, time.milliTimestamp() * time.millisecond) - 1000;
469		testing.expect(!is_problematic_timestamp(now_ns));
	529	testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s));
470	530	}
471	531
472	532	test "check that changing a file makes cache fail" {
...	...	@@ -479,18 +539,22 @@ test "check that changing a file makes cache fail" {
479	539
480	540	try cwd.writeFile(temp_file, original_temp_file_contents);
481	541
	542	while (isProblematicTimestamp(std.time.nanoTimestamp())) {
	543	std.time.sleep(1);
	544	}
	545
482	546	var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
483	547	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
484	548
485	549	{
486		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
487		defer ch.release() catch unreachable;
	550	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	551	defer ch.release();
488	552
489	553	ch.add("1234");
490	554	const temp_file_idx = try ch.addFile(temp_file, 100);
491	555
492	556	// There should be nothing in the cache
493		testing.expectEqual(@as(?[64]u8, null), try ch.hit());
	557	testing.expectEqual(@as(?[32]u8, null), try ch.hit());
494	558
495	559	testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
496	560
...	...	@@ -499,17 +563,22 @@ test "check that changing a file makes cache fail" {
499	563
500	564	try cwd.writeFile(temp_file, updated_temp_file_contents);
501	565
	566	while (isProblematicTimestamp(std.time.nanoTimestamp())) {
	567	std.time.sleep(1);
	568	}
	569
502	570	{
503		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
504		defer ch.release() catch unreachable;
	571	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	572	defer ch.release();
505	573
506	574	ch.add("1234");
507	575	const temp_file_idx = try ch.addFile(temp_file, 100);
508	576
509	577	// A file that we depend on has been updated, so the cache should not contain an entry for it
510		testing.expectEqual(@as(?[64]u8, null), try ch.hit());
	578	testing.expectEqual(@as(?[32]u8, null), try ch.hit());
511	579
512		testing.expect(mem.eql(u8, updated_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
	580	// The cache system does not keep the contents of re-hashed input files.
	581	testing.expect(ch.files.items[temp_file_idx].contents == null);
513	582
514	583	digest2 = ch.final();
515	584	}
...	...	@@ -529,19 +598,19 @@ test "no file inputs" {
529	598	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
530	599
531	600	{
532		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
533		defer ch.release() catch unreachable;
	601	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	602	defer ch.release();
534	603
535	604	ch.add("1234");
536	605
537	606	// There should be nothing in the cache
538		testing.expectEqual(@as(?[64]u8, null), try ch.hit());
	607	testing.expectEqual(@as(?[32]u8, null), try ch.hit());
539	608
540	609	digest1 = ch.final();
541	610	}
542	611	{
543		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
544		defer ch.release() catch unreachable;
	612	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	613	defer ch.release();
545	614
546	615	ch.add("1234");
547	616
...	...	@@ -561,55 +630,62 @@ test "CacheHashes with files added after initial hash work" {
561	630	try cwd.writeFile(temp_file1, "Hello, world!\n");
562	631	try cwd.writeFile(temp_file2, "Hello world the second!\n");
563	632
	633	while (isProblematicTimestamp(std.time.nanoTimestamp())) {
	634	std.time.sleep(1);
	635	}
	636
564	637	var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
565	638	var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
566	639	var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
567	640
568	641	{
569		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
570		defer ch.release() catch unreachable;
	642	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	643	defer ch.release();
571	644
572	645	ch.add("1234");
573	646	_ = try ch.addFile(temp_file1, null);
574	647
575	648	// There should be nothing in the cache
576		testing.expectEqual(@as(?[64]u8, null), try ch.hit());
	649	testing.expectEqual(@as(?[32]u8, null), try ch.hit());
577	650
578	651	_ = try ch.addFilePost(temp_file2);
579	652
580	653	digest1 = ch.final();
581	654	}
582	655	{
583		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
584		defer ch.release() catch unreachable;
	656	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	657	defer ch.release();
585	658
586	659	ch.add("1234");
587	660	_ = try ch.addFile(temp_file1, null);
588	661
589		// A file that we depend on has been updated, so the cache should not contain an entry for it
590	662	digest2 = (try ch.hit()).?;
591	663	}
	664	testing.expect(mem.eql(u8, &digest1, &digest2));
592	665
593	666	// Modify the file added after initial hash
594	667	try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
595	668
	669	while (isProblematicTimestamp(std.time.nanoTimestamp())) {
	670	std.time.sleep(1);
	671	}
	672
596	673	{
597		var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
598		defer ch.release() catch unreachable;
	674	var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
	675	defer ch.release();
599	676
600	677	ch.add("1234");
601	678	_ = try ch.addFile(temp_file1, null);
602	679
603	680	// A file that we depend on has been updated, so the cache should not contain an entry for it
604		testing.expectEqual(@as(?[64]u8, null), try ch.hit());
	681	testing.expectEqual(@as(?[32]u8, null), try ch.hit());
605	682
606	683	_ = try ch.addFilePost(temp_file2);
607	684
608	685	digest3 = ch.final();
609	686	}
610	687
611		testing.expect(mem.eql(u8, digest1[0..], digest2[0..]));
612		testing.expect(!mem.eql(u8, digest1[0..], digest3[0..]));
	688	testing.expect(!mem.eql(u8, &digest1, &digest3));
613	689
614	690	try cwd.deleteTree(temp_manifest_dir);
615	691	try cwd.deleteFile(temp_file1);

lib/std/fs/file.zig+3-3

...	...	@@ -27,6 +27,7 @@ pub const File = struct {
27	27	intended_io_mode: io.ModeOverride = io.default_mode,
28	28
29	29	pub const Mode = os.mode_t;
	30	pub const INode = os.ino_t;
30	31
31	32	pub const default_mode = switch (builtin.os.tag) {
32	33	.windows => 0,
...	...	@@ -215,15 +216,14 @@ pub const File = struct {
215	216
216	217	pub const Stat = struct {
217	218	/// A number that the system uses to point to the file metadata. This number is not guaranteed to be
218		/// unique across time, as some file systems may reuse an inode after it's file has been deleted.
	219	/// unique across time, as some file systems may reuse an inode after its file has been deleted.
219	220	/// Some systems may change the inode of a file over time.
220	221	///
221	222	/// On Linux, the inode _is_ structure that stores the metadata, and the inode _number_ is what
222	223	/// you see here: the index number of the inode.
223	224	///
224	225	/// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.
225		inode: os.ino_t,
226
	226	inode: INode,
227	227	size: u64,
228	228	mode: Mode,
229	229