authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 19:29:03-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 19:29:03-04:00
logcda102be020ef9c5c1425553ff611720f496f17e
treee43393b8686f5eb25fce441a4c41b4cb1190a0f7
parent6d5ec184ab1a1b8c155714801f7d6cb7ea6f5b8f

improvements to self-hosted cache hash system

* change miscellaneous things to more idiomatic zig style * change the digest length to 24 bytes instead of 48. This is still 70 more bits than UUIDs. For an analysis of probability of collisions, see: https://en.wikipedia.org/wiki/Universally_unique_identifier#Collisions * fix the API having the possibility of mismatched allocators * fix some error paths to behave properly * modify the guarantees about when file contents are loaded for input files * pwrite instead of seek + write * implement isProblematicTimestamp * fix tests with regards to a working isProblematicTimestamp function. this requires sleeping until the current timestamp becomes unproblematic. * introduce std.fs.File.INode, a cross platform type abstraction so that cache hash implementation does not need to reach into std.os.

2 files changed, 238 insertions(+), 162 deletions(-)

lib/std/cache_hash.zig+235-159
...@@ -1,18 +1,19 @@...@@ -1,18 +1,19 @@
1const Blake3 = @import("crypto.zig").Blake3;1const std = @import("std.zig");
2const fs = @import("fs.zig");2const Blake3 = std.crypto.Blake3;
3const base64 = @import("base64.zig");3const fs = std.fs;
4const ArrayList = @import("array_list.zig").ArrayList;4const base64 = std.base64;
5const debug = @import("debug.zig");5const ArrayList = std.ArrayList;
6const testing = @import("testing.zig");6const assert = std.debug.assert;
7const mem = @import("mem.zig");7const testing = std.testing;
8const fmt = @import("fmt.zig");8const mem = std.mem;
9const Allocator = mem.Allocator;9const fmt = std.fmt;
10const os = @import("os.zig");10const Allocator = std.mem.Allocator;
11const time = @import("time.zig");
1211
13const base64_encoder = fs.base64_encoder;12const base64_encoder = fs.base64_encoder;
14const base64_decoder = fs.base64_decoder;13const base64_decoder = fs.base64_decoder;
15const BIN_DIGEST_LEN = 48;14/// This is 70 more bits than UUIDs. For an analysis of probability of collisions, see:
15/// https://en.wikipedia.org/wiki/Universally_unique_identifier#Collisions
16const BIN_DIGEST_LEN = 24;
16const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);17const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
1718
18const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;19const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
...@@ -22,22 +23,23 @@ pub const File = struct {...@@ -22,22 +23,23 @@ pub const File = struct {
22 max_file_size: ?usize,23 max_file_size: ?usize,
23 stat: fs.File.Stat,24 stat: fs.File.Stat,
24 bin_digest: [BIN_DIGEST_LEN]u8,25 bin_digest: [BIN_DIGEST_LEN]u8,
25 contents: ?[]const u8 = null,26 contents: ?[]const u8,
2627
27 pub fn deinit(self: *@This(), alloc: *Allocator) void {28 pub fn deinit(self: *File, allocator: *Allocator) void {
28 if (self.path) |owned_slice| {29 if (self.path) |owned_slice| {
29 alloc.free(owned_slice);30 allocator.free(owned_slice);
30 self.path = null;31 self.path = null;
31 }32 }
32 if (self.contents) |contents| {33 if (self.contents) |contents| {
33 alloc.free(contents);34 allocator.free(contents);
34 self.contents = null;35 self.contents = null;
35 }36 }
37 self.* = undefined;
36 }38 }
37};39};
3840
39pub const CacheHash = struct {41pub const CacheHash = struct {
40 alloc: *Allocator,42 allocator: *Allocator,
41 blake3: Blake3,43 blake3: Blake3,
42 manifest_dir: fs.Dir,44 manifest_dir: fs.Dir,
43 manifest_file: ?fs.File,45 manifest_file: ?fs.File,
...@@ -45,24 +47,22 @@ pub const CacheHash = struct {...@@ -45,24 +47,22 @@ pub const CacheHash = struct {
45 files: ArrayList(File),47 files: ArrayList(File),
46 b64_digest: [BASE64_DIGEST_LEN]u8,48 b64_digest: [BASE64_DIGEST_LEN]u8,
4749
48 pub fn init(alloc: *Allocator, manifest_dir_path: []const u8) !@This() {50 /// Be sure to call release after successful initialization.
49 try fs.cwd().makePath(manifest_dir_path);51 pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
50 const manifest_dir = try fs.cwd().openDir(manifest_dir_path, .{});
51
52 return CacheHash{52 return CacheHash{
53 .alloc = alloc,53 .allocator = allocator,
54 .blake3 = Blake3.init(),54 .blake3 = Blake3.init(),
55 .manifest_dir = manifest_dir,55 .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
56 .manifest_file = null,56 .manifest_file = null,
57 .manifest_dirty = false,57 .manifest_dirty = false,
58 .files = ArrayList(File).init(alloc),58 .files = ArrayList(File).init(allocator),
59 .b64_digest = undefined,59 .b64_digest = undefined,
60 };60 };
61 }61 }
6262
63 /// Record a slice of bytes as an dependency of the process being cached63 /// Record a slice of bytes as an dependency of the process being cached
64 pub fn addSlice(self: *@This(), val: []const u8) void {64 pub fn addSlice(self: *CacheHash, val: []const u8) void {
65 debug.assert(self.manifest_file == null);65 assert(self.manifest_file == null);
6666
67 self.blake3.update(val);67 self.blake3.update(val);
68 self.blake3.update(&[_]u8{0});68 self.blake3.update(&[_]u8{0});
...@@ -70,8 +70,8 @@ pub const CacheHash = struct {...@@ -70,8 +70,8 @@ pub const CacheHash = struct {
7070
71 /// Convert the input value into bytes and record it as a dependency of the71 /// Convert the input value into bytes and record it as a dependency of the
72 /// process being cached72 /// process being cached
73 pub fn add(self: *@This(), val: var) void {73 pub fn add(self: *CacheHash, val: var) void {
74 debug.assert(self.manifest_file == null);74 assert(self.manifest_file == null);
7575
76 const valPtr = switch (@typeInfo(@TypeOf(val))) {76 const valPtr = switch (@typeInfo(@TypeOf(val))) {
77 .Int => &val,77 .Int => &val,
...@@ -96,16 +96,22 @@ pub const CacheHash = struct {...@@ -96,16 +96,22 @@ pub const CacheHash = struct {
96 /// ```96 /// ```
97 /// var file_contents = cache_hash.files.items[file_index].contents.?;97 /// var file_contents = cache_hash.files.items[file_index].contents.?;
98 /// ```98 /// ```
99 pub fn addFile(self: *@This(), file_path: []const u8, max_file_size: ?usize) !usize {99 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
100 debug.assert(self.manifest_file == null);100 assert(self.manifest_file == null);
101
102 try self.files.ensureCapacity(self.files.items.len + 1);
103 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
101104
102 const idx = self.files.items.len;105 const idx = self.files.items.len;
103 var cache_hash_file = try self.files.addOne();106 self.files.addOneAssumeCapacity().* = .{
104 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});107 .path = resolved_path,
105 cache_hash_file.max_file_size = max_file_size;108 .contents = null,
106 cache_hash_file.contents = null;109 .max_file_size = max_file_size,
110 .stat = undefined,
111 .bin_digest = undefined,
112 };
107113
108 self.addSlice(cache_hash_file.path.?);114 self.addSlice(resolved_path);
109115
110 return idx;116 return idx;
111 }117 }
...@@ -118,8 +124,8 @@ pub const CacheHash = struct {...@@ -118,8 +124,8 @@ pub const CacheHash = struct {
118 /// acquire the lock.124 /// acquire the lock.
119 ///125 ///
120 /// The lock on the manifest file is released when `CacheHash.release` is called.126 /// The lock on the manifest file is released when `CacheHash.release` is called.
121 pub fn hit(self: *@This()) !?[BASE64_DIGEST_LEN]u8 {127 pub fn hit(self: *CacheHash) !?[BASE64_DIGEST_LEN]u8 {
122 debug.assert(self.manifest_file == null);128 assert(self.manifest_file == null);
123129
124 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;130 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
125 self.blake3.final(&bin_digest);131 self.blake3.final(&bin_digest);
...@@ -129,8 +135,8 @@ pub const CacheHash = struct {...@@ -129,8 +135,8 @@ pub const CacheHash = struct {
129 self.blake3 = Blake3.init();135 self.blake3 = Blake3.init();
130 self.blake3.update(&bin_digest);136 self.blake3.update(&bin_digest);
131137
132 const manifest_file_path = try fmt.allocPrint(self.alloc, "{}.txt", .{self.b64_digest});138 const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});
133 defer self.alloc.free(manifest_file_path);139 defer self.allocator.free(manifest_file_path);
134140
135 if (self.files.items.len != 0) {141 if (self.files.items.len != 0) {
136 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{142 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{
...@@ -159,8 +165,8 @@ pub const CacheHash = struct {...@@ -159,8 +165,8 @@ pub const CacheHash = struct {
159 };165 };
160 }166 }
161167
162 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.alloc, MANIFEST_FILE_SIZE_MAX);168 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.allocator, MANIFEST_FILE_SIZE_MAX);
163 defer self.alloc.free(file_contents);169 defer self.allocator.free(file_contents);
164170
165 const input_file_count = self.files.items.len;171 const input_file_count = self.files.items.len;
166 var any_file_changed = false;172 var any_file_changed = false;
...@@ -169,15 +175,17 @@ pub const CacheHash = struct {...@@ -169,15 +175,17 @@ pub const CacheHash = struct {
169 while (line_iter.next()) |line| {175 while (line_iter.next()) |line| {
170 defer idx += 1;176 defer idx += 1;
171177
172 var cache_hash_file: *File = undefined;178 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
173 if (idx < input_file_count) {179 const new = try self.files.addOne();
174 cache_hash_file = &self.files.items[idx];180 new.* = .{
175 } else {181 .path = null,
176 cache_hash_file = try self.files.addOne();182 .contents = null,
177 cache_hash_file.path = null;183 .max_file_size = null,
178 cache_hash_file.max_file_size = null;184 .stat = undefined,
179 cache_hash_file.contents = null;185 .bin_digest = undefined,
180 }186 };
187 break :blk new;
188 };
181189
182 var iter = mem.tokenize(line, " ");190 var iter = mem.tokenize(line, " ");
183 const inode = iter.next() orelse return error.InvalidFormat;191 const inode = iter.next() orelse return error.InvalidFormat;
...@@ -185,7 +193,7 @@ pub const CacheHash = struct {...@@ -185,7 +193,7 @@ pub const CacheHash = struct {
185 const digest_str = iter.next() orelse return error.InvalidFormat;193 const digest_str = iter.next() orelse return error.InvalidFormat;
186 const file_path = iter.rest();194 const file_path = iter.rest();
187195
188 cache_hash_file.stat.inode = fmt.parseInt(os.ino_t, mtime_nsec_str, 10) catch return error.InvalidFormat;196 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, mtime_nsec_str, 10) catch return error.InvalidFormat;
189 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;197 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
190 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;198 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
191199
...@@ -199,7 +207,7 @@ pub const CacheHash = struct {...@@ -199,7 +207,7 @@ pub const CacheHash = struct {
199 }207 }
200208
201 if (cache_hash_file.path == null) {209 if (cache_hash_file.path == null) {
202 cache_hash_file.path = try mem.dupe(self.alloc, u8, file_path);210 cache_hash_file.path = try mem.dupe(self.allocator, u8, file_path);
203 }211 }
204212
205 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {213 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
...@@ -216,16 +224,16 @@ pub const CacheHash = struct {...@@ -216,16 +224,16 @@ pub const CacheHash = struct {
216224
217 cache_hash_file.stat = actual_stat;225 cache_hash_file.stat = actual_stat;
218226
219 if (is_problematic_timestamp(cache_hash_file.stat.mtime)) {227 if (isProblematicTimestamp(cache_hash_file.stat.mtime)) {
220 cache_hash_file.stat.mtime = 0;228 cache_hash_file.stat.mtime = 0;
221 cache_hash_file.stat.inode = 0;229 cache_hash_file.stat.inode = 0;
222 }230 }
223231
224 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;232 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
225 cache_hash_file.contents = try hash_file(self.alloc, &actual_digest, &this_file, cache_hash_file.max_file_size);233 try hashFile(this_file, &actual_digest);
226234
227 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {235 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
228 mem.copy(u8, &cache_hash_file.bin_digest, &actual_digest);236 cache_hash_file.bin_digest = actual_digest;
229 // keep going until we have the input file digests237 // keep going until we have the input file digests
230 any_file_changed = true;238 any_file_changed = true;
231 }239 }
...@@ -245,9 +253,9 @@ pub const CacheHash = struct {...@@ -245,9 +253,9 @@ pub const CacheHash = struct {
245253
246 // Remove files not in the initial hash254 // Remove files not in the initial hash
247 for (self.files.items[input_file_count..]) |*file| {255 for (self.files.items[input_file_count..]) |*file| {
248 file.deinit(self.alloc);256 file.deinit(self.allocator);
249 }257 }
250 try self.files.resize(input_file_count);258 self.files.shrink(input_file_count);
251259
252 for (self.files.items) |file| {260 for (self.files.items) |file| {
253 self.blake3.update(&file.bin_digest);261 self.blake3.update(&file.bin_digest);
...@@ -258,10 +266,8 @@ pub const CacheHash = struct {...@@ -258,10 +266,8 @@ pub const CacheHash = struct {
258 if (idx < input_file_count) {266 if (idx < input_file_count) {
259 self.manifest_dirty = true;267 self.manifest_dirty = true;
260 while (idx < input_file_count) : (idx += 1) {268 while (idx < input_file_count) : (idx += 1) {
261 var cache_hash_file = &self.files.items[idx];269 const ch_file = &self.files.items[idx];
262 const contents = self.populate_file_hash(cache_hash_file) catch |err| {270 try self.populateFileHash(ch_file);
263 return error.CacheUnavailable;
264 };
265 }271 }
266 return null;272 return null;
267 }273 }
...@@ -269,59 +275,97 @@ pub const CacheHash = struct {...@@ -269,59 +275,97 @@ pub const CacheHash = struct {
269 return self.final();275 return self.final();
270 }276 }
271277
272 fn populate_file_hash_fetch(self: *@This(), otherAlloc: *mem.Allocator, cache_hash_file: *File) !?[]u8 {278 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
273 debug.assert(cache_hash_file.path != null);279 const file = try fs.cwd().openFile(ch_file.path.?, .{});
274280 defer file.close();
275 const this_file = try fs.cwd().openFile(cache_hash_file.path.?, .{});
276 defer this_file.close();
277281
278 cache_hash_file.stat = try this_file.stat();282 ch_file.stat = try file.stat();
279283
280 if (is_problematic_timestamp(cache_hash_file.stat.mtime)) {284 if (isProblematicTimestamp(ch_file.stat.mtime)) {
281 cache_hash_file.stat.mtime = 0;285 ch_file.stat.mtime = 0;
282 cache_hash_file.stat.inode = 0;286 ch_file.stat.inode = 0;
283 }287 }
284288
285 const contents = try hash_file(otherAlloc, &cache_hash_file.bin_digest, &this_file, cache_hash_file.max_file_size);289 if (ch_file.max_file_size) |max_file_size| {
286 self.blake3.update(&cache_hash_file.bin_digest);290 if (ch_file.stat.size > max_file_size) {
291 return error.FileTooBig;
292 }
287293
288 return contents;294 const contents = try self.allocator.alloc(u8, ch_file.stat.size);
289 }295 errdefer self.allocator.free(contents);
296
297 // Hash while reading from disk, to keep the contents in the cpu cache while
298 // doing hashing.
299 var blake3 = Blake3.init();
300 var off: usize = 0;
301 while (true) {
302 // give me everything you've got, captain
303 const bytes_read = try file.read(contents[off..]);
304 if (bytes_read == 0) break;
305 blake3.update(contents[off..][0..bytes_read]);
306 off += bytes_read;
307 }
308 blake3.final(&ch_file.bin_digest);
290309
291 fn populate_file_hash(self: *@This(), cache_hash_file: *File) !void {310 ch_file.contents = contents;
292 cache_hash_file.contents = try self.populate_file_hash_fetch(self.alloc, cache_hash_file);311 } else {
312 try hashFile(file, &ch_file.bin_digest);
313 }
314
315 self.blake3.update(&ch_file.bin_digest);
293 }316 }
294317
295 /// Add a file as a dependency of process being cached, after the initial hash has been318 /// Add a file as a dependency of process being cached, after the initial hash has been
296 /// calculated. This is useful for processes that don't know the all the files that319 /// calculated. This is useful for processes that don't know the all the files that
297 /// are depended on ahead of time. For example, a source file that can import other files320 /// are depended on ahead of time. For example, a source file that can import other files
298 /// will need to be recompiled if the imported file is changed.321 /// will need to be recompiled if the imported file is changed.
299 ///322 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
300 /// Returns the contents of the file, allocated with the given allocator.323 assert(self.manifest_file != null);
301 pub fn addFilePostFetch(self: *@This(), otherAlloc: *mem.Allocator, file_path: []const u8, max_file_size_opt: ?usize) !?[]u8 {324
302 debug.assert(self.manifest_file != null);325 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
303326 errdefer self.allocator.free(resolved_path);
304 var cache_hash_file = try self.files.addOne();327
305 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});328 const new_ch_file = try self.files.addOne();
306 cache_hash_file.max_file_size = max_file_size_opt;329 new_ch_file.* = .{
307 cache_hash_file.contents = null;330 .path = resolved_path,
331 .max_file_size = max_file_size,
332 .stat = undefined,
333 .bin_digest = undefined,
334 .contents = null,
335 };
336 errdefer self.files.shrink(self.files.items.len - 1);
308337
309 const contents = try self.populate_file_hash_fetch(otherAlloc, cache_hash_file);338 try self.populateFileHash(new_ch_file);
310339
311 return contents;340 return new_ch_file.contents.?;
312 }341 }
313342
314 /// Add a file as a dependency of process being cached, after the initial hash has been343 /// Add a file as a dependency of process being cached, after the initial hash has been
315 /// calculated. This is useful for processes that don't know the all the files that344 /// calculated. This is useful for processes that don't know the all the files that
316 /// are depended on ahead of time. For example, a source file that can import other files345 /// are depended on ahead of time. For example, a source file that can import other files
317 /// will need to be recompiled if the imported file is changed.346 /// will need to be recompiled if the imported file is changed.
318 pub fn addFilePost(self: *@This(), file_path: []const u8) !void {347 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
319 _ = try self.addFilePostFetch(self.alloc, file_path, null);348 assert(self.manifest_file != null);
349
350 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
351 errdefer self.allocator.free(resolved_path);
352
353 const new_ch_file = try self.files.addOne();
354 new_ch_file.* = .{
355 .path = resolved_path,
356 .max_file_size = null,
357 .stat = undefined,
358 .bin_digest = undefined,
359 .contents = null,
360 };
361 errdefer self.files.shrink(self.files.items.len - 1);
362
363 try self.populateFileHash(new_ch_file);
320 }364 }
321365
322 /// Returns a base64 encoded hash of the inputs.366 /// Returns a base64 encoded hash of the inputs.
323 pub fn final(self: *@This()) [BASE64_DIGEST_LEN]u8 {367 pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
324 debug.assert(self.manifest_file != null);368 assert(self.manifest_file != null);
325369
326 // We don't close the manifest file yet, because we want to370 // We don't close the manifest file yet, because we want to
327 // keep it locked until the API user is done using it.371 // keep it locked until the API user is done using it.
...@@ -338,11 +382,11 @@ pub const CacheHash = struct {...@@ -338,11 +382,11 @@ pub const CacheHash = struct {
338 return out_digest;382 return out_digest;
339 }383 }
340384
341 pub fn write_manifest(self: *@This()) !void {385 pub fn writeManifest(self: *CacheHash) !void {
342 debug.assert(self.manifest_file != null);386 assert(self.manifest_file != null);
343387
344 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;388 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
345 var contents = ArrayList(u8).init(self.alloc);389 var contents = ArrayList(u8).init(self.allocator);
346 var outStream = contents.outStream();390 var outStream = contents.outStream();
347 defer contents.deinit();391 defer contents.deinit();
348392
...@@ -351,68 +395,78 @@ pub const CacheHash = struct {...@@ -351,68 +395,78 @@ pub const CacheHash = struct {
351 try outStream.print("{} {} {} {}\n", .{ file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path });395 try outStream.print("{} {} {} {}\n", .{ file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path });
352 }396 }
353397
354 try self.manifest_file.?.seekTo(0);398 try self.manifest_file.?.pwriteAll(contents.items, 0);
355 try self.manifest_file.?.writeAll(contents.items);399 self.manifest_dirty = false;
356 }400 }
357401
358 /// Releases the manifest file and frees any memory the CacheHash was using.402 /// Releases the manifest file and frees any memory the CacheHash was using.
359 /// `CacheHash.hit` must be called first.403 /// `CacheHash.hit` must be called first.
360 ///404 ///
361 /// Will also attempt to write to the manifest file if the manifest is dirty.405 /// Will also attempt to write to the manifest file if the manifest is dirty.
362 /// Writing to the manifest file is the only way that this file can return an406 /// Writing to the manifest file can fail, but this function ignores those errors.
363 /// error.407 /// To detect failures from writing the manifest, one may explicitly call
364 pub fn release(self: *@This()) !void {408 /// `writeManifest` before `release`.
409 pub fn release(self: *CacheHash) void {
365 if (self.manifest_file) |file| {410 if (self.manifest_file) |file| {
366 if (self.manifest_dirty) {411 if (self.manifest_dirty) {
367 try self.write_manifest();412 // To handle these errors, API users should call
413 // writeManifest before release().
414 self.writeManifest() catch {};
368 }415 }
369416
370 file.close();417 file.close();
371 }418 }
372419
373 for (self.files.items) |*file| {420 for (self.files.items) |*file| {
374 file.deinit(self.alloc);421 file.deinit(self.allocator);
375 }422 }
376 self.files.deinit();423 self.files.deinit();
377 self.manifest_dir.close();424 self.manifest_dir.close();
378 }425 }
379};426};
380427
381/// Hash the file, and return the contents as an array428fn hashFile(file: fs.File, bin_digest: []u8) !void {
382fn hash_file(alloc: *Allocator, bin_digest: []u8, handle: *const fs.File, max_file_size_opt: ?usize) !?[]u8 {
383 var blake3 = Blake3.init();429 var blake3 = Blake3.init();
384 var in_stream = handle.inStream();430 var buf: [1024]u8 = undefined;
385
386 if (max_file_size_opt) |max_file_size| {
387 const contents = try in_stream.readAllAlloc(alloc, max_file_size);
388
389 blake3.update(contents);
390431
391 blake3.final(bin_digest);432 while (true) {
392433 const bytes_read = try file.read(&buf);
393 return contents;434 if (bytes_read == 0) break;
394 } else {435 blake3.update(buf[0..bytes_read]);
395 var buf: [1024]u8 = undefined;
396
397 while (true) {
398 const bytes_read = try in_stream.read(buf[0..]);
399 if (bytes_read == 0) break;
400 blake3.update(buf[0..bytes_read]);
401 }
402
403 blake3.final(bin_digest);
404 return null;
405 }436 }
437
438 blake3.final(bin_digest);
406}439}
407440
408/// If the wall clock time, rounded to the same precision as the441/// If the wall clock time, rounded to the same precision as the
409/// mtime, is equal to the mtime, then we cannot rely on this mtime442/// mtime, is equal to the mtime, then we cannot rely on this mtime
410/// yet. We will instead save an mtime value that indicates the hash443/// yet. We will instead save an mtime value that indicates the hash
411/// must be unconditionally computed.444/// must be unconditionally computed.
412fn is_problematic_timestamp(file_mtime_ns: i64) bool {445/// This function recognizes the precision of mtime by looking at trailing
413 const now_ms = time.milliTimestamp();446/// zero bits of the seconds and nanoseconds.
414 const file_mtime_ms = @divFloor(file_mtime_ns, time.millisecond);447fn isProblematicTimestamp(fs_clock: i128) bool {
415 return now_ms == file_mtime_ms;448 const wall_clock = std.time.nanoTimestamp();
449
450 // We have to break the nanoseconds into seconds and remainder nanoseconds
451 // to detect precision of seconds, because looking at the zero bits in base
452 // 2 would not detect precision of the seconds value.
453 const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s));
454 const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s));
455 var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s));
456 var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s));
457
458 // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
459 if (fs_nsec == 0) {
460 wall_nsec = 0;
461 if (fs_sec == 0) {
462 wall_sec = 0;
463 } else {
464 wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec));
465 }
466 } else {
467 wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec));
468 }
469 return wall_nsec == fs_nsec and wall_sec == fs_sec;
416}470}
417471
418test "cache file and then recall it" {472test "cache file and then recall it" {
...@@ -423,12 +477,16 @@ test "cache file and then recall it" {...@@ -423,12 +477,16 @@ test "cache file and then recall it" {
423477
424 try cwd.writeFile(temp_file, "Hello, world!\n");478 try cwd.writeFile(temp_file, "Hello, world!\n");
425479
480 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
481 std.time.sleep(1);
482 }
483
426 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;484 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
427 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;485 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
428486
429 {487 {
430 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);488 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
431 defer ch.release() catch unreachable;489 defer ch.release();
432490
433 ch.add(true);491 ch.add(true);
434 ch.add(@as(u16, 1234));492 ch.add(@as(u16, 1234));
...@@ -436,13 +494,13 @@ test "cache file and then recall it" {...@@ -436,13 +494,13 @@ test "cache file and then recall it" {
436 _ = try ch.addFile(temp_file, null);494 _ = try ch.addFile(temp_file, null);
437495
438 // There should be nothing in the cache496 // There should be nothing in the cache
439 testing.expectEqual(@as(?[64]u8, null), try ch.hit());497 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
440498
441 digest1 = ch.final();499 digest1 = ch.final();
442 }500 }
443 {501 {
444 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);502 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
445 defer ch.release() catch unreachable;503 defer ch.release();
446504
447 ch.add(true);505 ch.add(true);
448 ch.add(@as(u16, 1234));506 ch.add(@as(u16, 1234));
...@@ -460,13 +518,15 @@ test "cache file and then recall it" {...@@ -460,13 +518,15 @@ test "cache file and then recall it" {
460}518}
461519
462test "give problematic timestamp" {520test "give problematic timestamp" {
463 const now_ns = @intCast(i64, time.milliTimestamp() * time.millisecond);521 var fs_clock = std.time.nanoTimestamp();
464 testing.expect(is_problematic_timestamp(now_ns));522 // to make it problematic, we make it only accurate to the second
523 fs_clock = @divTrunc(fs_clock, std.time.ns_per_s);
524 fs_clock *= std.time.ns_per_s;
525 testing.expect(isProblematicTimestamp(fs_clock));
465}526}
466527
467test "give nonproblematic timestamp" {528test "give nonproblematic timestamp" {
468 const now_ns = @intCast(i64, time.milliTimestamp() * time.millisecond) - 1000;529 testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s));
469 testing.expect(!is_problematic_timestamp(now_ns));
470}530}
471531
472test "check that changing a file makes cache fail" {532test "check that changing a file makes cache fail" {
...@@ -479,18 +539,22 @@ test "check that changing a file makes cache fail" {...@@ -479,18 +539,22 @@ test "check that changing a file makes cache fail" {
479539
480 try cwd.writeFile(temp_file, original_temp_file_contents);540 try cwd.writeFile(temp_file, original_temp_file_contents);
481541
542 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
543 std.time.sleep(1);
544 }
545
482 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;546 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
483 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;547 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
484548
485 {549 {
486 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);550 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
487 defer ch.release() catch unreachable;551 defer ch.release();
488552
489 ch.add("1234");553 ch.add("1234");
490 const temp_file_idx = try ch.addFile(temp_file, 100);554 const temp_file_idx = try ch.addFile(temp_file, 100);
491555
492 // There should be nothing in the cache556 // There should be nothing in the cache
493 testing.expectEqual(@as(?[64]u8, null), try ch.hit());557 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
494558
495 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));559 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
496560
...@@ -499,17 +563,22 @@ test "check that changing a file makes cache fail" {...@@ -499,17 +563,22 @@ test "check that changing a file makes cache fail" {
499563
500 try cwd.writeFile(temp_file, updated_temp_file_contents);564 try cwd.writeFile(temp_file, updated_temp_file_contents);
501565
566 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
567 std.time.sleep(1);
568 }
569
502 {570 {
503 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);571 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
504 defer ch.release() catch unreachable;572 defer ch.release();
505573
506 ch.add("1234");574 ch.add("1234");
507 const temp_file_idx = try ch.addFile(temp_file, 100);575 const temp_file_idx = try ch.addFile(temp_file, 100);
508576
509 // A file that we depend on has been updated, so the cache should not contain an entry for it577 // A file that we depend on has been updated, so the cache should not contain an entry for it
510 testing.expectEqual(@as(?[64]u8, null), try ch.hit());578 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
511579
512 testing.expect(mem.eql(u8, updated_temp_file_contents, ch.files.items[temp_file_idx].contents.?));580 // The cache system does not keep the contents of re-hashed input files.
581 testing.expect(ch.files.items[temp_file_idx].contents == null);
513582
514 digest2 = ch.final();583 digest2 = ch.final();
515 }584 }
...@@ -529,19 +598,19 @@ test "no file inputs" {...@@ -529,19 +598,19 @@ test "no file inputs" {
529 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;598 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
530599
531 {600 {
532 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);601 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
533 defer ch.release() catch unreachable;602 defer ch.release();
534603
535 ch.add("1234");604 ch.add("1234");
536605
537 // There should be nothing in the cache606 // There should be nothing in the cache
538 testing.expectEqual(@as(?[64]u8, null), try ch.hit());607 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
539608
540 digest1 = ch.final();609 digest1 = ch.final();
541 }610 }
542 {611 {
543 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);612 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
544 defer ch.release() catch unreachable;613 defer ch.release();
545614
546 ch.add("1234");615 ch.add("1234");
547616
...@@ -561,55 +630,62 @@ test "CacheHashes with files added after initial hash work" {...@@ -561,55 +630,62 @@ test "CacheHashes with files added after initial hash work" {
561 try cwd.writeFile(temp_file1, "Hello, world!\n");630 try cwd.writeFile(temp_file1, "Hello, world!\n");
562 try cwd.writeFile(temp_file2, "Hello world the second!\n");631 try cwd.writeFile(temp_file2, "Hello world the second!\n");
563632
633 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
634 std.time.sleep(1);
635 }
636
564 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;637 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
565 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;638 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
566 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;639 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
567640
568 {641 {
569 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);642 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
570 defer ch.release() catch unreachable;643 defer ch.release();
571644
572 ch.add("1234");645 ch.add("1234");
573 _ = try ch.addFile(temp_file1, null);646 _ = try ch.addFile(temp_file1, null);
574647
575 // There should be nothing in the cache648 // There should be nothing in the cache
576 testing.expectEqual(@as(?[64]u8, null), try ch.hit());649 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
577650
578 _ = try ch.addFilePost(temp_file2);651 _ = try ch.addFilePost(temp_file2);
579652
580 digest1 = ch.final();653 digest1 = ch.final();
581 }654 }
582 {655 {
583 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);656 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
584 defer ch.release() catch unreachable;657 defer ch.release();
585658
586 ch.add("1234");659 ch.add("1234");
587 _ = try ch.addFile(temp_file1, null);660 _ = try ch.addFile(temp_file1, null);
588661
589 // A file that we depend on has been updated, so the cache should not contain an entry for it
590 digest2 = (try ch.hit()).?;662 digest2 = (try ch.hit()).?;
591 }663 }
664 testing.expect(mem.eql(u8, &digest1, &digest2));
592665
593 // Modify the file added after initial hash666 // Modify the file added after initial hash
594 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");667 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
595668
669 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
670 std.time.sleep(1);
671 }
672
596 {673 {
597 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);674 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
598 defer ch.release() catch unreachable;675 defer ch.release();
599676
600 ch.add("1234");677 ch.add("1234");
601 _ = try ch.addFile(temp_file1, null);678 _ = try ch.addFile(temp_file1, null);
602679
603 // A file that we depend on has been updated, so the cache should not contain an entry for it680 // A file that we depend on has been updated, so the cache should not contain an entry for it
604 testing.expectEqual(@as(?[64]u8, null), try ch.hit());681 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
605682
606 _ = try ch.addFilePost(temp_file2);683 _ = try ch.addFilePost(temp_file2);
607684
608 digest3 = ch.final();685 digest3 = ch.final();
609 }686 }
610687
611 testing.expect(mem.eql(u8, digest1[0..], digest2[0..]));688 testing.expect(!mem.eql(u8, &digest1, &digest3));
612 testing.expect(!mem.eql(u8, digest1[0..], digest3[0..]));
613689
614 try cwd.deleteTree(temp_manifest_dir);690 try cwd.deleteTree(temp_manifest_dir);
615 try cwd.deleteFile(temp_file1);691 try cwd.deleteFile(temp_file1);
lib/std/fs/file.zig+3-3
...@@ -27,6 +27,7 @@ pub const File = struct {...@@ -27,6 +27,7 @@ pub const File = struct {
27 intended_io_mode: io.ModeOverride = io.default_mode,27 intended_io_mode: io.ModeOverride = io.default_mode,
2828
29 pub const Mode = os.mode_t;29 pub const Mode = os.mode_t;
30 pub const INode = os.ino_t;
3031
31 pub const default_mode = switch (builtin.os.tag) {32 pub const default_mode = switch (builtin.os.tag) {
32 .windows => 0,33 .windows => 0,
...@@ -215,15 +216,14 @@ pub const File = struct {...@@ -215,15 +216,14 @@ pub const File = struct {
215216
216 pub const Stat = struct {217 pub const Stat = struct {
217 /// A number that the system uses to point to the file metadata. This number is not guaranteed to be218 /// A number that the system uses to point to the file metadata. This number is not guaranteed to be
218 /// unique across time, as some file systems may reuse an inode after it's file has been deleted.219 /// unique across time, as some file systems may reuse an inode after its file has been deleted.
219 /// Some systems may change the inode of a file over time.220 /// Some systems may change the inode of a file over time.
220 ///221 ///
221 /// On Linux, the inode _is_ structure that stores the metadata, and the inode _number_ is what222 /// On Linux, the inode _is_ structure that stores the metadata, and the inode _number_ is what
222 /// you see here: the index number of the inode.223 /// you see here: the index number of the inode.
223 ///224 ///
224 /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.225 /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.
225 inode: os.ino_t,226 inode: INode,
226
227 size: u64,227 size: u64,
228 mode: Mode,228 mode: Mode,
229229