authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 21:04:10-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 21:04:10-04:00
log49f3defe5bb85801d333f3871d36ce8db7862e54
tree554ce56f894c66013dc8917f8dd0ee58ad77d025
parent69ff89fd1207bc95adf7a349319973c6070ce540
parent7c8d0cc67876ce2059a8a7b2bc35f4948a2e42c8

Merge branch 'leroycep-feature-cache-hash-zig'

closes #4635

5 files changed, 721 insertions(+), 9 deletions(-)

lib/std/cache_hash.zig created+709
......@@ -0,0 +1,709 @@
1const std = @import("std.zig");
2const Blake3 = std.crypto.Blake3;
3const fs = std.fs;
4const base64 = std.base64;
5const ArrayList = std.ArrayList;
6const assert = std.debug.assert;
7const testing = std.testing;
8const mem = std.mem;
9const fmt = std.fmt;
10const Allocator = std.mem.Allocator;
11
12const base64_encoder = fs.base64_encoder;
13const base64_decoder = fs.base64_decoder;
14/// This is 70 more bits than UUIDs. For an analysis of probability of collisions, see:
15/// https://en.wikipedia.org/wiki/Universally_unique_identifier#Collisions
16const BIN_DIGEST_LEN = 24;
17const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
18
19const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
20
21pub const File = struct {
22 path: ?[]const u8,
23 max_file_size: ?usize,
24 stat: fs.File.Stat,
25 bin_digest: [BIN_DIGEST_LEN]u8,
26 contents: ?[]const u8,
27
28 pub fn deinit(self: *File, allocator: *Allocator) void {
29 if (self.path) |owned_slice| {
30 allocator.free(owned_slice);
31 self.path = null;
32 }
33 if (self.contents) |contents| {
34 allocator.free(contents);
35 self.contents = null;
36 }
37 self.* = undefined;
38 }
39};
40
41pub const CacheHash = struct {
42 allocator: *Allocator,
43 blake3: Blake3,
44 manifest_dir: fs.Dir,
45 manifest_file: ?fs.File,
46 manifest_dirty: bool,
47 files: ArrayList(File),
48 b64_digest: [BASE64_DIGEST_LEN]u8,
49
50 /// Be sure to call release after successful initialization.
51 pub fn init(allocator: *Allocator, dir: fs.Dir, manifest_dir_path: []const u8) !CacheHash {
52 return CacheHash{
53 .allocator = allocator,
54 .blake3 = Blake3.init(),
55 .manifest_dir = try dir.makeOpenPath(manifest_dir_path, .{}),
56 .manifest_file = null,
57 .manifest_dirty = false,
58 .files = ArrayList(File).init(allocator),
59 .b64_digest = undefined,
60 };
61 }
62
63 /// Record a slice of bytes as an dependency of the process being cached
64 pub fn addSlice(self: *CacheHash, val: []const u8) void {
65 assert(self.manifest_file == null);
66
67 self.blake3.update(val);
68 self.blake3.update(&[_]u8{0});
69 }
70
71 /// Convert the input value into bytes and record it as a dependency of the
72 /// process being cached
73 pub fn add(self: *CacheHash, val: var) void {
74 assert(self.manifest_file == null);
75
76 const valPtr = switch (@typeInfo(@TypeOf(val))) {
77 .Int => &val,
78 .Pointer => val,
79 else => &val,
80 };
81
82 self.addSlice(mem.asBytes(valPtr));
83 }
84
85 /// Add a file as a dependency of process being cached. When `CacheHash.hit` is
86 /// called, the file's contents will be checked to ensure that it matches
87 /// the contents from previous times.
88 ///
89 /// Max file size will be used to determine the amount of space to the file contents
90 /// are allowed to take up in memory. If max_file_size is null, then the contents
91 /// will not be loaded into memory.
92 ///
93 /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it
94 /// to access the contents of the file after calling `CacheHash.hit()` like so:
95 ///
96 /// ```
97 /// var file_contents = cache_hash.files.items[file_index].contents.?;
98 /// ```
99 pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize {
100 assert(self.manifest_file == null);
101
102 try self.files.ensureCapacity(self.files.items.len + 1);
103 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
104
105 const idx = self.files.items.len;
106 self.files.addOneAssumeCapacity().* = .{
107 .path = resolved_path,
108 .contents = null,
109 .max_file_size = max_file_size,
110 .stat = undefined,
111 .bin_digest = undefined,
112 };
113
114 self.addSlice(resolved_path);
115
116 return idx;
117 }
118
119 /// Check the cache to see if the input exists in it. If it exists, a base64 encoding
120 /// of it's hash will be returned; otherwise, null will be returned.
121 ///
122 /// This function will also acquire an exclusive lock to the manifest file. This means
123 /// that a process holding a CacheHash will block any other process attempting to
124 /// acquire the lock.
125 ///
126 /// The lock on the manifest file is released when `CacheHash.release` is called.
127 pub fn hit(self: *CacheHash) !?[BASE64_DIGEST_LEN]u8 {
128 assert(self.manifest_file == null);
129
130 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
131 self.blake3.final(&bin_digest);
132
133 base64_encoder.encode(self.b64_digest[0..], &bin_digest);
134
135 self.blake3 = Blake3.init();
136 self.blake3.update(&bin_digest);
137
138 const manifest_file_path = try fmt.allocPrint(self.allocator, "{}.txt", .{self.b64_digest});
139 defer self.allocator.free(manifest_file_path);
140
141 if (self.files.items.len != 0) {
142 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{
143 .read = true,
144 .truncate = false,
145 .lock = .Exclusive,
146 });
147 } else {
148 // If there are no file inputs, we check if the manifest file exists instead of
149 // comparing the hashes on the files used for the cached item
150 self.manifest_file = self.manifest_dir.openFile(manifest_file_path, .{
151 .read = true,
152 .write = true,
153 .lock = .Exclusive,
154 }) catch |err| switch (err) {
155 error.FileNotFound => {
156 self.manifest_dirty = true;
157 self.manifest_file = try self.manifest_dir.createFile(manifest_file_path, .{
158 .read = true,
159 .truncate = false,
160 .lock = .Exclusive,
161 });
162 return null;
163 },
164 else => |e| return e,
165 };
166 }
167
168 const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.allocator, MANIFEST_FILE_SIZE_MAX);
169 defer self.allocator.free(file_contents);
170
171 const input_file_count = self.files.items.len;
172 var any_file_changed = false;
173 var line_iter = mem.tokenize(file_contents, "\n");
174 var idx: usize = 0;
175 while (line_iter.next()) |line| {
176 defer idx += 1;
177
178 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
179 const new = try self.files.addOne();
180 new.* = .{
181 .path = null,
182 .contents = null,
183 .max_file_size = null,
184 .stat = undefined,
185 .bin_digest = undefined,
186 };
187 break :blk new;
188 };
189
190 var iter = mem.tokenize(line, " ");
191 const inode = iter.next() orelse return error.InvalidFormat;
192 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
193 const digest_str = iter.next() orelse return error.InvalidFormat;
194 const file_path = iter.rest();
195
196 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, mtime_nsec_str, 10) catch return error.InvalidFormat;
197 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
198 base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
199
200 if (file_path.len == 0) {
201 return error.InvalidFormat;
202 }
203 if (cache_hash_file.path) |p| {
204 if (!mem.eql(u8, file_path, p)) {
205 return error.InvalidFormat;
206 }
207 }
208
209 if (cache_hash_file.path == null) {
210 cache_hash_file.path = try mem.dupe(self.allocator, u8, file_path);
211 }
212
213 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch {
214 return error.CacheUnavailable;
215 };
216 defer this_file.close();
217
218 const actual_stat = try this_file.stat();
219 const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
220 const inode_match = actual_stat.inode == cache_hash_file.stat.inode;
221
222 if (!mtime_match or !inode_match) {
223 self.manifest_dirty = true;
224
225 cache_hash_file.stat = actual_stat;
226
227 if (isProblematicTimestamp(cache_hash_file.stat.mtime)) {
228 cache_hash_file.stat.mtime = 0;
229 cache_hash_file.stat.inode = 0;
230 }
231
232 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
233 try hashFile(this_file, &actual_digest);
234
235 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
236 cache_hash_file.bin_digest = actual_digest;
237 // keep going until we have the input file digests
238 any_file_changed = true;
239 }
240 }
241
242 if (!any_file_changed) {
243 self.blake3.update(&cache_hash_file.bin_digest);
244 }
245 }
246
247 if (any_file_changed) {
248 // cache miss
249 // keep the manifest file open
250 // reset the hash
251 self.blake3 = Blake3.init();
252 self.blake3.update(&bin_digest);
253
254 // Remove files not in the initial hash
255 for (self.files.items[input_file_count..]) |*file| {
256 file.deinit(self.allocator);
257 }
258 self.files.shrink(input_file_count);
259
260 for (self.files.items) |file| {
261 self.blake3.update(&file.bin_digest);
262 }
263 return null;
264 }
265
266 if (idx < input_file_count) {
267 self.manifest_dirty = true;
268 while (idx < input_file_count) : (idx += 1) {
269 const ch_file = &self.files.items[idx];
270 try self.populateFileHash(ch_file);
271 }
272 return null;
273 }
274
275 return self.final();
276 }
277
278 fn populateFileHash(self: *CacheHash, ch_file: *File) !void {
279 const file = try fs.cwd().openFile(ch_file.path.?, .{});
280 defer file.close();
281
282 ch_file.stat = try file.stat();
283
284 if (isProblematicTimestamp(ch_file.stat.mtime)) {
285 ch_file.stat.mtime = 0;
286 ch_file.stat.inode = 0;
287 }
288
289 if (ch_file.max_file_size) |max_file_size| {
290 if (ch_file.stat.size > max_file_size) {
291 return error.FileTooBig;
292 }
293
294 const contents = try self.allocator.alloc(u8, @intCast(usize, ch_file.stat.size));
295 errdefer self.allocator.free(contents);
296
297 // Hash while reading from disk, to keep the contents in the cpu cache while
298 // doing hashing.
299 var blake3 = Blake3.init();
300 var off: usize = 0;
301 while (true) {
302 // give me everything you've got, captain
303 const bytes_read = try file.read(contents[off..]);
304 if (bytes_read == 0) break;
305 blake3.update(contents[off..][0..bytes_read]);
306 off += bytes_read;
307 }
308 blake3.final(&ch_file.bin_digest);
309
310 ch_file.contents = contents;
311 } else {
312 try hashFile(file, &ch_file.bin_digest);
313 }
314
315 self.blake3.update(&ch_file.bin_digest);
316 }
317
318 /// Add a file as a dependency of process being cached, after the initial hash has been
319 /// calculated. This is useful for processes that don't know the all the files that
320 /// are depended on ahead of time. For example, a source file that can import other files
321 /// will need to be recompiled if the imported file is changed.
322 pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 {
323 assert(self.manifest_file != null);
324
325 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
326 errdefer self.allocator.free(resolved_path);
327
328 const new_ch_file = try self.files.addOne();
329 new_ch_file.* = .{
330 .path = resolved_path,
331 .max_file_size = max_file_size,
332 .stat = undefined,
333 .bin_digest = undefined,
334 .contents = null,
335 };
336 errdefer self.files.shrink(self.files.items.len - 1);
337
338 try self.populateFileHash(new_ch_file);
339
340 return new_ch_file.contents.?;
341 }
342
343 /// Add a file as a dependency of process being cached, after the initial hash has been
344 /// calculated. This is useful for processes that don't know the all the files that
345 /// are depended on ahead of time. For example, a source file that can import other files
346 /// will need to be recompiled if the imported file is changed.
347 pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void {
348 assert(self.manifest_file != null);
349
350 const resolved_path = try fs.path.resolve(self.allocator, &[_][]const u8{file_path});
351 errdefer self.allocator.free(resolved_path);
352
353 const new_ch_file = try self.files.addOne();
354 new_ch_file.* = .{
355 .path = resolved_path,
356 .max_file_size = null,
357 .stat = undefined,
358 .bin_digest = undefined,
359 .contents = null,
360 };
361 errdefer self.files.shrink(self.files.items.len - 1);
362
363 try self.populateFileHash(new_ch_file);
364 }
365
366 /// Returns a base64 encoded hash of the inputs.
367 pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 {
368 assert(self.manifest_file != null);
369
370 // We don't close the manifest file yet, because we want to
371 // keep it locked until the API user is done using it.
372 // We also don't write out the manifest yet, because until
373 // cache_release is called we still might be working on creating
374 // the artifacts to cache.
375
376 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
377 self.blake3.final(&bin_digest);
378
379 var out_digest: [BASE64_DIGEST_LEN]u8 = undefined;
380 base64_encoder.encode(&out_digest, &bin_digest);
381
382 return out_digest;
383 }
384
385 pub fn writeManifest(self: *CacheHash) !void {
386 assert(self.manifest_file != null);
387
388 var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined;
389 var contents = ArrayList(u8).init(self.allocator);
390 var outStream = contents.outStream();
391 defer contents.deinit();
392
393 for (self.files.items) |file| {
394 base64_encoder.encode(encoded_digest[0..], &file.bin_digest);
395 try outStream.print("{} {} {} {}\n", .{ file.stat.inode, file.stat.mtime, encoded_digest[0..], file.path });
396 }
397
398 try self.manifest_file.?.pwriteAll(contents.items, 0);
399 self.manifest_dirty = false;
400 }
401
402 /// Releases the manifest file and frees any memory the CacheHash was using.
403 /// `CacheHash.hit` must be called first.
404 ///
405 /// Will also attempt to write to the manifest file if the manifest is dirty.
406 /// Writing to the manifest file can fail, but this function ignores those errors.
407 /// To detect failures from writing the manifest, one may explicitly call
408 /// `writeManifest` before `release`.
409 pub fn release(self: *CacheHash) void {
410 if (self.manifest_file) |file| {
411 if (self.manifest_dirty) {
412 // To handle these errors, API users should call
413 // writeManifest before release().
414 self.writeManifest() catch {};
415 }
416
417 file.close();
418 }
419
420 for (self.files.items) |*file| {
421 file.deinit(self.allocator);
422 }
423 self.files.deinit();
424 self.manifest_dir.close();
425 }
426};
427
428fn hashFile(file: fs.File, bin_digest: []u8) !void {
429 var blake3 = Blake3.init();
430 var buf: [1024]u8 = undefined;
431
432 while (true) {
433 const bytes_read = try file.read(&buf);
434 if (bytes_read == 0) break;
435 blake3.update(buf[0..bytes_read]);
436 }
437
438 blake3.final(bin_digest);
439}
440
441/// If the wall clock time, rounded to the same precision as the
442/// mtime, is equal to the mtime, then we cannot rely on this mtime
443/// yet. We will instead save an mtime value that indicates the hash
444/// must be unconditionally computed.
445/// This function recognizes the precision of mtime by looking at trailing
446/// zero bits of the seconds and nanoseconds.
447fn isProblematicTimestamp(fs_clock: i128) bool {
448 const wall_clock = std.time.nanoTimestamp();
449
450 // We have to break the nanoseconds into seconds and remainder nanoseconds
451 // to detect precision of seconds, because looking at the zero bits in base
452 // 2 would not detect precision of the seconds value.
453 const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s));
454 const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s));
455 var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s));
456 var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s));
457
458 // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock.
459 if (fs_nsec == 0) {
460 wall_nsec = 0;
461 if (fs_sec == 0) {
462 wall_sec = 0;
463 } else {
464 wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec));
465 }
466 } else {
467 wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec));
468 }
469 return wall_nsec == fs_nsec and wall_sec == fs_sec;
470}
471
472test "cache file and then recall it" {
473 if (std.Target.current.os.tag == .wasi) {
474 // https://github.com/ziglang/zig/issues/5437
475 return error.SkipZigTest;
476 }
477 const cwd = fs.cwd();
478
479 const temp_file = "test.txt";
480 const temp_manifest_dir = "temp_manifest_dir";
481
482 try cwd.writeFile(temp_file, "Hello, world!\n");
483
484 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
485 std.time.sleep(1);
486 }
487
488 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
489 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
490
491 {
492 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
493 defer ch.release();
494
495 ch.add(true);
496 ch.add(@as(u16, 1234));
497 ch.add("1234");
498 _ = try ch.addFile(temp_file, null);
499
500 // There should be nothing in the cache
501 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
502
503 digest1 = ch.final();
504 }
505 {
506 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
507 defer ch.release();
508
509 ch.add(true);
510 ch.add(@as(u16, 1234));
511 ch.add("1234");
512 _ = try ch.addFile(temp_file, null);
513
514 // Cache hit! We just "built" the same file
515 digest2 = (try ch.hit()).?;
516 }
517
518 testing.expectEqual(digest1, digest2);
519
520 try cwd.deleteTree(temp_manifest_dir);
521 try cwd.deleteFile(temp_file);
522}
523
524test "give problematic timestamp" {
525 var fs_clock = std.time.nanoTimestamp();
526 // to make it problematic, we make it only accurate to the second
527 fs_clock = @divTrunc(fs_clock, std.time.ns_per_s);
528 fs_clock *= std.time.ns_per_s;
529 testing.expect(isProblematicTimestamp(fs_clock));
530}
531
532test "give nonproblematic timestamp" {
533 testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s));
534}
535
536test "check that changing a file makes cache fail" {
537 if (std.Target.current.os.tag == .wasi) {
538 // https://github.com/ziglang/zig/issues/5437
539 return error.SkipZigTest;
540 }
541 const cwd = fs.cwd();
542
543 const temp_file = "cache_hash_change_file_test.txt";
544 const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
545 const original_temp_file_contents = "Hello, world!\n";
546 const updated_temp_file_contents = "Hello, world; but updated!\n";
547
548 try cwd.writeFile(temp_file, original_temp_file_contents);
549
550 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
551 std.time.sleep(1);
552 }
553
554 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
555 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
556
557 {
558 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
559 defer ch.release();
560
561 ch.add("1234");
562 const temp_file_idx = try ch.addFile(temp_file, 100);
563
564 // There should be nothing in the cache
565 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
566
567 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
568
569 digest1 = ch.final();
570 }
571
572 try cwd.writeFile(temp_file, updated_temp_file_contents);
573
574 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
575 std.time.sleep(1);
576 }
577
578 {
579 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
580 defer ch.release();
581
582 ch.add("1234");
583 const temp_file_idx = try ch.addFile(temp_file, 100);
584
585 // A file that we depend on has been updated, so the cache should not contain an entry for it
586 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
587
588 // The cache system does not keep the contents of re-hashed input files.
589 testing.expect(ch.files.items[temp_file_idx].contents == null);
590
591 digest2 = ch.final();
592 }
593
594 testing.expect(!mem.eql(u8, digest1[0..], digest2[0..]));
595
596 try cwd.deleteTree(temp_manifest_dir);
597 try cwd.deleteFile(temp_file);
598}
599
600test "no file inputs" {
601 if (std.Target.current.os.tag == .wasi) {
602 // https://github.com/ziglang/zig/issues/5437
603 return error.SkipZigTest;
604 }
605 const cwd = fs.cwd();
606 const temp_manifest_dir = "no_file_inputs_manifest_dir";
607 defer cwd.deleteTree(temp_manifest_dir) catch unreachable;
608
609 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
610 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
611
612 {
613 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
614 defer ch.release();
615
616 ch.add("1234");
617
618 // There should be nothing in the cache
619 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
620
621 digest1 = ch.final();
622 }
623 {
624 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
625 defer ch.release();
626
627 ch.add("1234");
628
629 digest2 = (try ch.hit()).?;
630 }
631
632 testing.expectEqual(digest1, digest2);
633}
634
635test "CacheHashes with files added after initial hash work" {
636 if (std.Target.current.os.tag == .wasi) {
637 // https://github.com/ziglang/zig/issues/5437
638 return error.SkipZigTest;
639 }
640 const cwd = fs.cwd();
641
642 const temp_file1 = "cache_hash_post_file_test1.txt";
643 const temp_file2 = "cache_hash_post_file_test2.txt";
644 const temp_manifest_dir = "cache_hash_post_file_manifest_dir";
645
646 try cwd.writeFile(temp_file1, "Hello, world!\n");
647 try cwd.writeFile(temp_file2, "Hello world the second!\n");
648
649 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
650 std.time.sleep(1);
651 }
652
653 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
654 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
655 var digest3: [BASE64_DIGEST_LEN]u8 = undefined;
656
657 {
658 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
659 defer ch.release();
660
661 ch.add("1234");
662 _ = try ch.addFile(temp_file1, null);
663
664 // There should be nothing in the cache
665 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
666
667 _ = try ch.addFilePost(temp_file2);
668
669 digest1 = ch.final();
670 }
671 {
672 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
673 defer ch.release();
674
675 ch.add("1234");
676 _ = try ch.addFile(temp_file1, null);
677
678 digest2 = (try ch.hit()).?;
679 }
680 testing.expect(mem.eql(u8, &digest1, &digest2));
681
682 // Modify the file added after initial hash
683 try cwd.writeFile(temp_file2, "Hello world the second, updated\n");
684
685 while (isProblematicTimestamp(std.time.nanoTimestamp())) {
686 std.time.sleep(1);
687 }
688
689 {
690 var ch = try CacheHash.init(testing.allocator, cwd, temp_manifest_dir);
691 defer ch.release();
692
693 ch.add("1234");
694 _ = try ch.addFile(temp_file1, null);
695
696 // A file that we depend on has been updated, so the cache should not contain an entry for it
697 testing.expectEqual(@as(?[32]u8, null), try ch.hit());
698
699 _ = try ch.addFilePost(temp_file2);
700
701 digest3 = ch.final();
702 }
703
704 testing.expect(!mem.eql(u8, &digest1, &digest3));
705
706 try cwd.deleteTree(temp_manifest_dir);
707 try cwd.deleteFile(temp_file1);
708 try cwd.deleteFile(temp_file2);
709}
lib/std/fs.zig+7-5
......@@ -49,11 +49,13 @@ pub const MAX_PATH_BYTES = switch (builtin.os.tag) {
4949 else => @compileError("Unsupported OS"),
5050};
5151
52/// Base64, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.
53pub const base64_encoder = base64.Base64Encoder.init(
54 "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_",
55 base64.standard_pad_char,
56);
52pub const base64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
53
54/// Base64 encoder, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.
55pub const base64_encoder = base64.Base64Encoder.init(base64_alphabet, base64.standard_pad_char);
56
57/// Base64 decoder, replacing the standard `+/` with `-_` so that it can be used in a file name on any filesystem.
58pub const base64_decoder = base64.Base64Decoder.init(base64_alphabet, base64.standard_pad_char);
5759
5860/// Whether or not async file system syscalls need a dedicated thread because the operating
5961/// system does not support non-blocking I/O on the file system.
lib/std/fs/file.zig+3-3
......@@ -27,6 +27,7 @@ pub const File = struct {
2727 intended_io_mode: io.ModeOverride = io.default_mode,
2828
2929 pub const Mode = os.mode_t;
30 pub const INode = os.ino_t;
3031
3132 pub const default_mode = switch (builtin.os.tag) {
3233 .windows => 0,
......@@ -215,15 +216,14 @@ pub const File = struct {
215216
216217 pub const Stat = struct {
217218 /// A number that the system uses to point to the file metadata. This number is not guaranteed to be
218 /// unique across time, as some file systems may reuse an inode after it's file has been deleted.
219 /// unique across time, as some file systems may reuse an inode after its file has been deleted.
219220 /// Some systems may change the inode of a file over time.
220221 ///
221222 /// On Linux, the inode _is_ structure that stores the metadata, and the inode _number_ is what
222223 /// you see here: the index number of the inode.
223224 ///
224225 /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem.
225 inode: os.ino_t,
226
226 inode: INode,
227227 size: u64,
228228 mode: Mode,
229229
lib/std/os/linux.zig+1-1
......@@ -417,7 +417,7 @@ pub fn ftruncate(fd: i32, length: u64) usize {
417417 }
418418}
419419
420pub fn pwrite(fd: i32, buf: [*]const u8, count: usize, offset: usize) usize {
420pub fn pwrite(fd: i32, buf: [*]const u8, count: usize, offset: u64) usize {
421421 if (@hasField(SYS, "pwrite64")) {
422422 if (require_aligned_register_pair) {
423423 return syscall6(
lib/std/std.zig+1
......@@ -31,6 +31,7 @@ pub const base64 = @import("base64.zig");
3131pub const build = @import("build.zig");
3232pub const builtin = @import("builtin.zig");
3333pub const c = @import("c.zig");
34pub const cache_hash = @import("cache_hash.zig");
3435pub const coff = @import("coff.zig");
3536pub const crypto = @import("crypto.zig");
3637pub const cstr = @import("cstr.zig");