authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-12-10 17:43:42-08:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-12-10 18:11:12-08:00
logd37ee79535188263bd6a907eb26a48364c4c12f2
tree9cff3c562afce6f3ddedfa236a7a819d92a9728a
parentc172877b81f4eff50cf214eb553c9df108fbd9eb

std.Build.Cache.hit: more discipline in error handling

Previous commits 2b0929929d67e222ca6a9523a3a594ed456c4a51 4ea2f441df36cec61e1017f4d795d4037326c98c had this text: > There are no dir components, so you would think that this was > unreachable, however we have observed on macOS two processes racing to > do openat() with O_CREAT manifest in ENOENT. This appears to have been a misunderstanding based on the issue report #12138 and corresponding PR #12139 in which the steps to reproduce removed the cache directory in a loop which also executed detached Zig compiler processes. There is no evidence for the macOS kernel bug however the ENOENT is easily explained by the removal of the cache directory. This commit reverts those commits, ultimately reporting the ENOENT as an error rather than repeating the create file operation. However this commit also adds an explicit error set to `std.Build.Cache.hit` as well as changing the `failed_file_index` to a proper diagnostic field that fully communicates what failed, leading to more informative error messages on failure to check the cache. The equivalent failure when occuring for AstGen performs a fatal process kill, reasoning being that the compiler has an invariant of the cache directory not being yanked out from underneath it while executing. This could be made a more granular error in the future but I suspect such thing is not valuable to pursue. Related to #18340 but does not solve it.

6 files changed, 140 insertions(+), 49 deletions(-)

lib/std/Build/Cache.zig+86-31
......@@ -40,7 +40,7 @@ pub fn addPrefix(cache: *Cache, directory: Directory) void {
4040
4141/// Be sure to call `Manifest.deinit` after successful initialization.
4242pub fn obtain(cache: *Cache) Manifest {
43 return Manifest{
43 return .{
4444 .cache = cache,
4545 .hash = cache.hash,
4646 .manifest_file = null,
......@@ -99,9 +99,9 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
9999}
100100
101101fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8 {
102 const relative = try std.fs.path.relative(allocator, prefix, path);
102 const relative = try fs.path.relative(allocator, prefix, path);
103103 errdefer allocator.free(relative);
104 var component_iterator = std.fs.path.NativeComponentIterator.init(relative) catch {
104 var component_iterator = fs.path.NativeComponentIterator.init(relative) catch {
105105 return error.NotASubPath;
106106 };
107107 if (component_iterator.root() != null) {
......@@ -327,13 +327,27 @@ pub const Manifest = struct {
327327 want_refresh_timestamp: bool = true,
328328 files: Files = .{},
329329 hex_digest: HexDigest,
330 /// Populated when hit() returns an error because of one
331 /// of the files listed in the manifest.
332 failed_file_index: ?usize = null,
330 diagnostic: Diagnostic = .none,
333331 /// Keeps track of the last time we performed a file system write to observe
334332 /// what time the file system thinks it is, according to its own granularity.
335333 recent_problematic_timestamp: i128 = 0,
336334
335 pub const Diagnostic = union(enum) {
336 none,
337 manifest_create: fs.File.OpenError,
338 manifest_read: fs.File.ReadError,
339 manifest_lock: fs.File.LockError,
340 file_open: FileOp,
341 file_stat: FileOp,
342 file_read: FileOp,
343 file_hash: FileOp,
344
345 pub const FileOp = struct {
346 file_index: usize,
347 err: anyerror,
348 };
349 };
350
337351 pub const Files = std.ArrayHashMapUnmanaged(File, void, FilesContext, false);
338352
339353 pub const FilesContext = struct {
......@@ -452,6 +466,15 @@ pub const Manifest = struct {
452466 return self.addDepFileMaybePost(dir, dep_file_basename);
453467 }
454468
469 pub const HitError = error{
470 /// Unable to check the cache for a reason that has been recorded into
471 /// the `diagnostic` field.
472 CacheCheckFailed,
473 /// A cache manifest file exists however it could not be parsed.
474 InvalidFormat,
475 OutOfMemory,
476 };
477
455478 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
456479 /// A hex encoding of its hash is available by calling `final`.
457480 ///
......@@ -464,11 +487,11 @@ pub const Manifest = struct {
464487 /// The lock on the manifest file is released when `deinit` is called. As another
465488 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
466489 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
467 pub fn hit(self: *Manifest) !bool {
490 pub fn hit(self: *Manifest) HitError!bool {
468491 const gpa = self.cache.gpa;
469492 assert(self.manifest_file == null);
470493
471 self.failed_file_index = null;
494 self.diagnostic = .none;
472495
473496 const ext = ".txt";
474497 var manifest_file_path: [hex_digest_len + ext.len]u8 = undefined;
......@@ -496,17 +519,19 @@ pub const Manifest = struct {
496519 break;
497520 } else |err| switch (err) {
498521 error.WouldBlock => {
499 self.manifest_file = try self.cache.manifest_dir.openFile(&manifest_file_path, .{
522 self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{
500523 .mode = .read_write,
501524 .lock = .shared,
502 });
525 }) catch |e| {
526 self.diagnostic = .{ .manifest_create = e };
527 return error.CacheCheckFailed;
528 };
503529 break;
504530 },
505 // There are no dir components, so you would think that this was
506 // unreachable, however we have observed on macOS two processes racing
507 // to do openat() with O_CREAT manifest in ENOENT.
508 error.FileNotFound => continue,
509 else => |e| return e,
531 else => |e| {
532 self.diagnostic = .{ .manifest_create = e };
533 return error.CacheCheckFailed;
534 },
510535 }
511536 }
512537
......@@ -514,7 +539,14 @@ pub const Manifest = struct {
514539
515540 const input_file_count = self.files.entries.len;
516541 while (true) : (self.unhit(bin_digest, input_file_count)) {
517 const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
542 const file_contents = self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max) catch |err| switch (err) {
543 error.OutOfMemory => return error.OutOfMemory,
544 error.StreamTooLong => return error.OutOfMemory,
545 else => |e| {
546 self.diagnostic = .{ .manifest_read = e };
547 return error.CacheCheckFailed;
548 },
549 };
518550 defer gpa.free(file_contents);
519551
520552 var any_file_changed = false;
......@@ -526,8 +558,11 @@ pub const Manifest = struct {
526558 while (idx < input_file_count) : (idx += 1) {
527559 const ch_file = &self.files.keys()[idx];
528560 self.populateFileHash(ch_file) catch |err| {
529 self.failed_file_index = idx;
530 return err;
561 self.diagnostic = .{ .file_hash = .{
562 .file_index = idx,
563 .err = err,
564 } };
565 return error.CacheCheckFailed;
531566 };
532567 }
533568 return false;
......@@ -605,13 +640,22 @@ pub const Manifest = struct {
605640 if (try self.upgradeToExclusiveLock()) continue;
606641 return false;
607642 },
608 else => return error.CacheUnavailable,
643 else => |e| {
644 self.diagnostic = .{ .file_open = .{
645 .file_index = idx,
646 .err = e,
647 } };
648 return error.CacheCheckFailed;
649 },
609650 };
610651 defer this_file.close();
611652
612653 const actual_stat = this_file.stat() catch |err| {
613 self.failed_file_index = idx;
614 return err;
654 self.diagnostic = .{ .file_stat = .{
655 .file_index = idx,
656 .err = err,
657 } };
658 return error.CacheCheckFailed;
615659 };
616660 const size_match = actual_stat.size == cache_hash_file.stat.size;
617661 const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime;
......@@ -634,8 +678,11 @@ pub const Manifest = struct {
634678
635679 var actual_digest: BinDigest = undefined;
636680 hashFile(this_file, &actual_digest) catch |err| {
637 self.failed_file_index = idx;
638 return err;
681 self.diagnostic = .{ .file_read = .{
682 .file_index = idx,
683 .err = err,
684 } };
685 return error.CacheCheckFailed;
639686 };
640687
641688 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
......@@ -662,17 +709,22 @@ pub const Manifest = struct {
662709 if (try self.upgradeToExclusiveLock()) continue;
663710 self.manifest_dirty = true;
664711 while (idx < input_file_count) : (idx += 1) {
665 const ch_file = &self.files.keys()[idx];
666 self.populateFileHash(ch_file) catch |err| {
667 self.failed_file_index = idx;
668 return err;
712 self.populateFileHash(&self.files.keys()[idx]) catch |err| {
713 self.diagnostic = .{ .file_hash = .{
714 .file_index = idx,
715 .err = err,
716 } };
717 return error.CacheCheckFailed;
669718 };
670719 }
671720 return false;
672721 }
673722
674723 if (self.want_shared_lock) {
675 try self.downgradeToSharedLock();
724 self.downgradeToSharedLock() catch |err| {
725 self.diagnostic = .{ .manifest_lock = err };
726 return error.CacheCheckFailed;
727 };
676728 }
677729
678730 return true;
......@@ -1010,7 +1062,7 @@ pub const Manifest = struct {
10101062 self.have_exclusive_lock = false;
10111063 }
10121064
1013 fn upgradeToExclusiveLock(self: *Manifest) !bool {
1065 fn upgradeToExclusiveLock(self: *Manifest) error{CacheCheckFailed}!bool {
10141066 if (self.have_exclusive_lock) return false;
10151067 assert(self.manifest_file != null);
10161068
......@@ -1022,7 +1074,10 @@ pub const Manifest = struct {
10221074 // Here we intentionally have a period where the lock is released, in case there are
10231075 // other processes holding a shared lock.
10241076 manifest_file.unlock();
1025 try manifest_file.lock(.exclusive);
1077 manifest_file.lock(.exclusive) catch |err| {
1078 self.diagnostic = .{ .manifest_lock = err };
1079 return error.CacheCheckFailed;
1080 };
10261081 }
10271082 self.have_exclusive_lock = true;
10281083 return true;
......@@ -1132,7 +1187,7 @@ pub fn writeSmallFile(dir: fs.Dir, sub_path: []const u8, data: []const u8) !void
11321187 }
11331188}
11341189
1135fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) !void {
1190fn hashFile(file: fs.File, bin_digest: *[Hasher.mac_length]u8) fs.File.PReadError!void {
11361191 var buf: [1024]u8 = undefined;
11371192 var hasher = hasher_init;
11381193 var off: u64 = 0;
lib/std/Build/Step.zig+18-5
......@@ -754,11 +754,24 @@ pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool {
754754 return is_hit;
755755}
756756
757fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) anyerror {
758 const i = man.failed_file_index orelse return err;
759 const pp = man.files.keys()[i].prefixed_path;
760 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
761 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });
757fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: Build.Cache.Manifest.HitError) error{ OutOfMemory, MakeFailed } {
758 switch (err) {
759 error.CacheCheckFailed => switch (man.diagnostic) {
760 .none => unreachable,
761 .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {s} {s}", .{
762 @tagName(man.diagnostic), @errorName(e),
763 }),
764 .file_open, .file_stat, .file_read, .file_hash => |op| {
765 const pp = man.files.keys()[op.file_index].prefixed_path;
766 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
767 return s.fail("failed to check cache: '{s}{s}' {s} {s}", .{
768 prefix, pp.sub_path, @tagName(man.diagnostic), @errorName(op.err),
769 });
770 },
771 },
772 error.OutOfMemory => return error.OutOfMemory,
773 error.InvalidFormat => return s.fail("failed check cache: invalid manifest file format", .{}),
774 }
762775}
763776
764777/// Prefer `writeManifestAndWatch` unless you already added watch inputs
lib/std/posix.zig+4
......@@ -1537,6 +1537,10 @@ pub const OpenError = error{
15371537 ProcessFdQuotaExceeded,
15381538 SystemFdQuotaExceeded,
15391539 NoDevice,
1540 /// Either:
1541 /// * One of the path components does not exist.
1542 /// * Cwd was used, but cwd has been deleted.
1543 /// * The path associated with the open directory handle has been deleted.
15401544 FileNotFound,
15411545
15421546 /// The path exceeded `max_path_bytes` bytes.
src/Compilation.zig+23-8
......@@ -2048,15 +2048,30 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
20482048 whole.cache_manifest = &man;
20492049 try addNonIncrementalStuffToCacheManifest(comp, arena, &man);
20502050
2051 const is_hit = man.hit() catch |err| {
2052 const i = man.failed_file_index orelse return err;
2053 const pp = man.files.keys()[i].prefixed_path;
2054 const prefix = man.cache.prefixes()[pp.prefix];
2055 return comp.setMiscFailure(
2051 const is_hit = man.hit() catch |err| switch (err) {
2052 error.CacheCheckFailed => switch (man.diagnostic) {
2053 .none => unreachable,
2054 .manifest_create, .manifest_read, .manifest_lock => |e| return comp.setMiscFailure(
2055 .check_whole_cache,
2056 "failed to check cache: {s} {s}",
2057 .{ @tagName(man.diagnostic), @errorName(e) },
2058 ),
2059 .file_open, .file_stat, .file_read, .file_hash => |op| {
2060 const pp = man.files.keys()[op.file_index].prefixed_path;
2061 const prefix = man.cache.prefixes()[pp.prefix];
2062 return comp.setMiscFailure(
2063 .check_whole_cache,
2064 "failed to check cache: '{}{s}' {s} {s}",
2065 .{ prefix, pp.sub_path, @tagName(man.diagnostic), @errorName(op.err) },
2066 );
2067 },
2068 },
2069 error.OutOfMemory => return error.OutOfMemory,
2070 error.InvalidFormat => return comp.setMiscFailure(
20562071 .check_whole_cache,
2057 "unable to check cache: stat file '{}{s}' failed: {s}",
2058 .{ prefix, pp.sub_path, @errorName(err) },
2059 );
2072 "failed check cache: invalid manifest file format",
2073 .{},
2074 ),
20602075 };
20612076 if (is_hit) {
20622077 // In this case the cache hit contains the full set of file system inputs. Nice!
src/Zcu/PerThread.zig+8-4
......@@ -135,10 +135,14 @@ pub fn astGenFile(
135135 error.PipeBusy => unreachable, // it's not a pipe
136136 error.NoDevice => unreachable, // it's not a pipe
137137 error.WouldBlock => unreachable, // not asking for non-blocking I/O
138 // There are no dir components, so you would think that this was
139 // unreachable, however we have observed on macOS two processes racing
140 // to do openat() with O_CREAT manifest in ENOENT.
141 error.FileNotFound => continue,
138 error.FileNotFound => {
139 // Since there are no dir components this could only occur if
140 // `zir_dir` is deleted after the compiler process obtains an
141 // open directory handle.
142 std.process.fatal("cache directory '{}' unexpectedly removed during compiler execution", .{
143 cache_directory,
144 });
145 },
142146
143147 else => |e| return e, // Retryable errors are handled at callsite.
144148 };
src/link.zig+1-1
......@@ -768,7 +768,7 @@ pub const File = struct {
768768 /// TODO audit this error set. most of these should be collapsed into one error,
769769 /// and Diags.Flags should be updated to convey the meaning to the user.
770770 pub const FlushError = error{
771 CacheUnavailable,
771 CacheCheckFailed,
772772 CurrentWorkingDirectoryUnlinked,
773773 DivisionByZero,
774774 DllImportLibraryNotFound,