authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-10-30 16:51:13-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-10-30 17:04:09-07:00
log30b8b29f88362d18ea6523a859b29f7bc6dec622
tree87f7a6cc6bed6f9cd51a68bb7c143af393028a71
parenta5c96c49d0eedbce40ad9a58d1b236f1eaeabd03

re-apply "Fix C include files not being in `whole` cache (#11365)"

This reverts commit 06310e3d4eb47fed88b175891cb5865bb050f020, reapplying commit a430630002bf02162ccbf8d3eb10fd73e490cefd. I deeply apologize to @moosichu and those affected by this bug. The original fix was actually fine. When I reverted it, I misremembered how the Cache API works. I thought the fix was going to introduce nondeterminism into the hash, but I forgot that the order of files in the manifest doesn't actually matter when checking for a cache hit. Actually, it does matter a little bit. This fix has a subtle downside which is that it does introduce the possibility of false negatives when checking for cache hits of 2+ iterations ago. For example, if the code goes from "foo", to "bar", and then back to "foo", it may look like a cache miss when it should have been a hit because 2 iterations ago the code was the same. However, this is an uncommon use case, and all it does is cause a bit of wasted time and disk space. That said, my suggestion from earlier still applies and would be a nice follow-up enhancement to this fix: The proper solution to this is to, in whole cache mode, append the hash inputs to some data structure, and then after the compilation is complete, do some kind of sorting on the hash inputs so that they will be the same order every time, then apply them in sequence. No lock on the Cache object is needed for this scheme. closes #11063

4 files changed, 23 insertions(+), 10 deletions(-)

src/Cache.zig+1-1
...@@ -690,7 +690,7 @@ pub const Manifest = struct {...@@ -690,7 +690,7 @@ pub const Manifest = struct {
690 while (true) {690 while (true) {
691 switch (it.next() orelse return) {691 switch (it.next() orelse return) {
692 .target, .target_must_resolve => return,692 .target, .target_must_resolve => return,
693 .prereq => |bytes| try self.addFilePost(bytes),693 .prereq => |file_path| try self.addFilePost(file_path),
694 else => |err| {694 else => |err| {
695 try err.printError(error_buf.writer());695 try err.printError(error_buf.writer());
696 log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });696 log.err("failed parsing {s}: {s}", .{ dep_file_basename, error_buf.items });
src/Compilation.zig+9-1
...@@ -48,6 +48,7 @@ bin_file: *link.File,...@@ -48,6 +48,7 @@ bin_file: *link.File,
48c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},48c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
49/// This is a pointer to a local variable inside `update()`.49/// This is a pointer to a local variable inside `update()`.
50whole_cache_manifest: ?*Cache.Manifest = null,50whole_cache_manifest: ?*Cache.Manifest = null,
51whole_cache_manifest_mutex: std.Thread.Mutex = .{},
5152
52link_error_flags: link.File.ErrorFlags = .{},53link_error_flags: link.File.ErrorFlags = .{},
5354
...@@ -2199,8 +2200,8 @@ pub fn update(comp: *Compilation) !void {...@@ -2199,8 +2200,8 @@ pub fn update(comp: *Compilation) !void {
2199 // We are about to obtain this lock, so here we give other processes a chance first.2200 // We are about to obtain this lock, so here we give other processes a chance first.
2200 comp.bin_file.releaseLock();2201 comp.bin_file.releaseLock();
22012202
2202 comp.whole_cache_manifest = &man;
2203 man = comp.cache_parent.obtain();2203 man = comp.cache_parent.obtain();
2204 comp.whole_cache_manifest = &man;
2204 try comp.addNonIncrementalStuffToCacheManifest(&man);2205 try comp.addNonIncrementalStuffToCacheManifest(&man);
22052206
2206 const is_hit = man.hit() catch |err| {2207 const is_hit = man.hit() catch |err| {
...@@ -3598,6 +3599,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {...@@ -3598,6 +3599,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8) !CImportResult {
3598 const dep_basename = std.fs.path.basename(out_dep_path);3599 const dep_basename = std.fs.path.basename(out_dep_path);
3599 try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);3600 try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
3600 if (comp.whole_cache_manifest) |whole_cache_manifest| {3601 if (comp.whole_cache_manifest) |whole_cache_manifest| {
3602 comp.whole_cache_manifest_mutex.lock();
3603 defer comp.whole_cache_manifest_mutex.unlock();
3601 try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename);3604 try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename);
3602 }3605 }
36033606
...@@ -4060,6 +4063,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P...@@ -4060,6 +4063,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P
4060 const dep_basename = std.fs.path.basename(dep_file_path);4063 const dep_basename = std.fs.path.basename(dep_file_path);
4061 // Add the files depended on to the cache system.4064 // Add the files depended on to the cache system.
4062 try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);4065 try man.addDepFilePost(zig_cache_tmp_dir, dep_basename);
4066 if (comp.whole_cache_manifest) |whole_cache_manifest| {
4067 comp.whole_cache_manifest_mutex.lock();
4068 defer comp.whole_cache_manifest_mutex.unlock();
4069 try whole_cache_manifest.addDepFilePost(zig_cache_tmp_dir, dep_basename);
4070 }
4063 // Just to save disk space, we delete the file because it is never needed again.4071 // Just to save disk space, we delete the file because it is never needed again.
4064 zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| {4072 zig_cache_tmp_dir.deleteFile(dep_basename) catch |err| {
4065 log.warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) });4073 log.warn("failed to delete '{s}': {s}", .{ dep_file_path, @errorName(err) });
src/Module.zig+8-4
...@@ -4523,7 +4523,7 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {...@@ -4523,7 +4523,7 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {
4523 error.AnalysisFail => {},4523 error.AnalysisFail => {},
4524 }4524 }
45254525
4526 if (mod.comp.whole_cache_manifest) |man| {4526 if (mod.comp.whole_cache_manifest) |whole_cache_manifest| {
4527 const source = file.getSource(gpa) catch |err| {4527 const source = file.getSource(gpa) catch |err| {
4528 try reportRetryableFileError(mod, file, "unable to load source: {s}", .{@errorName(err)});4528 try reportRetryableFileError(mod, file, "unable to load source: {s}", .{@errorName(err)});
4529 return error.AnalysisFail;4529 return error.AnalysisFail;
...@@ -4541,7 +4541,9 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {...@@ -4541,7 +4541,9 @@ pub fn semaFile(mod: *Module, file: *File) SemaError!void {
4541 };4541 };
4542 errdefer gpa.free(resolved_path);4542 errdefer gpa.free(resolved_path);
45434543
4544 try man.addFilePostContents(resolved_path, source.bytes, source.stat);4544 mod.comp.whole_cache_manifest_mutex.lock();
4545 defer mod.comp.whole_cache_manifest_mutex.unlock();
4546 try whole_cache_manifest.addFilePostContents(resolved_path, source.bytes, source.stat);
4545 }4547 }
4546 } else {4548 } else {
4547 new_decl.analysis = .file_failure;4549 new_decl.analysis = .file_failure;
...@@ -5036,10 +5038,12 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb...@@ -5036,10 +5038,12 @@ pub fn embedFile(mod: *Module, cur_file: *File, rel_file_path: []const u8) !*Emb
5036 resolved_root_path, resolved_path, sub_file_path, rel_file_path,5038 resolved_root_path, resolved_path, sub_file_path, rel_file_path,
5037 });5039 });
50385040
5039 if (mod.comp.whole_cache_manifest) |man| {5041 if (mod.comp.whole_cache_manifest) |whole_cache_manifest| {
5040 const copied_resolved_path = try gpa.dupe(u8, resolved_path);5042 const copied_resolved_path = try gpa.dupe(u8, resolved_path);
5041 errdefer gpa.free(copied_resolved_path);5043 errdefer gpa.free(copied_resolved_path);
5042 try man.addFilePostContents(copied_resolved_path, bytes, stat);5044 mod.comp.whole_cache_manifest_mutex.lock();
5045 defer mod.comp.whole_cache_manifest_mutex.unlock();
5046 try whole_cache_manifest.addFilePostContents(copied_resolved_path, bytes, stat);
5043 }5047 }
50445048
5045 keep_resolved_path = true; // It's now owned by embed_table.5049 keep_resolved_path = true; // It's now owned by embed_table.
src/stage1.zig+5-4
...@@ -459,10 +459,11 @@ export fn stage2_fetch_file(...@@ -459,10 +459,11 @@ export fn stage2_fetch_file(
459 const comp = @intToPtr(*Compilation, stage1.userdata);459 const comp = @intToPtr(*Compilation, stage1.userdata);
460 const file_path = path_ptr[0..path_len];460 const file_path = path_ptr[0..path_len];
461 const max_file_size = std.math.maxInt(u32);461 const max_file_size = std.math.maxInt(u32);
462 const contents = if (comp.whole_cache_manifest) |man|462 const contents = if (comp.whole_cache_manifest) |man| blk: {
463 man.addFilePostFetch(file_path, max_file_size) catch return null463 comp.whole_cache_manifest_mutex.lock();
464 else464 defer comp.whole_cache_manifest_mutex.unlock();
465 std.fs.cwd().readFileAlloc(comp.gpa, file_path, max_file_size) catch return null;465 break :blk man.addFilePostFetch(file_path, max_file_size) catch return null;
466 } else std.fs.cwd().readFileAlloc(comp.gpa, file_path, max_file_size) catch return null;
466 result_len.* = contents.len;467 result_len.* = contents.len;
467 // TODO https://github.com/ziglang/zig/issues/3328#issuecomment-716749475468 // TODO https://github.com/ziglang/zig/issues/3328#issuecomment-716749475
468 if (contents.len == 0) return @intToPtr(?[*]const u8, 0x1);469 if (contents.len == 0) return @intToPtr(?[*]const u8, 0x1);