authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-12-10 21:12:05-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-12-11 00:33:59-07:00
logfd4c98cbb7af337895f003d83d6cd2fb3447d6c2
tree463edc33a5093b5d255164fb04c278beedf9a963
parent21236c015140cbb1061e6161faa8011915d15de3

stage2: detect redundant C/C++ source files

Cache exposes BinDigest. Compilation gains a set of a BinDigest for every C/C++ source file. We detect when the same source/flags have already been added and emit a compile error. This prevents a deadlock in the caching system. Closes #7308

2 files changed, 22 insertions(+), 8 deletions(-)

src/Cache.zig+9-8
......@@ -26,6 +26,7 @@ pub fn obtain(cache: *const Cache) Manifest {
2626/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
2727pub const bin_digest_len = 16;
2828pub const hex_digest_len = bin_digest_len * 2;
29pub const BinDigest = [bin_digest_len]u8;
2930
3031const manifest_file_size_max = 50 * 1024 * 1024;
3132
......@@ -41,7 +42,7 @@ pub const File = struct {
4142 path: ?[]const u8,
4243 max_file_size: ?usize,
4344 stat: fs.File.Stat,
44 bin_digest: [bin_digest_len]u8,
45 bin_digest: BinDigest,
4546 contents: ?[]const u8,
4647
4748 pub fn deinit(self: *File, allocator: *Allocator) void {
......@@ -139,16 +140,16 @@ pub const HashHelper = struct {
139140 return copy.final();
140141 }
141142
142 pub fn peekBin(hh: HashHelper) [bin_digest_len]u8 {
143 pub fn peekBin(hh: HashHelper) BinDigest {
143144 var copy = hh;
144 var bin_digest: [bin_digest_len]u8 = undefined;
145 var bin_digest: BinDigest = undefined;
145146 copy.hasher.final(&bin_digest);
146147 return bin_digest;
147148 }
148149
149150 /// Returns a hex encoded hash of the inputs, mutating the state of the hasher.
150151 pub fn final(hh: *HashHelper) [hex_digest_len]u8 {
151 var bin_digest: [bin_digest_len]u8 = undefined;
152 var bin_digest: BinDigest = undefined;
152153 hh.hasher.final(&bin_digest);
153154
154155 var out_digest: [hex_digest_len]u8 = undefined;
......@@ -241,7 +242,7 @@ pub const Manifest = struct {
241242 const ext = ".txt";
242243 var manifest_file_path: [self.hex_digest.len + ext.len]u8 = undefined;
243244
244 var bin_digest: [bin_digest_len]u8 = undefined;
245 var bin_digest: BinDigest = undefined;
245246 self.hash.hasher.final(&bin_digest);
246247
247248 _ = std.fmt.bufPrint(&self.hex_digest, "{x}", .{bin_digest}) catch unreachable;
......@@ -347,7 +348,7 @@ pub const Manifest = struct {
347348 cache_hash_file.stat.inode = 0;
348349 }
349350
350 var actual_digest: [bin_digest_len]u8 = undefined;
351 var actual_digest: BinDigest = undefined;
351352 try hashFile(this_file, &actual_digest);
352353
353354 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
......@@ -381,7 +382,7 @@ pub const Manifest = struct {
381382 return true;
382383 }
383384
384 pub fn unhit(self: *Manifest, bin_digest: [bin_digest_len]u8, input_file_count: usize) void {
385 pub fn unhit(self: *Manifest, bin_digest: BinDigest, input_file_count: usize) void {
385386 // Reset the hash.
386387 self.hash.hasher = hasher_init;
387388 self.hash.hasher.update(&bin_digest);
......@@ -530,7 +531,7 @@ pub const Manifest = struct {
530531 // cache_release is called we still might be working on creating
531532 // the artifacts to cache.
532533
533 var bin_digest: [bin_digest_len]u8 = undefined;
534 var bin_digest: BinDigest = undefined;
534535 self.hash.hasher.final(&bin_digest);
535536
536537 var out_digest: [hex_digest_len]u8 = undefined;
src/Compilation.zig+13
......@@ -33,6 +33,7 @@ gpa: *Allocator,
3333arena_state: std.heap.ArenaAllocator.State,
3434bin_file: *link.File,
3535c_object_table: std.AutoArrayHashMapUnmanaged(*CObject, void) = .{},
36c_object_cache_digest_set: std.AutoHashMapUnmanaged(Cache.BinDigest, void) = .{},
3637stage1_lock: ?Cache.Lock = null,
3738stage1_cache_manifest: *Cache.Manifest = undefined,
3839
......@@ -1110,6 +1111,7 @@ pub fn destroy(self: *Compilation) void {
11101111 entry.key.destroy(gpa);
11111112 }
11121113 self.c_object_table.deinit(gpa);
1114 self.c_object_cache_digest_set.deinit(gpa);
11131115
11141116 for (self.failed_c_objects.items()) |entry| {
11151117 entry.value.destroy(gpa);
......@@ -1682,6 +1684,17 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: *
16821684 }
16831685 }
16841686
1687 {
1688 const gop = try comp.c_object_cache_digest_set.getOrPut(comp.gpa, man.hash.peekBin());
1689 if (gop.found_existing) {
1690 return comp.failCObj(
1691 c_object,
1692 "the same source file was already added to the same compilation with the same flags",
1693 .{},
1694 );
1695 }
1696 }
1697
16851698 var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa);
16861699 defer arena_allocator.deinit();
16871700 const arena = &arena_allocator.allocator;