diff --git a/lib/std/cache_hash.zig b/lib/std/cache_hash.zig deleted file mode 100644 index 2075bedc22d72551bf72bc49a230cceebece2fda..0000000000000000000000000000000000000000 --- a/lib/std/cache_hash.zig +++ /dev/null @@ -1,845 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2015-2020 Zig Contributors -// This file is part of [zig](https://ziglang.org/), which is MIT licensed. -// The MIT license requires this copyright notice to be included in all copies -// and substantial portions of the software. -const std = @import("std.zig"); -const crypto = std.crypto; -const fs = std.fs; -const base64 = std.base64; -const assert = std.debug.assert; -const testing = std.testing; -const mem = std.mem; -const fmt = std.fmt; -const Allocator = std.mem.Allocator; - -pub const base64_encoder = fs.base64_encoder; -pub const base64_decoder = fs.base64_decoder; -/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 -/// We round up to 18 to avoid the `==` padding after base64 encoding. -pub const BIN_DIGEST_LEN = 18; -pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); - -const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; - -/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it -/// provides enough collision resistance for the CacheHash use cases, while being one of our -/// fastest options right now. -pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); - -/// Initial state, that can be copied. -pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length); - -pub const File = struct { - path: ?[]const u8, - max_file_size: ?usize, - stat: fs.File.Stat, - bin_digest: [BIN_DIGEST_LEN]u8, - contents: ?[]const u8, - - pub fn deinit(self: *File, allocator: *Allocator) void { - if (self.path) |owned_slice| { - allocator.free(owned_slice); - self.path = null; - } - if (self.contents) |contents| { - allocator.free(contents); - self.contents = null; - } - self.* = undefined; - } -}; - -pub const Cache = struct { - gpa: *Allocator, - manifest_dir: fs.Dir, - hash: HashHelper = .{}, - - /// Be sure to call `CacheHash.deinit` after successful initialization. - pub fn obtain(cache: *const Cache) CacheHash { - return CacheHash{ - .cache = cache, - .hash = cache.hash, - .manifest_file = null, - .manifest_dirty = false, - .b64_digest = undefined, - }; - } -}; - -pub const HashHelper = struct { - hasher: Hasher = hasher_init, - - /// Record a slice of bytes as an dependency of the process being cached - pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { - hh.hasher.update(mem.asBytes(&bytes.len)); - hh.hasher.update(bytes); - } - - pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { - hh.add(optional_bytes != null); - hh.addBytes(optional_bytes orelse return); - } - - pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { - hh.add(list_of_bytes.len); - for (list_of_bytes) |bytes| hh.addBytes(bytes); - } - - /// Convert the input value into bytes and record it as a dependency of the process being cached. - pub fn add(hh: *HashHelper, x: anytype) void { - switch (@TypeOf(x)) { - std.builtin.Version => { - hh.add(x.major); - hh.add(x.minor); - hh.add(x.patch); - return; - }, - else => {}, - } - - switch (@typeInfo(@TypeOf(x))) { - .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), - else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), - } - } - - pub fn addOptional(hh: *HashHelper, optional: anytype) void { - hh.add(optional != null); - hh.add(optional orelse return); - } - - /// Returns a base64 encoded hash of the inputs, without modifying state. - pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 { - var copy = hh; - return copy.final(); - } - - /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher. - pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 { - var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; - hh.hasher.final(&bin_digest); - - var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; - base64_encoder.encode(&out_digest, &bin_digest); - - return out_digest; - } -}; - -pub const Lock = struct { - manifest_file: fs.File, - - pub fn release(lock: *Lock) void { - lock.manifest_file.close(); - lock.* = undefined; - } -}; - -/// CacheHash manages project-local `zig-cache` directories. -/// This is not a general-purpose cache. -/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input. -pub const CacheHash = struct { - cache: *const Cache, - /// Current state for incremental hashing. - hash: HashHelper, - manifest_file: ?fs.File, - manifest_dirty: bool, - files: std.ArrayListUnmanaged(File) = .{}, - b64_digest: [BASE64_DIGEST_LEN]u8, - - /// Add a file as a dependency of process being cached. When `hit` is - /// called, the file's contents will be checked to ensure that it matches - /// the contents from previous times. - /// - /// Max file size will be used to determine the amount of space to the file contents - /// are allowed to take up in memory. If max_file_size is null, then the contents - /// will not be loaded into memory. - /// - /// Returns the index of the entry in the `files` array list. You can use it - /// to access the contents of the file after calling `hit()` like so: - /// - /// ``` - /// var file_contents = cache_hash.files.items[file_index].contents.?; - /// ``` - pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize { - assert(self.manifest_file == null); - - try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1); - const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); - - const idx = self.files.items.len; - self.files.addOneAssumeCapacity().* = .{ - .path = resolved_path, - .contents = null, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - }; - - self.hash.addBytes(resolved_path); - - return idx; - } - - pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void { - self.hash.add(optional_file_path != null); - const file_path = optional_file_path orelse return; - _ = try self.addFile(file_path, null); - } - - pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void { - self.hash.add(list_of_files.len); - for (list_of_files) |file_path| { - _ = try self.addFile(file_path, null); - } - } - - /// Check the cache to see if the input exists in it. If it exists, returns `true`. - /// A base64 encoding of its hash is available by calling `final`. - /// - /// This function will also acquire an exclusive lock to the manifest file. This means - /// that a process holding a CacheHash will block any other process attempting to - /// acquire the lock. - /// - /// The lock on the manifest file is released when `deinit` is called. As another - /// option, one may call `toOwnedLock` to obtain a smaller object which can represent - /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. - pub fn hit(self: *CacheHash) !bool { - assert(self.manifest_file == null); - - const ext = ".txt"; - var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined; - - var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; - self.hash.hasher.final(&bin_digest); - - base64_encoder.encode(self.b64_digest[0..], &bin_digest); - - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - - mem.copy(u8, &manifest_file_path, &self.b64_digest); - manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*; - - if (self.files.items.len != 0) { - self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ - .read = true, - .truncate = false, - .lock = .Exclusive, - }); - } else { - // If there are no file inputs, we check if the manifest file exists instead of - // comparing the hashes on the files used for the cached item - self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{ - .read = true, - .write = true, - .lock = .Exclusive, - }) catch |err| switch (err) { - error.FileNotFound => { - self.manifest_dirty = true; - self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ - .read = true, - .truncate = false, - .lock = .Exclusive, - }); - return false; - }, - else => |e| return e, - }; - } - - const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX); - defer self.cache.gpa.free(file_contents); - - const input_file_count = self.files.items.len; - var any_file_changed = false; - var line_iter = mem.tokenize(file_contents, "\n"); - var idx: usize = 0; - while (line_iter.next()) |line| { - defer idx += 1; - - const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: { - const new = try self.files.addOne(self.cache.gpa); - new.* = .{ - .path = null, - .contents = null, - .max_file_size = null, - .stat = undefined, - .bin_digest = undefined, - }; - break :blk new; - }; - - var iter = mem.tokenize(line, " "); - const size = iter.next() orelse return error.InvalidFormat; - const inode = iter.next() orelse return error.InvalidFormat; - const mtime_nsec_str = iter.next() orelse return error.InvalidFormat; - const digest_str = iter.next() orelse return error.InvalidFormat; - const file_path = iter.rest(); - - cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; - cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; - cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; - base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; - - if (file_path.len == 0) { - return error.InvalidFormat; - } - if (cache_hash_file.path) |p| { - if (!mem.eql(u8, file_path, p)) { - return error.InvalidFormat; - } - } - - if (cache_hash_file.path == null) { - cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path); - } - - const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch { - return error.CacheUnavailable; - }; - defer this_file.close(); - - const actual_stat = try this_file.stat(); - const size_match = actual_stat.size == cache_hash_file.stat.size; - const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime; - const inode_match = actual_stat.inode == cache_hash_file.stat.inode; - - if (!size_match or !mtime_match or !inode_match) { - self.manifest_dirty = true; - - cache_hash_file.stat = actual_stat; - - if (isProblematicTimestamp(cache_hash_file.stat.mtime)) { - cache_hash_file.stat.mtime = 0; - cache_hash_file.stat.inode = 0; - } - - var actual_digest: [BIN_DIGEST_LEN]u8 = undefined; - try hashFile(this_file, &actual_digest); - - if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { - cache_hash_file.bin_digest = actual_digest; - // keep going until we have the input file digests - any_file_changed = true; - } - } - - if (!any_file_changed) { - self.hash.hasher.update(&cache_hash_file.bin_digest); - } - } - - if (any_file_changed) { - // cache miss - // keep the manifest file open - // reset the hash - self.hash.hasher = hasher_init; - self.hash.hasher.update(&bin_digest); - - // Remove files not in the initial hash - for (self.files.items[input_file_count..]) |*file| { - file.deinit(self.cache.gpa); - } - self.files.shrinkRetainingCapacity(input_file_count); - - for (self.files.items) |file| { - self.hash.hasher.update(&file.bin_digest); - } - return false; - } - - if (idx < input_file_count) { - self.manifest_dirty = true; - while (idx < input_file_count) : (idx += 1) { - const ch_file = &self.files.items[idx]; - try self.populateFileHash(ch_file); - } - return false; - } - - return true; - } - - fn populateFileHash(self: *CacheHash, ch_file: *File) !void { - const file = try fs.cwd().openFile(ch_file.path.?, .{}); - defer file.close(); - - ch_file.stat = try file.stat(); - - if (isProblematicTimestamp(ch_file.stat.mtime)) { - ch_file.stat.mtime = 0; - ch_file.stat.inode = 0; - } - - if (ch_file.max_file_size) |max_file_size| { - if (ch_file.stat.size > max_file_size) { - return error.FileTooBig; - } - - const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size)); - errdefer self.cache.gpa.free(contents); - - // Hash while reading from disk, to keep the contents in the cpu cache while - // doing hashing. - var hasher = hasher_init; - var off: usize = 0; - while (true) { - // give me everything you've got, captain - const bytes_read = try file.read(contents[off..]); - if (bytes_read == 0) break; - hasher.update(contents[off..][0..bytes_read]); - off += bytes_read; - } - hasher.final(&ch_file.bin_digest); - - ch_file.contents = contents; - } else { - try hashFile(file, &ch_file.bin_digest); - } - - self.hash.hasher.update(&ch_file.bin_digest); - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. This is useful for processes that don't know the all the files that - /// are depended on ahead of time. For example, a source file that can import other files - /// will need to be recompiled if the imported file is changed. - pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 { - assert(self.manifest_file != null); - - const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); - errdefer self.cache.gpa.free(resolved_path); - - const new_ch_file = try self.files.addOne(self.cache.gpa); - new_ch_file.* = .{ - .path = resolved_path, - .max_file_size = max_file_size, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - }; - errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); - - try self.populateFileHash(new_ch_file); - - return new_ch_file.contents.?; - } - - /// Add a file as a dependency of process being cached, after the initial hash has been - /// calculated. This is useful for processes that don't know the all the files that - /// are depended on ahead of time. For example, a source file that can import other files - /// will need to be recompiled if the imported file is changed. - pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void { - assert(self.manifest_file != null); - - const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); - errdefer self.cache.gpa.free(resolved_path); - - const new_ch_file = try self.files.addOne(self.cache.gpa); - new_ch_file.* = .{ - .path = resolved_path, - .max_file_size = null, - .stat = undefined, - .bin_digest = undefined, - .contents = null, - }; - errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); - - try self.populateFileHash(new_ch_file); - } - - /// Returns a base64 encoded hash of the inputs. - pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 { - assert(self.manifest_file != null); - - // We don't close the manifest file yet, because we want to - // keep it locked until the API user is done using it. - // We also don't write out the manifest yet, because until - // cache_release is called we still might be working on creating - // the artifacts to cache. - - var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; - self.hash.hasher.final(&bin_digest); - - var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; - base64_encoder.encode(&out_digest, &bin_digest); - - return out_digest; - } - - pub fn writeManifest(self: *CacheHash) !void { - assert(self.manifest_file != null); - if (!self.manifest_dirty) return; - - var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined; - var contents = std.ArrayList(u8).init(self.cache.gpa); - var writer = contents.writer(); - defer contents.deinit(); - - for (self.files.items) |file| { - base64_encoder.encode(encoded_digest[0..], &file.bin_digest); - try writer.print("{} {} {} {} {}\n", .{ - file.stat.size, - file.stat.inode, - file.stat.mtime, - encoded_digest[0..], - file.path, - }); - } - - try self.manifest_file.?.pwriteAll(contents.items, 0); - self.manifest_dirty = false; - } - - /// Obtain only the data needed to maintain a lock on the manifest file. - /// The `CacheHash` remains safe to deinit. - /// Don't forget to call `writeManifest` before this! - pub fn toOwnedLock(self: *CacheHash) Lock { - const manifest_file = self.manifest_file.?; - self.manifest_file = null; - return Lock{ .manifest_file = manifest_file }; - } - - /// Releases the manifest file and frees any memory the CacheHash was using. - /// `CacheHash.hit` must be called first. - /// Don't forget to call `writeManifest` before this! - pub fn deinit(self: *CacheHash) void { - if (self.manifest_file) |file| { - file.close(); - } - for (self.files.items) |*file| { - file.deinit(self.cache.gpa); - } - self.files.deinit(self.cache.gpa); - } -}; - -fn hashFile(file: fs.File, bin_digest: []u8) !void { - var buf: [1024]u8 = undefined; - - var hasher = hasher_init; - while (true) { - const bytes_read = try file.read(&buf); - if (bytes_read == 0) break; - hasher.update(buf[0..bytes_read]); - } - - hasher.final(bin_digest); -} - -/// If the wall clock time, rounded to the same precision as the -/// mtime, is equal to the mtime, then we cannot rely on this mtime -/// yet. We will instead save an mtime value that indicates the hash -/// must be unconditionally computed. -/// This function recognizes the precision of mtime by looking at trailing -/// zero bits of the seconds and nanoseconds. -fn isProblematicTimestamp(fs_clock: i128) bool { - const wall_clock = std.time.nanoTimestamp(); - - // We have to break the nanoseconds into seconds and remainder nanoseconds - // to detect precision of seconds, because looking at the zero bits in base - // 2 would not detect precision of the seconds value. - const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s)); - const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s)); - var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s)); - var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s)); - - // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock. - if (fs_nsec == 0) { - wall_nsec = 0; - if (fs_sec == 0) { - wall_sec = 0; - } else { - wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec)); - } - } else { - wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec)); - } - return wall_nsec == fs_nsec and wall_sec == fs_sec; -} - -test "cache file and then recall it" { - if (std.Target.current.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - - const temp_file = "test.txt"; - const temp_manifest_dir = "temp_manifest_dir"; - - const ts = std.time.nanoTimestamp(); - try cwd.writeFile(temp_file, "Hello, world!\n"); - - while (isProblematicTimestamp(ts)) { - std.time.sleep(1); - } - - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); - - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); - - digest1 = ch.final(); - try ch.writeManifest(); - } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.add(true); - ch.hash.add(@as(u16, 1234)); - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file, null); - - // Cache hit! We just "built" the same file - testing.expect(try ch.hit()); - digest2 = ch.final(); - - try ch.writeManifest(); - } - - testing.expectEqual(digest1, digest2); - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteFile(temp_file); -} - -test "give problematic timestamp" { - var fs_clock = std.time.nanoTimestamp(); - // to make it problematic, we make it only accurate to the second - fs_clock = @divTrunc(fs_clock, std.time.ns_per_s); - fs_clock *= std.time.ns_per_s; - testing.expect(isProblematicTimestamp(fs_clock)); -} - -test "give nonproblematic timestamp" { - testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s)); -} - -test "check that changing a file makes cache fail" { - if (std.Target.current.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - - const temp_file = "cache_hash_change_file_test.txt"; - const temp_manifest_dir = "cache_hash_change_file_manifest_dir"; - const original_temp_file_contents = "Hello, world!\n"; - const updated_temp_file_contents = "Hello, world; but updated!\n"; - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteTree(temp_file); - - const ts = std.time.nanoTimestamp(); - try cwd.writeFile(temp_file, original_temp_file_contents); - - while (isProblematicTimestamp(ts)) { - std.time.sleep(1); - } - - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); - - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); - - testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); - - digest1 = ch.final(); - - try ch.writeManifest(); - } - - try cwd.writeFile(temp_file, updated_temp_file_contents); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - const temp_file_idx = try ch.addFile(temp_file, 100); - - // A file that we depend on has been updated, so the cache should not contain an entry for it - testing.expectEqual(false, try ch.hit()); - - // The cache system does not keep the contents of re-hashed input files. - testing.expect(ch.files.items[temp_file_idx].contents == null); - - digest2 = ch.final(); - - try ch.writeManifest(); - } - - testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteTree(temp_file); -} - -test "no file inputs" { - if (std.Target.current.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - const temp_manifest_dir = "no_file_inputs_manifest_dir"; - defer cwd.deleteTree(temp_manifest_dir) catch unreachable; - - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); - - digest1 = ch.final(); - - try ch.writeManifest(); - } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - - testing.expect(try ch.hit()); - digest2 = ch.final(); - try ch.writeManifest(); - } - - testing.expectEqual(digest1, digest2); -} - -test "CacheHashes with files added after initial hash work" { - if (std.Target.current.os.tag == .wasi) { - // https://github.com/ziglang/zig/issues/5437 - return error.SkipZigTest; - } - const cwd = fs.cwd(); - - const temp_file1 = "cache_hash_post_file_test1.txt"; - const temp_file2 = "cache_hash_post_file_test2.txt"; - const temp_manifest_dir = "cache_hash_post_file_manifest_dir"; - - const ts1 = std.time.nanoTimestamp(); - try cwd.writeFile(temp_file1, "Hello, world!\n"); - try cwd.writeFile(temp_file2, "Hello world the second!\n"); - - while (isProblematicTimestamp(ts1)) { - std.time.sleep(1); - } - - var digest1: [BASE64_DIGEST_LEN]u8 = undefined; - var digest2: [BASE64_DIGEST_LEN]u8 = undefined; - var digest3: [BASE64_DIGEST_LEN]u8 = undefined; - - var cache = Cache{ - .gpa = testing.allocator, - .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), - }; - defer cache.manifest_dir.close(); - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - // There should be nothing in the cache - testing.expectEqual(false, try ch.hit()); - - _ = try ch.addFilePost(temp_file2); - - digest1 = ch.final(); - try ch.writeManifest(); - } - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - testing.expect(try ch.hit()); - digest2 = ch.final(); - - try ch.writeManifest(); - } - testing.expect(mem.eql(u8, &digest1, &digest2)); - - // Modify the file added after initial hash - const ts2 = std.time.nanoTimestamp(); - try cwd.writeFile(temp_file2, "Hello world the second, updated\n"); - - while (isProblematicTimestamp(ts2)) { - std.time.sleep(1); - } - - { - var ch = cache.obtain(); - defer ch.deinit(); - - ch.hash.addBytes("1234"); - _ = try ch.addFile(temp_file1, null); - - // A file that we depend on has been updated, so the cache should not contain an entry for it - testing.expectEqual(false, try ch.hit()); - - _ = try ch.addFilePost(temp_file2); - - digest3 = ch.final(); - - try ch.writeManifest(); - } - - testing.expect(!mem.eql(u8, &digest1, &digest3)); - - try cwd.deleteTree(temp_manifest_dir); - try cwd.deleteFile(temp_file1); - try cwd.deleteFile(temp_file2); -} diff --git a/lib/std/std.zig b/lib/std/std.zig index 4236b292983b45e85992440b587a49dc8e471080..56b75e565690c3e2a78679cd2689cf7e4ec5bd23 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -48,7 +48,6 @@ pub const base64 = @import("base64.zig"); pub const build = @import("build.zig"); pub const builtin = @import("builtin.zig"); pub const c = @import("c.zig"); -pub const cache_hash = @import("cache_hash.zig"); pub const coff = @import("coff.zig"); pub const compress = @import("compress.zig"); pub const crypto = @import("crypto.zig"); diff --git a/src-self-hosted/Cache.zig b/src-self-hosted/Cache.zig new file mode 100644 index 0000000000000000000000000000000000000000..b41c0a312ff6d26a73f82bab42a05b7f4f7b265d --- /dev/null +++ b/src-self-hosted/Cache.zig @@ -0,0 +1,839 @@ +gpa: *Allocator, +manifest_dir: fs.Dir, +hash: HashHelper = .{}, + +const Cache = @This(); +const std = @import("std"); +const crypto = std.crypto; +const fs = std.fs; +const base64 = std.base64; +const assert = std.debug.assert; +const testing = std.testing; +const mem = std.mem; +const fmt = std.fmt; +const Allocator = std.mem.Allocator; + +/// Be sure to call `CacheHash.deinit` after successful initialization. +pub fn obtain(cache: *const Cache) CacheHash { + return CacheHash{ + .cache = cache, + .hash = cache.hash, + .manifest_file = null, + .manifest_dirty = false, + .b64_digest = undefined, + }; +} + +pub const base64_encoder = fs.base64_encoder; +pub const base64_decoder = fs.base64_decoder; +/// 16 would be 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6 +/// We round up to 18 to avoid the `==` padding after base64 encoding. +pub const BIN_DIGEST_LEN = 18; +pub const BASE64_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN); + +const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024; + +/// The type used for hashing file contents. Currently, this is SipHash128(1, 3), because it +/// provides enough collision resistance for the CacheHash use cases, while being one of our +/// fastest options right now. +pub const Hasher = crypto.auth.siphash.SipHash128(1, 3); + +/// Initial state, that can be copied. +pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.minimum_key_length); + +pub const File = struct { + path: ?[]const u8, + max_file_size: ?usize, + stat: fs.File.Stat, + bin_digest: [BIN_DIGEST_LEN]u8, + contents: ?[]const u8, + + pub fn deinit(self: *File, allocator: *Allocator) void { + if (self.path) |owned_slice| { + allocator.free(owned_slice); + self.path = null; + } + if (self.contents) |contents| { + allocator.free(contents); + self.contents = null; + } + self.* = undefined; + } +}; + +pub const HashHelper = struct { + hasher: Hasher = hasher_init, + + /// Record a slice of bytes as an dependency of the process being cached + pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { + hh.hasher.update(mem.asBytes(&bytes.len)); + hh.hasher.update(bytes); + } + + pub fn addOptionalBytes(hh: *HashHelper, optional_bytes: ?[]const u8) void { + hh.add(optional_bytes != null); + hh.addBytes(optional_bytes orelse return); + } + + pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { + hh.add(list_of_bytes.len); + for (list_of_bytes) |bytes| hh.addBytes(bytes); + } + + /// Convert the input value into bytes and record it as a dependency of the process being cached. + pub fn add(hh: *HashHelper, x: anytype) void { + switch (@TypeOf(x)) { + std.builtin.Version => { + hh.add(x.major); + hh.add(x.minor); + hh.add(x.patch); + return; + }, + else => {}, + } + + switch (@typeInfo(@TypeOf(x))) { + .Bool, .Int, .Enum, .Array => hh.addBytes(mem.asBytes(&x)), + else => @compileError("unable to hash type " ++ @typeName(@TypeOf(x))), + } + } + + pub fn addOptional(hh: *HashHelper, optional: anytype) void { + hh.add(optional != null); + hh.add(optional orelse return); + } + + /// Returns a base64 encoded hash of the inputs, without modifying state. + pub fn peek(hh: HashHelper) [BASE64_DIGEST_LEN]u8 { + var copy = hh; + return copy.final(); + } + + /// Returns a base64 encoded hash of the inputs, mutating the state of the hasher. + pub fn final(hh: *HashHelper) [BASE64_DIGEST_LEN]u8 { + var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + hh.hasher.final(&bin_digest); + + var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; + base64_encoder.encode(&out_digest, &bin_digest); + + return out_digest; + } +}; + +pub const Lock = struct { + manifest_file: fs.File, + + pub fn release(lock: *Lock) void { + lock.manifest_file.close(); + lock.* = undefined; + } +}; + +/// CacheHash manages project-local `zig-cache` directories. +/// This is not a general-purpose cache. +/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input. +pub const CacheHash = struct { + cache: *const Cache, + /// Current state for incremental hashing. + hash: HashHelper, + manifest_file: ?fs.File, + manifest_dirty: bool, + files: std.ArrayListUnmanaged(File) = .{}, + b64_digest: [BASE64_DIGEST_LEN]u8, + + /// Add a file as a dependency of process being cached. When `hit` is + /// called, the file's contents will be checked to ensure that it matches + /// the contents from previous times. + /// + /// Max file size will be used to determine the amount of space to the file contents + /// are allowed to take up in memory. If max_file_size is null, then the contents + /// will not be loaded into memory. + /// + /// Returns the index of the entry in the `files` array list. You can use it + /// to access the contents of the file after calling `hit()` like so: + /// + /// ``` + /// var file_contents = cache_hash.files.items[file_index].contents.?; + /// ``` + pub fn addFile(self: *CacheHash, file_path: []const u8, max_file_size: ?usize) !usize { + assert(self.manifest_file == null); + + try self.files.ensureCapacity(self.cache.gpa, self.files.items.len + 1); + const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); + + const idx = self.files.items.len; + self.files.addOneAssumeCapacity().* = .{ + .path = resolved_path, + .contents = null, + .max_file_size = max_file_size, + .stat = undefined, + .bin_digest = undefined, + }; + + self.hash.addBytes(resolved_path); + + return idx; + } + + pub fn addOptionalFile(self: *CacheHash, optional_file_path: ?[]const u8) !void { + self.hash.add(optional_file_path != null); + const file_path = optional_file_path orelse return; + _ = try self.addFile(file_path, null); + } + + pub fn addListOfFiles(self: *CacheHash, list_of_files: []const []const u8) !void { + self.hash.add(list_of_files.len); + for (list_of_files) |file_path| { + _ = try self.addFile(file_path, null); + } + } + + /// Check the cache to see if the input exists in it. If it exists, returns `true`. + /// A base64 encoding of its hash is available by calling `final`. + /// + /// This function will also acquire an exclusive lock to the manifest file. This means + /// that a process holding a CacheHash will block any other process attempting to + /// acquire the lock. + /// + /// The lock on the manifest file is released when `deinit` is called. As another + /// option, one may call `toOwnedLock` to obtain a smaller object which can represent + /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called. + pub fn hit(self: *CacheHash) !bool { + assert(self.manifest_file == null); + + const ext = ".txt"; + var manifest_file_path: [self.b64_digest.len + ext.len]u8 = undefined; + + var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + self.hash.hasher.final(&bin_digest); + + base64_encoder.encode(self.b64_digest[0..], &bin_digest); + + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); + + mem.copy(u8, &manifest_file_path, &self.b64_digest); + manifest_file_path[self.b64_digest.len..][0..ext.len].* = ext.*; + + if (self.files.items.len != 0) { + self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ + .read = true, + .truncate = false, + .lock = .Exclusive, + }); + } else { + // If there are no file inputs, we check if the manifest file exists instead of + // comparing the hashes on the files used for the cached item + self.manifest_file = self.cache.manifest_dir.openFile(&manifest_file_path, .{ + .read = true, + .write = true, + .lock = .Exclusive, + }) catch |err| switch (err) { + error.FileNotFound => { + self.manifest_dirty = true; + self.manifest_file = try self.cache.manifest_dir.createFile(&manifest_file_path, .{ + .read = true, + .truncate = false, + .lock = .Exclusive, + }); + return false; + }, + else => |e| return e, + }; + } + + const file_contents = try self.manifest_file.?.inStream().readAllAlloc(self.cache.gpa, MANIFEST_FILE_SIZE_MAX); + defer self.cache.gpa.free(file_contents); + + const input_file_count = self.files.items.len; + var any_file_changed = false; + var line_iter = mem.tokenize(file_contents, "\n"); + var idx: usize = 0; + while (line_iter.next()) |line| { + defer idx += 1; + + const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: { + const new = try self.files.addOne(self.cache.gpa); + new.* = .{ + .path = null, + .contents = null, + .max_file_size = null, + .stat = undefined, + .bin_digest = undefined, + }; + break :blk new; + }; + + var iter = mem.tokenize(line, " "); + const size = iter.next() orelse return error.InvalidFormat; + const inode = iter.next() orelse return error.InvalidFormat; + const mtime_nsec_str = iter.next() orelse return error.InvalidFormat; + const digest_str = iter.next() orelse return error.InvalidFormat; + const file_path = iter.rest(); + + cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat; + cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat; + cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat; + base64_decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat; + + if (file_path.len == 0) { + return error.InvalidFormat; + } + if (cache_hash_file.path) |p| { + if (!mem.eql(u8, file_path, p)) { + return error.InvalidFormat; + } + } + + if (cache_hash_file.path == null) { + cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path); + } + + const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .read = true }) catch { + return error.CacheUnavailable; + }; + defer this_file.close(); + + const actual_stat = try this_file.stat(); + const size_match = actual_stat.size == cache_hash_file.stat.size; + const mtime_match = actual_stat.mtime == cache_hash_file.stat.mtime; + const inode_match = actual_stat.inode == cache_hash_file.stat.inode; + + if (!size_match or !mtime_match or !inode_match) { + self.manifest_dirty = true; + + cache_hash_file.stat = actual_stat; + + if (isProblematicTimestamp(cache_hash_file.stat.mtime)) { + cache_hash_file.stat.mtime = 0; + cache_hash_file.stat.inode = 0; + } + + var actual_digest: [BIN_DIGEST_LEN]u8 = undefined; + try hashFile(this_file, &actual_digest); + + if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) { + cache_hash_file.bin_digest = actual_digest; + // keep going until we have the input file digests + any_file_changed = true; + } + } + + if (!any_file_changed) { + self.hash.hasher.update(&cache_hash_file.bin_digest); + } + } + + if (any_file_changed) { + // cache miss + // keep the manifest file open + // reset the hash + self.hash.hasher = hasher_init; + self.hash.hasher.update(&bin_digest); + + // Remove files not in the initial hash + for (self.files.items[input_file_count..]) |*file| { + file.deinit(self.cache.gpa); + } + self.files.shrinkRetainingCapacity(input_file_count); + + for (self.files.items) |file| { + self.hash.hasher.update(&file.bin_digest); + } + return false; + } + + if (idx < input_file_count) { + self.manifest_dirty = true; + while (idx < input_file_count) : (idx += 1) { + const ch_file = &self.files.items[idx]; + try self.populateFileHash(ch_file); + } + return false; + } + + return true; + } + + fn populateFileHash(self: *CacheHash, ch_file: *File) !void { + const file = try fs.cwd().openFile(ch_file.path.?, .{}); + defer file.close(); + + ch_file.stat = try file.stat(); + + if (isProblematicTimestamp(ch_file.stat.mtime)) { + ch_file.stat.mtime = 0; + ch_file.stat.inode = 0; + } + + if (ch_file.max_file_size) |max_file_size| { + if (ch_file.stat.size > max_file_size) { + return error.FileTooBig; + } + + const contents = try self.cache.gpa.alloc(u8, @intCast(usize, ch_file.stat.size)); + errdefer self.cache.gpa.free(contents); + + // Hash while reading from disk, to keep the contents in the cpu cache while + // doing hashing. + var hasher = hasher_init; + var off: usize = 0; + while (true) { + // give me everything you've got, captain + const bytes_read = try file.read(contents[off..]); + if (bytes_read == 0) break; + hasher.update(contents[off..][0..bytes_read]); + off += bytes_read; + } + hasher.final(&ch_file.bin_digest); + + ch_file.contents = contents; + } else { + try hashFile(file, &ch_file.bin_digest); + } + + self.hash.hasher.update(&ch_file.bin_digest); + } + + /// Add a file as a dependency of process being cached, after the initial hash has been + /// calculated. This is useful for processes that don't know the all the files that + /// are depended on ahead of time. For example, a source file that can import other files + /// will need to be recompiled if the imported file is changed. + pub fn addFilePostFetch(self: *CacheHash, file_path: []const u8, max_file_size: usize) ![]u8 { + assert(self.manifest_file != null); + + const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); + errdefer self.cache.gpa.free(resolved_path); + + const new_ch_file = try self.files.addOne(self.cache.gpa); + new_ch_file.* = .{ + .path = resolved_path, + .max_file_size = max_file_size, + .stat = undefined, + .bin_digest = undefined, + .contents = null, + }; + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); + + try self.populateFileHash(new_ch_file); + + return new_ch_file.contents.?; + } + + /// Add a file as a dependency of process being cached, after the initial hash has been + /// calculated. This is useful for processes that don't know the all the files that + /// are depended on ahead of time. For example, a source file that can import other files + /// will need to be recompiled if the imported file is changed. + pub fn addFilePost(self: *CacheHash, file_path: []const u8) !void { + assert(self.manifest_file != null); + + const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path}); + errdefer self.cache.gpa.free(resolved_path); + + const new_ch_file = try self.files.addOne(self.cache.gpa); + new_ch_file.* = .{ + .path = resolved_path, + .max_file_size = null, + .stat = undefined, + .bin_digest = undefined, + .contents = null, + }; + errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); + + try self.populateFileHash(new_ch_file); + } + + /// Returns a base64 encoded hash of the inputs. + pub fn final(self: *CacheHash) [BASE64_DIGEST_LEN]u8 { + assert(self.manifest_file != null); + + // We don't close the manifest file yet, because we want to + // keep it locked until the API user is done using it. + // We also don't write out the manifest yet, because until + // cache_release is called we still might be working on creating + // the artifacts to cache. + + var bin_digest: [BIN_DIGEST_LEN]u8 = undefined; + self.hash.hasher.final(&bin_digest); + + var out_digest: [BASE64_DIGEST_LEN]u8 = undefined; + base64_encoder.encode(&out_digest, &bin_digest); + + return out_digest; + } + + pub fn writeManifest(self: *CacheHash) !void { + assert(self.manifest_file != null); + if (!self.manifest_dirty) return; + + var encoded_digest: [BASE64_DIGEST_LEN]u8 = undefined; + var contents = std.ArrayList(u8).init(self.cache.gpa); + var writer = contents.writer(); + defer contents.deinit(); + + for (self.files.items) |file| { + base64_encoder.encode(encoded_digest[0..], &file.bin_digest); + try writer.print("{} {} {} {} {}\n", .{ + file.stat.size, + file.stat.inode, + file.stat.mtime, + encoded_digest[0..], + file.path, + }); + } + + try self.manifest_file.?.pwriteAll(contents.items, 0); + self.manifest_dirty = false; + } + + /// Obtain only the data needed to maintain a lock on the manifest file. + /// The `CacheHash` remains safe to deinit. + /// Don't forget to call `writeManifest` before this! + pub fn toOwnedLock(self: *CacheHash) Lock { + const manifest_file = self.manifest_file.?; + self.manifest_file = null; + return Lock{ .manifest_file = manifest_file }; + } + + /// Releases the manifest file and frees any memory the CacheHash was using. + /// `CacheHash.hit` must be called first. + /// Don't forget to call `writeManifest` before this! + pub fn deinit(self: *CacheHash) void { + if (self.manifest_file) |file| { + file.close(); + } + for (self.files.items) |*file| { + file.deinit(self.cache.gpa); + } + self.files.deinit(self.cache.gpa); + } +}; + +fn hashFile(file: fs.File, bin_digest: []u8) !void { + var buf: [1024]u8 = undefined; + + var hasher = hasher_init; + while (true) { + const bytes_read = try file.read(&buf); + if (bytes_read == 0) break; + hasher.update(buf[0..bytes_read]); + } + + hasher.final(bin_digest); +} + +/// If the wall clock time, rounded to the same precision as the +/// mtime, is equal to the mtime, then we cannot rely on this mtime +/// yet. We will instead save an mtime value that indicates the hash +/// must be unconditionally computed. +/// This function recognizes the precision of mtime by looking at trailing +/// zero bits of the seconds and nanoseconds. +fn isProblematicTimestamp(fs_clock: i128) bool { + const wall_clock = std.time.nanoTimestamp(); + + // We have to break the nanoseconds into seconds and remainder nanoseconds + // to detect precision of seconds, because looking at the zero bits in base + // 2 would not detect precision of the seconds value. + const fs_sec = @intCast(i64, @divFloor(fs_clock, std.time.ns_per_s)); + const fs_nsec = @intCast(i64, @mod(fs_clock, std.time.ns_per_s)); + var wall_sec = @intCast(i64, @divFloor(wall_clock, std.time.ns_per_s)); + var wall_nsec = @intCast(i64, @mod(wall_clock, std.time.ns_per_s)); + + // First make all the least significant zero bits in the fs_clock, also zero bits in the wall clock. + if (fs_nsec == 0) { + wall_nsec = 0; + if (fs_sec == 0) { + wall_sec = 0; + } else { + wall_sec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_sec)); + } + } else { + wall_nsec &= @as(i64, -1) << @intCast(u6, @ctz(i64, fs_nsec)); + } + return wall_nsec == fs_nsec and wall_sec == fs_sec; +} + +test "cache file and then recall it" { + if (std.Target.current.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + + const temp_file = "test.txt"; + const temp_manifest_dir = "temp_manifest_dir"; + + const ts = std.time.nanoTimestamp(); + try cwd.writeFile(temp_file, "Hello, world!\n"); + + while (isProblematicTimestamp(ts)) { + std.time.sleep(1); + } + + var digest1: [BASE64_DIGEST_LEN]u8 = undefined; + var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file, null); + + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); + + digest1 = ch.final(); + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.add(true); + ch.hash.add(@as(u16, 1234)); + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file, null); + + // Cache hit! We just "built" the same file + testing.expect(try ch.hit()); + digest2 = ch.final(); + + try ch.writeManifest(); + } + + testing.expectEqual(digest1, digest2); + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteFile(temp_file); +} + +test "give problematic timestamp" { + var fs_clock = std.time.nanoTimestamp(); + // to make it problematic, we make it only accurate to the second + fs_clock = @divTrunc(fs_clock, std.time.ns_per_s); + fs_clock *= std.time.ns_per_s; + testing.expect(isProblematicTimestamp(fs_clock)); +} + +test "give nonproblematic timestamp" { + testing.expect(!isProblematicTimestamp(std.time.nanoTimestamp() - std.time.ns_per_s)); +} + +test "check that changing a file makes cache fail" { + if (std.Target.current.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + + const temp_file = "cache_hash_change_file_test.txt"; + const temp_manifest_dir = "cache_hash_change_file_manifest_dir"; + const original_temp_file_contents = "Hello, world!\n"; + const updated_temp_file_contents = "Hello, world; but updated!\n"; + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteTree(temp_file); + + const ts = std.time.nanoTimestamp(); + try cwd.writeFile(temp_file, original_temp_file_contents); + + while (isProblematicTimestamp(ts)) { + std.time.sleep(1); + } + + var digest1: [BASE64_DIGEST_LEN]u8 = undefined; + var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + const temp_file_idx = try ch.addFile(temp_file, 100); + + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); + + testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?)); + + digest1 = ch.final(); + + try ch.writeManifest(); + } + + try cwd.writeFile(temp_file, updated_temp_file_contents); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + const temp_file_idx = try ch.addFile(temp_file, 100); + + // A file that we depend on has been updated, so the cache should not contain an entry for it + testing.expectEqual(false, try ch.hit()); + + // The cache system does not keep the contents of re-hashed input files. + testing.expect(ch.files.items[temp_file_idx].contents == null); + + digest2 = ch.final(); + + try ch.writeManifest(); + } + + testing.expect(!mem.eql(u8, digest1[0..], digest2[0..])); + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteTree(temp_file); +} + +test "no file inputs" { + if (std.Target.current.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + const temp_manifest_dir = "no_file_inputs_manifest_dir"; + defer cwd.deleteTree(temp_manifest_dir) catch unreachable; + + var digest1: [BASE64_DIGEST_LEN]u8 = undefined; + var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); + + digest1 = ch.final(); + + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + + testing.expect(try ch.hit()); + digest2 = ch.final(); + try ch.writeManifest(); + } + + testing.expectEqual(digest1, digest2); +} + +test "CacheHashes with files added after initial hash work" { + if (std.Target.current.os.tag == .wasi) { + // https://github.com/ziglang/zig/issues/5437 + return error.SkipZigTest; + } + const cwd = fs.cwd(); + + const temp_file1 = "cache_hash_post_file_test1.txt"; + const temp_file2 = "cache_hash_post_file_test2.txt"; + const temp_manifest_dir = "cache_hash_post_file_manifest_dir"; + + const ts1 = std.time.nanoTimestamp(); + try cwd.writeFile(temp_file1, "Hello, world!\n"); + try cwd.writeFile(temp_file2, "Hello world the second!\n"); + + while (isProblematicTimestamp(ts1)) { + std.time.sleep(1); + } + + var digest1: [BASE64_DIGEST_LEN]u8 = undefined; + var digest2: [BASE64_DIGEST_LEN]u8 = undefined; + var digest3: [BASE64_DIGEST_LEN]u8 = undefined; + + var cache = Cache{ + .gpa = testing.allocator, + .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}), + }; + defer cache.manifest_dir.close(); + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + // There should be nothing in the cache + testing.expectEqual(false, try ch.hit()); + + _ = try ch.addFilePost(temp_file2); + + digest1 = ch.final(); + try ch.writeManifest(); + } + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + testing.expect(try ch.hit()); + digest2 = ch.final(); + + try ch.writeManifest(); + } + testing.expect(mem.eql(u8, &digest1, &digest2)); + + // Modify the file added after initial hash + const ts2 = std.time.nanoTimestamp(); + try cwd.writeFile(temp_file2, "Hello world the second, updated\n"); + + while (isProblematicTimestamp(ts2)) { + std.time.sleep(1); + } + + { + var ch = cache.obtain(); + defer ch.deinit(); + + ch.hash.addBytes("1234"); + _ = try ch.addFile(temp_file1, null); + + // A file that we depend on has been updated, so the cache should not contain an entry for it + testing.expectEqual(false, try ch.hit()); + + _ = try ch.addFilePost(temp_file2); + + digest3 = ch.final(); + + try ch.writeManifest(); + } + + testing.expect(!mem.eql(u8, &digest1, &digest3)); + + try cwd.deleteTree(temp_manifest_dir); + try cwd.deleteFile(temp_file1); + try cwd.deleteFile(temp_file2); +} diff --git a/src-self-hosted/Compilation.zig b/src-self-hosted/Compilation.zig index 1f42c8784903b185f117a082fc71e9a4c27ca1f2..606eefa4eedca7ca6518615b7652e7e700a69b15 100644 --- a/src-self-hosted/Compilation.zig +++ b/src-self-hosted/Compilation.zig @@ -17,6 +17,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const glibc = @import("glibc.zig"); const fatal = @import("main.zig").fatal; const Module = @import("Module.zig"); +const Cache = @import("Cache.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. gpa: *Allocator, @@ -46,7 +47,7 @@ disable_c_depfile: bool, c_source_files: []const CSourceFile, clang_argv: []const []const u8, -cache_parent: *std.cache_hash.Cache, +cache_parent: *Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, zig_lib_directory: Directory, @@ -78,7 +79,7 @@ owned_link_dir: ?std.fs.Dir, pub const InnerError = Module.InnerError; pub const CRTFile = struct { - lock: std.cache_hash.Lock, + lock: Cache.Lock, full_object_path: []const u8, fn deinit(self: *CRTFile, gpa: *Allocator) void { @@ -128,7 +129,7 @@ pub const CObject = struct { /// This is a file system lock on the cache hash manifest representing this /// object. It prevents other invocations of the Zig compiler from interfering /// with this object until released. - lock: std.cache_hash.Lock, + lock: Cache.Lock, }, /// There will be a corresponding ErrorMsg in Compilation.failed_c_objects. failure, @@ -404,7 +405,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { // find the same binary and incrementally update it even if there are modified source files. // We do this even if outputting to the current directory because we need somewhere to store // incremental compilation metadata. - const cache = try arena.create(std.cache_hash.Cache); + const cache = try arena.create(Cache); cache.* = .{ .gpa = gpa, .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}), diff --git a/src-self-hosted/introspect.zig b/src-self-hosted/introspect.zig index 8041142a6793eba2a00987f56f34f6af510bdc90..067326ebb68a6473185970546ab1870e9a551bd9 100644 --- a/src-self-hosted/introspect.zig +++ b/src-self-hosted/introspect.zig @@ -1,7 +1,6 @@ const std = @import("std"); const mem = std.mem; const fs = std.fs; -const CacheHash = std.cache_hash.CacheHash; const Compilation = @import("Compilation.zig"); /// Returns the sub_path that worked, or `null` if none did. diff --git a/src-self-hosted/link.zig b/src-self-hosted/link.zig index 832015faae99db3698e5c16ee2d540d5784a5883..a7393cc913706d5a6759653c35d53aa1625376ff 100644 --- a/src-self-hosted/link.zig +++ b/src-self-hosted/link.zig @@ -6,6 +6,7 @@ const fs = std.fs; const trace = @import("tracy.zig").trace; const Package = @import("Package.zig"); const Type = @import("type.zig").Type; +const Cache = @import("Cache.zig"); const build_options = @import("build_options"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; @@ -92,7 +93,7 @@ pub const File = struct { /// Prevents other processes from clobbering files in the output directory /// of this linking operation. - lock: ?std.cache_hash.Lock = null, + lock: ?Cache.Lock = null, pub const LinkBlock = union { elf: Elf.TextBlock, @@ -239,7 +240,7 @@ pub const File = struct { } } - pub fn toOwnedLock(self: *File) std.cache_hash.Lock { + pub fn toOwnedLock(self: *File) Cache.Lock { const lock = self.lock.?; self.lock = null; return lock;