authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-19 13:48:32-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-22 20:57:56-07:00
log21bd13626d66c36c327bb317bd09cad979d92327
tree0339aef23b4655448e6a71cdfca1840f66c69092
parent32ce2f91a92c23d46c6836a6dd68ae0f08bb04c5

Cache: introduce prefixes to manifests

Before, cache manifest files would have absolute file paths. This is problematic for two reasons: * Absolute file paths are not portable. Some operating systems such as WASI have trouble with them. The files themselves are less portable; they cannot be migrated from one user's home directory to another's. And finally they can break due to file paths exceeding maximum path component size. * They would prevent some advanced use cases of Zig, where the lib dir has a different path in a different invocation but is ultimately the same Zig version and lib directory as before. This commit adds a new column that specifies the prefix directory for each file. 0 is an escape hatch and has the previous behavior. The other two prefixes introduced are zig lib directory, and the cache directory. This means files in zig-cache manifests can reference files local to these directories. In practice, this means it is possible to use a different file path for the zig lib directory in a subsequent run of zig and have it still take advantage of the global cache, provided that the files inside remain unchanged. closes #13050

4 files changed, 157 insertions(+), 48 deletions(-)

src/Cache.zig+136-39
......@@ -1,3 +1,7 @@
1//! Manages `zig-cache` directories.
2//! This is not a general-purpose cache. It is designed to be fast and simple,
3//! not to withstand attacks using specially-crafted input.
4
15gpa: Allocator,
26manifest_dir: fs.Dir,
37hash: HashHelper = .{},
......@@ -5,6 +9,14 @@ hash: HashHelper = .{},
59recent_problematic_timestamp: i128 = 0,
610mutex: std.Thread.Mutex = .{},
711
12/// A set of strings such as the zig library directory or project source root, which
13/// are stripped from the file paths before putting into the cache. They
14/// are replaced with single-character indicators. This is not to save
15/// space but to eliminate absolute file paths. This improves portability
16/// and usefulness of the cache for advanced use cases.
17prefixes_buffer: [3]Compilation.Directory = undefined,
18prefixes_len: usize = 0,
19
820const Cache = @This();
921const std = @import("std");
1022const builtin = @import("builtin");
......@@ -18,6 +30,11 @@ const Allocator = std.mem.Allocator;
1830const Compilation = @import("Compilation.zig");
1931const log = std.log.scoped(.cache);
2032
33pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void {
34 cache.prefixes_buffer[cache.prefixes_len] = directory;
35 cache.prefixes_len += 1;
36}
37
2138/// Be sure to call `Manifest.deinit` after successful initialization.
2239pub fn obtain(cache: *Cache) Manifest {
2340 return Manifest{
......@@ -29,6 +46,48 @@ pub fn obtain(cache: *Cache) Manifest {
2946 };
3047}
3148
49pub fn prefixes(cache: *const Cache) []const Compilation.Directory {
50 return cache.prefixes_buffer[0..cache.prefixes_len];
51}
52
53const PrefixedPath = struct {
54 prefix: u8,
55 sub_path: []u8,
56};
57
58fn findPrefix(cache: *const Cache, file_path: []const u8) !PrefixedPath {
59 const gpa = cache.gpa;
60 const resolved_path = try fs.path.resolve(gpa, &[_][]const u8{file_path});
61 errdefer gpa.free(resolved_path);
62 return findPrefixResolved(cache, resolved_path);
63}
64
65/// Takes ownership of `resolved_path` on success.
66fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath {
67 const gpa = cache.gpa;
68 const prefixes_slice = cache.prefixes();
69 var i: u8 = 1; // Start at 1 to skip over checking the null prefix.
70 while (i < prefixes_slice.len) : (i += 1) {
71 const p = prefixes_slice[i].path.?;
72 if (mem.startsWith(u8, resolved_path, p)) {
73 // +1 to skip over the path separator here
74 const sub_path = try gpa.dupe(u8, resolved_path[p.len + 1 ..]);
75 gpa.free(resolved_path);
76 return PrefixedPath{
77 .prefix = @intCast(u8, i),
78 .sub_path = sub_path,
79 };
80 } else {
81 log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p });
82 }
83 }
84
85 return PrefixedPath{
86 .prefix = 0,
87 .sub_path = resolved_path,
88 };
89}
90
3291/// This is 128 bits - Even with 2^54 cache entries, the probably of a collision would be under 10^-6
3392pub const bin_digest_len = 16;
3493pub const hex_digest_len = bin_digest_len * 2;
......@@ -45,7 +104,7 @@ pub const Hasher = crypto.auth.siphash.SipHash128(1, 3);
45104pub const hasher_init: Hasher = Hasher.init(&[_]u8{0} ** Hasher.key_length);
46105
47106pub const File = struct {
48 path: ?[]const u8,
107 prefixed_path: ?PrefixedPath,
49108 max_file_size: ?usize,
50109 stat: Stat,
51110 bin_digest: BinDigest,
......@@ -57,13 +116,13 @@ pub const File = struct {
57116 mtime: i128,
58117 };
59118
60 pub fn deinit(self: *File, allocator: Allocator) void {
61 if (self.path) |owned_slice| {
62 allocator.free(owned_slice);
63 self.path = null;
119 pub fn deinit(self: *File, gpa: Allocator) void {
120 if (self.prefixed_path) |pp| {
121 gpa.free(pp.sub_path);
122 self.prefixed_path = null;
64123 }
65124 if (self.contents) |contents| {
66 allocator.free(contents);
125 gpa.free(contents);
67126 self.contents = null;
68127 }
69128 self.* = undefined;
......@@ -175,9 +234,6 @@ pub const Lock = struct {
175234 }
176235};
177236
178/// Manifest manages project-local `zig-cache` directories.
179/// This is not a general-purpose cache.
180/// It is designed to be fast and simple, not to withstand attacks using specially-crafted input.
181237pub const Manifest = struct {
182238 cache: *Cache,
183239 /// Current state for incremental hashing.
......@@ -220,21 +276,27 @@ pub const Manifest = struct {
220276 pub fn addFile(self: *Manifest, file_path: []const u8, max_file_size: ?usize) !usize {
221277 assert(self.manifest_file == null);
222278
223 try self.files.ensureUnusedCapacity(self.cache.gpa, 1);
224 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
279 const gpa = self.cache.gpa;
280 try self.files.ensureUnusedCapacity(gpa, 1);
281 const prefixed_path = try self.cache.findPrefix(file_path);
282 errdefer gpa.free(prefixed_path.sub_path);
283
284 log.debug("Manifest.addFile {s} -> {d} {s}", .{
285 file_path, prefixed_path.prefix, prefixed_path.sub_path,
286 });
225287
226 const idx = self.files.items.len;
227288 self.files.addOneAssumeCapacity().* = .{
228 .path = resolved_path,
289 .prefixed_path = prefixed_path,
229290 .contents = null,
230291 .max_file_size = max_file_size,
231292 .stat = undefined,
232293 .bin_digest = undefined,
233294 };
234295
235 self.hash.addBytes(resolved_path);
296 self.hash.add(prefixed_path.prefix);
297 self.hash.addBytes(prefixed_path.sub_path);
236298
237 return idx;
299 return self.files.items.len - 1;
238300 }
239301
240302 pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void {
......@@ -281,6 +343,7 @@ pub const Manifest = struct {
281343 /// option, one may call `toOwnedLock` to obtain a smaller object which can represent
282344 /// the lock. `deinit` is safe to call whether or not `toOwnedLock` has been called.
283345 pub fn hit(self: *Manifest) !bool {
346 const gpa = self.cache.gpa;
284347 assert(self.manifest_file == null);
285348
286349 self.failed_file_index = null;
......@@ -362,8 +425,8 @@ pub const Manifest = struct {
362425
363426 self.want_refresh_timestamp = true;
364427
365 const file_contents = try self.manifest_file.?.reader().readAllAlloc(self.cache.gpa, manifest_file_size_max);
366 defer self.cache.gpa.free(file_contents);
428 const file_contents = try self.manifest_file.?.reader().readAllAlloc(gpa, manifest_file_size_max);
429 defer gpa.free(file_contents);
367430
368431 const input_file_count = self.files.items.len;
369432 var any_file_changed = false;
......@@ -373,9 +436,9 @@ pub const Manifest = struct {
373436 defer idx += 1;
374437
375438 const cache_hash_file = if (idx < input_file_count) &self.files.items[idx] else blk: {
376 const new = try self.files.addOne(self.cache.gpa);
439 const new = try self.files.addOne(gpa);
377440 new.* = .{
378 .path = null,
441 .prefixed_path = null,
379442 .contents = null,
380443 .max_file_size = null,
381444 .stat = undefined,
......@@ -389,27 +452,35 @@ pub const Manifest = struct {
389452 const inode = iter.next() orelse return error.InvalidFormat;
390453 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
391454 const digest_str = iter.next() orelse return error.InvalidFormat;
455 const prefix_str = iter.next() orelse return error.InvalidFormat;
392456 const file_path = iter.rest();
393457
394458 cache_hash_file.stat.size = fmt.parseInt(u64, size, 10) catch return error.InvalidFormat;
395459 cache_hash_file.stat.inode = fmt.parseInt(fs.File.INode, inode, 10) catch return error.InvalidFormat;
396460 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
397461 _ = std.fmt.hexToBytes(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
462 const prefix = fmt.parseInt(u8, prefix_str, 10) catch return error.InvalidFormat;
463 if (prefix >= self.cache.prefixes_len) return error.InvalidFormat;
398464
399465 if (file_path.len == 0) {
400466 return error.InvalidFormat;
401467 }
402 if (cache_hash_file.path) |p| {
403 if (!mem.eql(u8, file_path, p)) {
468 if (cache_hash_file.prefixed_path) |pp| {
469 if (pp.prefix != prefix or !mem.eql(u8, file_path, pp.sub_path)) {
404470 return error.InvalidFormat;
405471 }
406472 }
407473
408 if (cache_hash_file.path == null) {
409 cache_hash_file.path = try self.cache.gpa.dupe(u8, file_path);
474 if (cache_hash_file.prefixed_path == null) {
475 cache_hash_file.prefixed_path = .{
476 .prefix = prefix,
477 .sub_path = try gpa.dupe(u8, file_path),
478 };
410479 }
411480
412 const this_file = fs.cwd().openFile(cache_hash_file.path.?, .{ .mode = .read_only }) catch |err| switch (err) {
481 const pp = cache_hash_file.prefixed_path.?;
482 const dir = self.cache.prefixes()[pp.prefix].handle;
483 const this_file = dir.openFile(pp.sub_path, .{ .mode = .read_only }) catch |err| switch (err) {
413484 error.FileNotFound => {
414485 try self.upgradeToExclusiveLock();
415486 return false;
......@@ -535,8 +606,9 @@ pub const Manifest = struct {
535606 }
536607
537608 fn populateFileHash(self: *Manifest, ch_file: *File) !void {
538 log.debug("populateFileHash {s}", .{ch_file.path.?});
539 const file = try fs.cwd().openFile(ch_file.path.?, .{});
609 const pp = ch_file.prefixed_path.?;
610 const dir = self.cache.prefixes()[pp.prefix].handle;
611 const file = try dir.openFile(pp.sub_path, .{});
540612 defer file.close();
541613
542614 const actual_stat = try file.stat();
......@@ -588,12 +660,17 @@ pub const Manifest = struct {
588660 pub fn addFilePostFetch(self: *Manifest, file_path: []const u8, max_file_size: usize) ![]const u8 {
589661 assert(self.manifest_file != null);
590662
591 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
592 errdefer self.cache.gpa.free(resolved_path);
663 const gpa = self.cache.gpa;
664 const prefixed_path = try self.cache.findPrefix(file_path);
665 errdefer gpa.free(prefixed_path.sub_path);
666
667 log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{
668 file_path, prefixed_path.prefix, prefixed_path.sub_path,
669 });
593670
594 const new_ch_file = try self.files.addOne(self.cache.gpa);
671 const new_ch_file = try self.files.addOne(gpa);
595672 new_ch_file.* = .{
596 .path = resolved_path,
673 .prefixed_path = prefixed_path,
597674 .max_file_size = max_file_size,
598675 .stat = undefined,
599676 .bin_digest = undefined,
......@@ -613,12 +690,17 @@ pub const Manifest = struct {
613690 pub fn addFilePost(self: *Manifest, file_path: []const u8) !void {
614691 assert(self.manifest_file != null);
615692
616 const resolved_path = try fs.path.resolve(self.cache.gpa, &[_][]const u8{file_path});
617 errdefer self.cache.gpa.free(resolved_path);
693 const gpa = self.cache.gpa;
694 const prefixed_path = try self.cache.findPrefix(file_path);
695 errdefer gpa.free(prefixed_path.sub_path);
696
697 log.debug("Manifest.addFilePost {s} -> {d} {s}", .{
698 file_path, prefixed_path.prefix, prefixed_path.sub_path,
699 });
618700
619 const new_ch_file = try self.files.addOne(self.cache.gpa);
701 const new_ch_file = try self.files.addOne(gpa);
620702 new_ch_file.* = .{
621 .path = resolved_path,
703 .prefixed_path = prefixed_path,
622704 .max_file_size = null,
623705 .stat = undefined,
624706 .bin_digest = undefined,
......@@ -633,17 +715,27 @@ pub const Manifest = struct {
633715 /// On success, cache takes ownership of `resolved_path`.
634716 pub fn addFilePostContents(
635717 self: *Manifest,
636 resolved_path: []const u8,
718 resolved_path: []u8,
637719 bytes: []const u8,
638720 stat: File.Stat,
639721 ) error{OutOfMemory}!void {
640722 assert(self.manifest_file != null);
723 const gpa = self.cache.gpa;
641724
642 const ch_file = try self.files.addOne(self.cache.gpa);
725 const ch_file = try self.files.addOne(gpa);
643726 errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1);
644727
728 log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path});
729
730 const prefixed_path = try self.cache.findPrefixResolved(resolved_path);
731 errdefer gpa.free(prefixed_path.sub_path);
732
733 log.debug("Manifest.addFilePostContents -> {d} {s}", .{
734 prefixed_path.prefix, prefixed_path.sub_path,
735 });
736
645737 ch_file.* = .{
646 .path = resolved_path,
738 .prefixed_path = prefixed_path,
647739 .max_file_size = null,
648740 .stat = stat,
649741 .bin_digest = undefined,
......@@ -742,12 +834,13 @@ pub const Manifest = struct {
742834 "{s}",
743835 .{std.fmt.fmtSliceHexLower(&file.bin_digest)},
744836 ) catch unreachable;
745 try writer.print("{d} {d} {d} {s} {s}\n", .{
837 try writer.print("{d} {d} {d} {s} {d} {s}\n", .{
746838 file.stat.size,
747839 file.stat.inode,
748840 file.stat.mtime,
749841 &encoded_digest,
750 file.path.?,
842 file.prefixed_path.?.prefix,
843 file.prefixed_path.?.sub_path,
751844 });
752845 }
753846
......@@ -889,6 +982,7 @@ test "cache file and then recall it" {
889982 .gpa = testing.allocator,
890983 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
891984 };
985 cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
892986 defer cache.manifest_dir.close();
893987
894988 {
......@@ -960,6 +1054,7 @@ test "check that changing a file makes cache fail" {
9601054 .gpa = testing.allocator,
9611055 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
9621056 };
1057 cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
9631058 defer cache.manifest_dir.close();
9641059
9651060 {
......@@ -1022,6 +1117,7 @@ test "no file inputs" {
10221117 .gpa = testing.allocator,
10231118 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
10241119 };
1120 cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
10251121 defer cache.manifest_dir.close();
10261122
10271123 {
......@@ -1080,6 +1176,7 @@ test "Manifest with files added after initial hash work" {
10801176 .gpa = testing.allocator,
10811177 .manifest_dir = try cwd.makeOpenPath(temp_manifest_dir, .{}),
10821178 };
1179 cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
10831180 defer cache.manifest_dir.close();
10841181
10851182 {
src/Compilation.zig+14-9
......@@ -1456,23 +1456,27 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation {
14561456 else => @as(u8, 3),
14571457 };
14581458
1459 // We put everything into the cache hash that *cannot be modified during an incremental update*.
1460 // For example, one cannot change the target between updates, but one can change source files,
1461 // so the target goes into the cache hash, but source files do not. This is so that we can
1462 // find the same binary and incrementally update it even if there are modified source files.
1463 // We do this even if outputting to the current directory because we need somewhere to store
1464 // incremental compilation metadata.
1459 // We put everything into the cache hash that *cannot be modified
1460 // during an incremental update*. For example, one cannot change the
1461 // target between updates, but one can change source files, so the
1462 // target goes into the cache hash, but source files do not. This is so
1463 // that we can find the same binary and incrementally update it even if
1464 // there are modified source files. We do this even if outputting to
1465 // the current directory because we need somewhere to store incremental
1466 // compilation metadata.
14651467 const cache = try arena.create(Cache);
14661468 cache.* = .{
14671469 .gpa = gpa,
14681470 .manifest_dir = try options.local_cache_directory.handle.makeOpenPath("h", .{}),
14691471 };
1472 cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
1473 cache.addPrefix(options.zig_lib_directory);
1474 cache.addPrefix(options.local_cache_directory);
14701475 errdefer cache.manifest_dir.close();
14711476
14721477 // This is shared hasher state common to zig source and all C source files.
14731478 cache.hash.addBytes(build_options.version);
14741479 cache.hash.add(builtin.zig_backend);
1475 cache.hash.addBytes(options.zig_lib_directory.path orelse ".");
14761480 cache.hash.add(options.optimize_mode);
14771481 cache.hash.add(options.target.cpu.arch);
14781482 cache.hash.addBytes(options.target.cpu.model.name);
......@@ -2265,8 +2269,9 @@ pub fn update(comp: *Compilation) !void {
22652269 const is_hit = man.hit() catch |err| {
22662270 // TODO properly bubble these up instead of emitting a warning
22672271 const i = man.failed_file_index orelse return err;
2268 const file_path = man.files.items[i].path orelse return err;
2269 std.log.warn("{s}: {s}", .{ @errorName(err), file_path });
2272 const pp = man.files.items[i].prefixed_path orelse return err;
2273 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
2274 std.log.warn("{s}: {s}{s}", .{ @errorName(err), prefix, pp.sub_path });
22702275 return err;
22712276 };
22722277 if (is_hit) {
src/glibc.zig+3
......@@ -653,6 +653,9 @@ pub fn buildSharedObjects(comp: *Compilation) !void {
653653 .gpa = comp.gpa,
654654 .manifest_dir = try comp.global_cache_directory.handle.makeOpenPath("h", .{}),
655655 };
656 cache.addPrefix(.{ .path = null, .handle = fs.cwd() });
657 cache.addPrefix(comp.zig_lib_directory);
658 cache.addPrefix(comp.global_cache_directory);
656659 defer cache.manifest_dir.close();
657660
658661 var man = cache.obtain();
src/mingw.zig+4
......@@ -302,6 +302,10 @@ pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void {
302302 .gpa = comp.gpa,
303303 .manifest_dir = comp.cache_parent.manifest_dir,
304304 };
305 for (comp.cache_parent.prefixes()) |prefix| {
306 cache.addPrefix(prefix);
307 }
308
305309 cache.hash.addBytes(build_options.version);
306310 cache.hash.addOptionalBytes(comp.zig_lib_directory.path);
307311 cache.hash.add(target.cpu.arch);