authorgravatar for leroycepearson@geemili.xyzLeRoyce Pearson <leroycepearson@geemili.xyz> 2020-03-05 00:07:17-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 13:48:43-04:00
log3158dc424e48d3da52d9a8f99cba65a4ef850f2e
tree85d8359db4040a7c840dd9e3c05f16383c8337b9
parent0ecdbdb3cb3d9558b5d2dbd928ead124d98c74ca

Partially implement cache hash API in zig

2 files changed, 330 insertions(+), 0 deletions(-)

lib/std/cache_hash.zig created+329
......@@ -0,0 +1,329 @@
1const Blake3 = @import("crypto.zig").Blake3;
2const fs = @import("fs.zig");
3const File = fs.File;
4const base64 = @import("base64.zig");
5const ArrayList = @import("array_list.zig").ArrayList;
6const debug = @import("debug.zig");
7const testing = @import("testing.zig");
8const mem = @import("mem.zig");
9const fmt = @import("fmt.zig");
10const Allocator = mem.Allocator;
11const Buffer = @import("buffer.zig").Buffer;
12const os = @import("os.zig");
13
14const base64_alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_";
15const base64_pad_char = '=';
16const encoder = base64.Base64Encoder.init(base64_alphabet, base64_pad_char);
17const decoder = base64.Base64Decoder.init(base64_alphabet, base64_pad_char);
18const BIN_DIGEST_LEN = 32;
19
20pub const CacheHashFile = struct {
21 path: ?[]const u8,
22 stat: fs.File.Stat,
23 file_handle: os.fd_t,
24 bin_digest: [BIN_DIGEST_LEN]u8,
25 contents: ?[]const u8,
26
27 pub fn deinit(self: *@This(), alloc: *Allocator) void {
28 if (self.path) |owned_slice| {
29 alloc.free(owned_slice);
30 self.path = null;
31 }
32 if (self.contents) |owned_slice| {
33 alloc.free(owned_slice);
34 self.contents = null;
35 }
36 }
37};
38
39pub const CacheHash = struct {
40 alloc: *Allocator,
41 blake3: Blake3,
42 manifest_dir: []const u8,
43 manifest_file_path: ?[]const u8,
44 manifest_file: ?File,
45 manifest_dirty: bool,
46 force_check_manifest: bool,
47 files: ArrayList(CacheHashFile),
48 b64_digest: ArrayList(u8),
49
50 pub fn init(alloc: *Allocator, manifest_dir_path: []const u8) !@This() {
51 return CacheHash{
52 .alloc = alloc,
53 .blake3 = Blake3.init(),
54 .manifest_dir = manifest_dir_path,
55 .manifest_file_path = null,
56 .manifest_file = null,
57 .manifest_dirty = false,
58 .force_check_manifest = false,
59 .files = ArrayList(CacheHashFile).init(alloc),
60 .b64_digest = ArrayList(u8).init(alloc),
61 };
62 }
63
64 pub fn cache_buf(self: *@This(), val: []const u8) !void {
65 debug.assert(self.manifest_file_path == null);
66
67 var temp_buffer = try self.alloc.alloc(u8, val.len + 1);
68 defer self.alloc.free(temp_buffer);
69
70 mem.copy(u8, temp_buffer, val);
71 temp_buffer[val.len] = 0;
72
73 self.blake3.update(temp_buffer);
74 }
75
76 pub fn cache_file(self: *@This(), file_path: []const u8) !void {
77 debug.assert(self.manifest_file_path == null);
78
79 var cache_hash_file = try self.files.addOne();
80 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
81
82 try self.cache_buf(cache_hash_file.path.?);
83 }
84
85 pub fn hit(self: *@This(), out_digest: *ArrayList(u8)) !bool {
86 debug.assert(self.manifest_file_path == null);
87
88 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
89 self.blake3.final(&bin_digest);
90
91 const OUT_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
92 try self.b64_digest.resize(OUT_DIGEST_LEN);
93 encoder.encode(self.b64_digest.toSlice(), &bin_digest);
94
95 if (self.files.toSlice().len == 0 and !self.force_check_manifest) {
96 try out_digest.resize(OUT_DIGEST_LEN);
97 mem.copy(u8, out_digest.toSlice(), self.b64_digest.toSlice());
98 return true;
99 }
100
101 self.blake3 = Blake3.init();
102 self.blake3.update(&bin_digest);
103
104 {
105 const manifest_file_path_slice = try fs.path.join(self.alloc, &[_][]const u8{ self.manifest_dir, self.b64_digest.toSlice() });
106 var path_buf = ArrayList(u8).fromOwnedSlice(self.alloc, manifest_file_path_slice);
107 defer path_buf.deinit();
108 try path_buf.appendSlice(".txt");
109
110 self.manifest_file_path = path_buf.toOwnedSlice();
111 }
112
113 const cwd = fs.cwd();
114
115 try cwd.makePath(self.manifest_dir);
116
117 // TODO: Open file with a file lock
118 self.manifest_file = try cwd.createFile(self.manifest_file_path.?, .{ .read = true, .truncate = false });
119
120 // TODO: Figure out a good max value?
121 const file_contents = try self.manifest_file.?.inStream().stream.readAllAlloc(self.alloc, 16 * 1024);
122 defer self.alloc.free(file_contents);
123
124 const input_file_count = self.files.len;
125 var any_file_changed = false;
126 var line_iter = mem.tokenize(file_contents, "\n");
127 var idx: usize = 0;
128 while (line_iter.next()) |line| {
129 defer idx += 1;
130
131 var cache_hash_file: *CacheHashFile = undefined;
132 if (idx < input_file_count) {
133 cache_hash_file = self.files.ptrAt(idx);
134 } else {
135 cache_hash_file = try self.files.addOne();
136 cache_hash_file.path = null;
137 }
138
139 var iter = mem.tokenize(line, " ");
140 const file_handle_str = iter.next() orelse return error.InvalidFormat;
141 const mtime_nsec_str = iter.next() orelse return error.InvalidFormat;
142 const digest_str = iter.next() orelse return error.InvalidFormat;
143 const file_path = iter.rest();
144
145 cache_hash_file.file_handle = fmt.parseInt(os.fd_t, file_handle_str, 10) catch return error.InvalidFormat;
146 cache_hash_file.stat.mtime = fmt.parseInt(i64, mtime_nsec_str, 10) catch return error.InvalidFormat;
147 decoder.decode(&cache_hash_file.bin_digest, digest_str) catch return error.InvalidFormat;
148
149 if (file_path.len == 0) {
150 return error.InvalidFormat;
151 }
152 if (cache_hash_file.path != null and !mem.eql(u8, file_path, cache_hash_file.path.?)) {
153 return error.InvalidFormat;
154 }
155 cache_hash_file.path = try mem.dupe(self.alloc, u8, file_path);
156
157 const this_file = cwd.openFile(cache_hash_file.path.?, .{ .read = true }) catch {
158 self.manifest_file.?.close();
159 self.manifest_file = null;
160 return error.CacheUnavailable;
161 };
162 defer this_file.close();
163 cache_hash_file.stat = try this_file.stat();
164 // TODO: check mtime
165 if (false) {} else {
166 self.manifest_dirty = true;
167
168 // TODO: check for problematic timestamp
169
170 var actual_digest: [32]u8 = undefined;
171 try hash_file(self.alloc, &actual_digest, &this_file);
172
173 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
174 mem.copy(u8, &cache_hash_file.bin_digest, &actual_digest);
175 // keep going until we have the input file digests
176 any_file_changed = true;
177 }
178 }
179
180 if (!any_file_changed) {
181 self.blake3.update(&cache_hash_file.bin_digest);
182 }
183 }
184
185 if (any_file_changed) {
186 // cache miss
187 // keep the manifest file open (TODO: with rw lock)
188 // reset the hash
189 self.blake3 = Blake3.init();
190 self.blake3.update(&bin_digest);
191 try self.files.resize(input_file_count);
192 for (self.files.toSlice()) |file| {
193 self.blake3.update(&file.bin_digest);
194 }
195 return false;
196 }
197
198 if (idx < input_file_count or idx == 0) {
199 self.manifest_dirty = true;
200 while (idx < input_file_count) : (idx += 1) {
201 var cache_hash_file = self.files.ptrAt(idx);
202 self.populate_file_hash(cache_hash_file) catch |err| {
203 self.manifest_file.?.close();
204 self.manifest_file = null;
205 return error.CacheUnavailable;
206 };
207 }
208 return false;
209 }
210
211 try self.final(out_digest);
212 return true;
213 }
214
215 pub fn populate_file_hash(self: *@This(), cache_hash_file: *CacheHashFile) !void {
216 debug.assert(cache_hash_file.path != null);
217
218 const this_file = try fs.cwd().openFile(cache_hash_file.path.?, .{});
219 defer this_file.close();
220
221 cache_hash_file.stat = try this_file.stat();
222
223 // TODO: check for problematic timestamp
224
225 try hash_file(self.alloc, &cache_hash_file.bin_digest, &this_file);
226 self.blake3.update(&cache_hash_file.bin_digest);
227 }
228
229 pub fn final(self: *@This(), out_digest: *ArrayList(u8)) !void {
230 debug.assert(self.manifest_file_path != null);
231
232 var bin_digest: [BIN_DIGEST_LEN]u8 = undefined;
233 self.blake3.final(&bin_digest);
234
235 const OUT_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
236 try out_digest.resize(OUT_DIGEST_LEN);
237 encoder.encode(out_digest.toSlice(), &bin_digest);
238 }
239
240 pub fn write_manifest(self: *@This()) !void {
241 debug.assert(self.manifest_file_path != null);
242
243 const OUT_DIGEST_LEN = base64.Base64Encoder.calcSize(BIN_DIGEST_LEN);
244 var encoded_digest = try Buffer.initSize(self.alloc, OUT_DIGEST_LEN);
245 defer encoded_digest.deinit();
246 var contents = try Buffer.init(self.alloc, "");
247 defer contents.deinit();
248
249 for (self.files.toSlice()) |file| {
250 encoder.encode(encoded_digest.toSlice(), &file.bin_digest);
251 try contents.print("{} {} {} {}\n", .{ file.file_handle, file.stat.mtime, encoded_digest.toSlice(), file.path });
252 }
253
254 try self.manifest_file.?.seekTo(0);
255 try self.manifest_file.?.writeAll(contents.toSlice());
256 }
257
258 pub fn release(self: *@This()) void {
259 debug.assert(self.manifest_file_path != null);
260
261 if (self.manifest_dirty) {
262 self.write_manifest() catch |err| {
263 debug.warn("Unable to write cache file '{}': {}\n", .{ self.manifest_file_path, err });
264 };
265 }
266
267 self.manifest_file.?.close();
268 if (self.manifest_file_path) |owned_slice| {
269 self.alloc.free(owned_slice);
270 }
271 for (self.files.toSlice()) |*file| {
272 file.deinit(self.alloc);
273 }
274 self.files.deinit();
275 self.b64_digest.deinit();
276 }
277};
278
279fn hash_file(alloc: *Allocator, bin_digest: []u8, handle: *const File) !void {
280 var blake3 = Blake3.init();
281 var in_stream = handle.inStream().stream;
282
283 const contents = try handle.inStream().stream.readAllAlloc(alloc, 64 * 1024);
284 defer alloc.free(contents);
285
286 blake3.update(contents);
287
288 blake3.final(bin_digest);
289}
290
291test "see if imported" {
292 const cwd = fs.cwd();
293
294 const temp_manifest_dir = "temp_manifest_dir";
295
296 try cwd.writeFile("test.txt", "Hello, world!\n");
297
298 var digest1 = try ArrayList(u8).initCapacity(testing.allocator, 32);
299 defer digest1.deinit();
300 var digest2 = try ArrayList(u8).initCapacity(testing.allocator, 32);
301 defer digest2.deinit();
302
303 {
304 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
305 defer ch.release();
306
307 try ch.cache_buf("1234");
308 try ch.cache_file("test.txt");
309
310 // There should be nothing in the cache
311 debug.assert((try ch.hit(&digest1)) == false);
312
313 try ch.final(&digest1);
314 }
315 {
316 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
317 defer ch.release();
318
319 try ch.cache_buf("1234");
320 try ch.cache_file("test.txt");
321
322 // Cache hit! We just "built" the same file
323 debug.assert((try ch.hit(&digest2)) == true);
324 }
325
326 debug.assert(mem.eql(u8, digest1.toSlice(), digest2.toSlice()));
327
328 try cwd.deleteTree(temp_manifest_dir);
329}
lib/std/std.zig+1
......@@ -31,6 +31,7 @@ pub const base64 = @import("base64.zig");
3131pub const build = @import("build.zig");
3232pub const builtin = @import("builtin.zig");
3333pub const c = @import("c.zig");
34pub const cache_hash = @import("cache_hash.zig");
3435pub const coff = @import("coff.zig");
3536pub const crypto = @import("crypto.zig");
3637pub const cstr = @import("cstr.zig");