authorgravatar for leroycepearson@geemili.xyzLeRoyce Pearson <leroycepearson@geemili.xyz> 2020-05-01 23:06:10-06:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 13:48:43-04:00
logc3c332c9ecf8ab982ca1b704a54b7d1b439b520d
tree2957c21b54dde838e66910ae2ca914cc79240bc5
parent5a1c6a362743882ba2923570c24f77f555a39504

Add max_file_size argument

1 files changed, 50 insertions(+), 22 deletions(-)

lib/std/cache_hash.zig+50-22
......@@ -19,6 +19,7 @@ const MANIFEST_FILE_SIZE_MAX = 50 * 1024 * 1024;
1919
2020pub const File = struct {
2121 path: ?[]const u8,
22 max_file_size: ?usize,
2223 stat: fs.File.Stat,
2324 bin_digest: [BIN_DIGEST_LEN]u8,
2425 contents: ?[]const u8 = null,
......@@ -85,18 +86,23 @@ pub const CacheHash = struct {
8586 /// called, the file's contents will be checked to ensure that it matches
8687 /// the contents from previous times.
8788 ///
89 /// Max file size will be used to determine the amount of space to the file contents
90 /// are allowed to take up in memory. If max_file_size is null, then the contents
91 /// will not be loaded into memory.
92 ///
8893 /// Returns the index of the entry in the `CacheHash.files` ArrayList. You can use it
8994 /// to access the contents of the file after calling `CacheHash.hit()` like so:
9095 ///
9196 /// ```
9297 /// var file_contents = cache_hash.files.items[file_index].contents.?;
9398 /// ```
94 pub fn addFile(self: *@This(), file_path: []const u8) !usize {
99 pub fn addFile(self: *@This(), file_path: []const u8, max_file_size: ?usize) !usize {
95100 debug.assert(self.manifest_file == null);
96101
97102 const idx = self.files.items.len;
98103 var cache_hash_file = try self.files.addOne();
99104 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
105 cache_hash_file.max_file_size = max_file_size;
100106
101107 self.addSlice(cache_hash_file.path.?);
102108
......@@ -168,6 +174,7 @@ pub const CacheHash = struct {
168174 } else {
169175 cache_hash_file = try self.files.addOne();
170176 cache_hash_file.path = null;
177 cache_hash_file.max_file_size = null;
171178 }
172179
173180 var iter = mem.tokenize(line, " ");
......@@ -213,7 +220,7 @@ pub const CacheHash = struct {
213220 }
214221
215222 var actual_digest: [BIN_DIGEST_LEN]u8 = undefined;
216 cache_hash_file.contents = try hash_file(self.alloc, &actual_digest, &this_file);
223 cache_hash_file.contents = try hash_file(self.alloc, &actual_digest, &this_file, cache_hash_file.max_file_size);
217224
218225 if (!mem.eql(u8, &cache_hash_file.bin_digest, &actual_digest)) {
219226 mem.copy(u8, &cache_hash_file.bin_digest, &actual_digest);
......@@ -260,7 +267,7 @@ pub const CacheHash = struct {
260267 return self.final();
261268 }
262269
263 fn populate_file_hash_fetch(self: *@This(), otherAlloc: *mem.Allocator, cache_hash_file: *File) ![]u8 {
270 fn populate_file_hash_fetch(self: *@This(), otherAlloc: *mem.Allocator, cache_hash_file: *File) !?[]u8 {
264271 debug.assert(cache_hash_file.path != null);
265272
266273 const this_file = try fs.cwd().openFile(cache_hash_file.path.?, .{});
......@@ -273,7 +280,7 @@ pub const CacheHash = struct {
273280 cache_hash_file.stat.inode = 0;
274281 }
275282
276 const contents = try hash_file(otherAlloc, &cache_hash_file.bin_digest, &this_file);
283 const contents = try hash_file(otherAlloc, &cache_hash_file.bin_digest, &this_file, cache_hash_file.max_file_size);
277284 self.blake3.update(&cache_hash_file.bin_digest);
278285
279286 return contents;
......@@ -289,13 +296,15 @@ pub const CacheHash = struct {
289296 /// will need to be recompiled if the imported file is changed.
290297 ///
291298 /// Returns the contents of the file, allocated with the given allocator.
292 pub fn addFilePostFetch(self: *@This(), otherAlloc: *mem.Allocator, file_path: []const u8) ![]u8 {
299 pub fn addFilePostFetch(self: *@This(), otherAlloc: *mem.Allocator, file_path: []const u8, max_file_size_opt: ?usize) !?[]u8 {
293300 debug.assert(self.manifest_file != null);
294301
295302 var cache_hash_file = try self.files.addOne();
296303 cache_hash_file.path = try fs.path.resolve(self.alloc, &[_][]const u8{file_path});
297304
298 return try self.populate_file_hash_fetch(otherAlloc, cache_hash_file);
305 const contents = try self.populate_file_hash_fetch(otherAlloc, cache_hash_file);
306
307 return contents;
299308 }
300309
301310 /// Add a file as a dependency of process being cached, after the initial hash has been
......@@ -303,8 +312,7 @@ pub const CacheHash = struct {
303312 /// are depended on ahead of time. For example, a source file that can import other files
304313 /// will need to be recompiled if the imported file is changed.
305314 pub fn addFilePost(self: *@This(), file_path: []const u8) !void {
306 const contents = try self.addFilePostFetch(self.alloc, file_path);
307 self.alloc.free(contents);
315 _ = try self.addFilePostFetch(self.alloc, file_path, null);
308316 }
309317
310318 /// Returns a base64 encoded hash of the inputs.
......@@ -367,16 +375,30 @@ pub const CacheHash = struct {
367375};
368376
369377/// Hash the file, and return the contents as an array
370fn hash_file(alloc: *Allocator, bin_digest: []u8, handle: *const fs.File) ![]u8 {
378fn hash_file(alloc: *Allocator, bin_digest: []u8, handle: *const fs.File, max_file_size_opt: ?usize) !?[]u8 {
371379 var blake3 = Blake3.init();
380 var in_stream = handle.inStream();
372381
373 const contents = try handle.inStream().readAllAlloc(alloc, 64 * 1024);
382 if (max_file_size_opt) |max_file_size| {
383 const contents = try in_stream.readAllAlloc(alloc, max_file_size);
374384
375 blake3.update(contents);
385 blake3.update(contents);
376386
377 blake3.final(bin_digest);
387 blake3.final(bin_digest);
378388
379 return contents;
389 return contents;
390 } else {
391 var buf: [1024]u8 = undefined;
392
393 while (true) {
394 const bytes_read = try in_stream.read(buf[0..]);
395 if (bytes_read == 0) break;
396 blake3.update(buf[0..bytes_read]);
397 }
398
399 blake3.final(bin_digest);
400 return null;
401 }
380402}
381403
382404/// If the wall clock time, rounded to the same precision as the
......@@ -407,7 +429,7 @@ test "cache file and then recall it" {
407429 ch.add(true);
408430 ch.add(@as(u16, 1234));
409431 ch.add("1234");
410 _ = try ch.addFile(temp_file);
432 _ = try ch.addFile(temp_file, null);
411433
412434 // There should be nothing in the cache
413435 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
......@@ -421,7 +443,7 @@ test "cache file and then recall it" {
421443 ch.add(true);
422444 ch.add(@as(u16, 1234));
423445 ch.add("1234");
424 _ = try ch.addFile(temp_file);
446 _ = try ch.addFile(temp_file, null);
425447
426448 // Cache hit! We just "built" the same file
427449 digest2 = (try ch.hit()).?;
......@@ -448,8 +470,10 @@ test "check that changing a file makes cache fail" {
448470
449471 const temp_file = "cache_hash_change_file_test.txt";
450472 const temp_manifest_dir = "cache_hash_change_file_manifest_dir";
473 const original_temp_file_contents = "Hello, world!\n";
474 const updated_temp_file_contents = "Hello, world; but updated!\n";
451475
452 try cwd.writeFile(temp_file, "Hello, world!\n");
476 try cwd.writeFile(temp_file, original_temp_file_contents);
453477
454478 var digest1: [BASE64_DIGEST_LEN]u8 = undefined;
455479 var digest2: [BASE64_DIGEST_LEN]u8 = undefined;
......@@ -459,26 +483,30 @@ test "check that changing a file makes cache fail" {
459483 defer ch.release() catch unreachable;
460484
461485 ch.add("1234");
462 _ = try ch.addFile(temp_file);
486 const temp_file_idx = try ch.addFile(temp_file, 100);
463487
464488 // There should be nothing in the cache
465489 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
466490
491 testing.expect(mem.eql(u8, original_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
492
467493 digest1 = ch.final();
468494 }
469495
470 try cwd.writeFile(temp_file, "Hello, world; but updated!\n");
496 try cwd.writeFile(temp_file, updated_temp_file_contents);
471497
472498 {
473499 var ch = try CacheHash.init(testing.allocator, temp_manifest_dir);
474500 defer ch.release() catch unreachable;
475501
476502 ch.add("1234");
477 _ = try ch.addFile(temp_file);
503 const temp_file_idx = try ch.addFile(temp_file, 100);
478504
479505 // A file that we depend on has been updated, so the cache should not contain an entry for it
480506 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
481507
508 testing.expect(mem.eql(u8, updated_temp_file_contents, ch.files.items[temp_file_idx].contents.?));
509
482510 digest2 = ch.final();
483511 }
484512
......@@ -538,7 +566,7 @@ test "CacheHashes with files added after initial hash work" {
538566 defer ch.release() catch unreachable;
539567
540568 ch.add("1234");
541 _ = try ch.addFile(temp_file1);
569 _ = try ch.addFile(temp_file1, null);
542570
543571 // There should be nothing in the cache
544572 testing.expectEqual(@as(?[64]u8, null), try ch.hit());
......@@ -552,7 +580,7 @@ test "CacheHashes with files added after initial hash work" {
552580 defer ch.release() catch unreachable;
553581
554582 ch.add("1234");
555 _ = try ch.addFile(temp_file1);
583 _ = try ch.addFile(temp_file1, null);
556584
557585 // A file that we depend on has been updated, so the cache should not contain an entry for it
558586 digest2 = (try ch.hit()).?;
......@@ -566,7 +594,7 @@ test "CacheHashes with files added after initial hash work" {
566594 defer ch.release() catch unreachable;
567595
568596 ch.add("1234");
569 _ = try ch.addFile(temp_file1);
597 _ = try ch.addFile(temp_file1, null);
570598
571599 // A file that we depend on has been updated, so the cache should not contain an entry for it
572600 testing.expectEqual(@as(?[64]u8, null), try ch.hit());