| author |  mlarouche <michael.larouche@gmail.com> 2020-10-09 16:50:43-04:00 |
| committer | mlarouche <michael.larouche@gmail.com> 2020-10-09 16:50:43-04:00 |
| log | 57912964af0247a7aa940f76d09a8994d8fe1ec8 |
| tree | 89d818f19130c03b55926d4b6beb927be9bc85c2 |
| parent | 04b0ffdd13e32be0ef5cc84983f8bb830db7520f |
Use regular file for caching stage 1 hash digest instead of symlink, fix zig build caching on Windows
Fix #65007 files changed, 36 insertions(+), 27 deletions(-)
lib/std/fs.zig+9
| ... | @@ -1490,6 +1490,15 @@ pub const Dir = struct { | ... | @@ -1490,6 +1490,15 @@ pub const Dir = struct { |
| 1490 | return os.windows.ReadLink(self.fd, sub_path_w, buffer); | 1490 | return os.windows.ReadLink(self.fd, sub_path_w, buffer); |
| 1491 | } | 1491 | } |
| 1492 | 1492 | ||
| 1493 | /// Read all of file contents using a preallocated buffer | ||
| 1494 | pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 { | ||
| 1495 | var file = try self.openFile(file_path, .{}); | ||
| 1496 | defer file.close(); | ||
| 1497 | |||
| 1498 | const end_index = try file.readAll(buffer); | ||
| 1499 | return buffer[0..end_index]; | ||
| 1500 | } | ||
| 1501 | |||
| 1493 | /// On success, caller owns returned buffer. | 1502 | /// On success, caller owns returned buffer. |
| 1494 | /// If the file is larger than `max_bytes`, returns `error.FileTooBig`. | 1503 | /// If the file is larger than `max_bytes`, returns `error.FileTooBig`. |
| 1495 | pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 { | 1504 | pub fn readFileAlloc(self: Dir, allocator: *mem.Allocator, file_path: []const u8, max_bytes: usize) ![]u8 { |
src/Compilation.zig+4-4
| ... | @@ -2605,8 +2605,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node | ... | @@ -2605,8 +2605,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node |
| 2605 | 2605 | ||
| 2606 | // We use an extra hex-encoded byte here to store some flags. | 2606 | // We use an extra hex-encoded byte here to store some flags. |
| 2607 | var prev_digest_buf: [digest.len + 2]u8 = undefined; | 2607 | var prev_digest_buf: [digest.len + 2]u8 = undefined; |
| 2608 | const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { | 2608 | const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { |
| 2609 | log.debug("stage1 {} new_digest={} readlink error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); | 2609 | log.debug("stage1 {} new_digest={} readFile error: {}", .{ mod.root_pkg.root_src_path, digest, @errorName(err) }); |
| 2610 | // Handle this as a cache miss. | 2610 | // Handle this as a cache miss. |
| 2611 | break :blk prev_digest_buf[0..0]; | 2611 | break :blk prev_digest_buf[0..0]; |
| 2612 | }; | 2612 | }; |
| ... | @@ -2792,8 +2792,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node | ... | @@ -2792,8 +2792,8 @@ fn updateStage1Module(comp: *Compilation, main_progress_node: *std.Progress.Node |
| 2792 | log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{ | 2792 | log.debug("saved digest + flags: '{s}' (byte = {}) have_winmain_crt_startup={}", .{ |
| 2793 | digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup, | 2793 | digest_plus_flags, stage1_flags_byte, mod.stage1_flags.have_winmain_crt_startup, |
| 2794 | }); | 2794 | }); |
| 2795 | directory.handle.symLink(&digest_plus_flags, id_symlink_basename, .{}) catch |err| { | 2795 | directory.handle.writeFile(id_symlink_basename, &digest_plus_flags) catch |err| { |
| 2796 | log.warn("failed to save stage1 hash digest symlink: {}", .{@errorName(err)}); | 2796 | log.warn("failed to save stage1 hash digest file: {}", .{@errorName(err)}); |
| 2797 | }; | 2797 | }; |
| 2798 | // Again failure here only means an unnecessary cache miss. | 2798 | // Again failure here only means an unnecessary cache miss. |
| 2799 | man.writeManifest() catch |err| { | 2799 | man.writeManifest() catch |err| { |
src/link.zig+4-4
| ... | @@ -466,8 +466,8 @@ pub const File = struct { | ... | @@ -466,8 +466,8 @@ pub const File = struct { |
| 466 | const digest = ch.final(); | 466 | const digest = ch.final(); |
| 467 | 467 | ||
| 468 | var prev_digest_buf: [digest.len]u8 = undefined; | 468 | var prev_digest_buf: [digest.len]u8 = undefined; |
| 469 | const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| b: { | 469 | const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| b: { |
| 470 | log.debug("archive new_digest={} readlink error: {}", .{ digest, @errorName(err) }); | 470 | log.debug("archive new_digest={} readFile error: {}", .{ digest, @errorName(err) }); |
| 471 | break :b prev_digest_buf[0..0]; | 471 | break :b prev_digest_buf[0..0]; |
| 472 | }; | 472 | }; |
| 473 | if (mem.eql(u8, prev_digest, &digest)) { | 473 | if (mem.eql(u8, prev_digest, &digest)) { |
| ... | @@ -512,8 +512,8 @@ pub const File = struct { | ... | @@ -512,8 +512,8 @@ pub const File = struct { |
| 512 | const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type); | 512 | const bad = llvm.WriteArchive(full_out_path_z, object_files.items.ptr, object_files.items.len, os_type); |
| 513 | if (bad) return error.UnableToWriteArchive; | 513 | if (bad) return error.UnableToWriteArchive; |
| 514 | 514 | ||
| 515 | directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { | 515 | directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { |
| 516 | std.log.warn("failed to save archive hash digest symlink: {}", .{@errorName(err)}); | 516 | std.log.warn("failed to save archive hash digest file: {}", .{@errorName(err)}); |
| 517 | }; | 517 | }; |
| 518 | 518 | ||
| 519 | ch.writeManifest() catch |err| { | 519 | ch.writeManifest() catch |err| { |
src/link/Coff.zig+5-5
| ... | @@ -854,8 +854,8 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { | ... | @@ -854,8 +854,8 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { |
| 854 | _ = try man.hit(); | 854 | _ = try man.hit(); |
| 855 | digest = man.final(); | 855 | digest = man.final(); |
| 856 | var prev_digest_buf: [digest.len]u8 = undefined; | 856 | var prev_digest_buf: [digest.len]u8 = undefined; |
| 857 | const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { | 857 | const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { |
| 858 | log.debug("COFF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); | 858 | log.debug("COFF LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); |
| 859 | // Handle this as a cache miss. | 859 | // Handle this as a cache miss. |
| 860 | break :blk prev_digest_buf[0..0]; | 860 | break :blk prev_digest_buf[0..0]; |
| 861 | }; | 861 | }; |
| ... | @@ -1180,10 +1180,10 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { | ... | @@ -1180,10 +1180,10 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { |
| 1180 | } | 1180 | } |
| 1181 | 1181 | ||
| 1182 | if (!self.base.options.disable_lld_caching) { | 1182 | if (!self.base.options.disable_lld_caching) { |
| 1183 | // Update the dangling symlink with the digest. If it fails we can continue; it only | 1183 | // Update the dangling file with the digest. If it fails we can continue; it only |
| 1184 | // means that the next invocation will have an unnecessary cache miss. | 1184 | // means that the next invocation will have an unnecessary cache miss. |
| 1185 | directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { | 1185 | directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { |
| 1186 | std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); | 1186 | std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); |
| 1187 | }; | 1187 | }; |
| 1188 | // Again failure here only means an unnecessary cache miss. | 1188 | // Again failure here only means an unnecessary cache miss. |
| 1189 | man.writeManifest() catch |err| { | 1189 | man.writeManifest() catch |err| { |
src/link/Elf.zig+5-5
| ... | @@ -1326,8 +1326,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { | ... | @@ -1326,8 +1326,8 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { |
| 1326 | digest = man.final(); | 1326 | digest = man.final(); |
| 1327 | 1327 | ||
| 1328 | var prev_digest_buf: [digest.len]u8 = undefined; | 1328 | var prev_digest_buf: [digest.len]u8 = undefined; |
| 1329 | const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { | 1329 | const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { |
| 1330 | log.debug("ELF LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); | 1330 | log.debug("ELF LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); |
| 1331 | // Handle this as a cache miss. | 1331 | // Handle this as a cache miss. |
| 1332 | break :blk prev_digest_buf[0..0]; | 1332 | break :blk prev_digest_buf[0..0]; |
| 1333 | }; | 1333 | }; |
| ... | @@ -1647,10 +1647,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { | ... | @@ -1647,10 +1647,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { |
| 1647 | } | 1647 | } |
| 1648 | 1648 | ||
| 1649 | if (!self.base.options.disable_lld_caching) { | 1649 | if (!self.base.options.disable_lld_caching) { |
| 1650 | // Update the dangling symlink with the digest. If it fails we can continue; it only | 1650 | // Update the dangling file with the digest. If it fails we can continue; it only |
| 1651 | // means that the next invocation will have an unnecessary cache miss. | 1651 | // means that the next invocation will have an unnecessary cache miss. |
| 1652 | directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { | 1652 | directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { |
| 1653 | std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); | 1653 | std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); |
| 1654 | }; | 1654 | }; |
| 1655 | // Again failure here only means an unnecessary cache miss. | 1655 | // Again failure here only means an unnecessary cache miss. |
| 1656 | man.writeManifest() catch |err| { | 1656 | man.writeManifest() catch |err| { |
src/link/MachO.zig+5-5
| ... | @@ -419,8 +419,8 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { | ... | @@ -419,8 +419,8 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { |
| 419 | digest = man.final(); | 419 | digest = man.final(); |
| 420 | 420 | ||
| 421 | var prev_digest_buf: [digest.len]u8 = undefined; | 421 | var prev_digest_buf: [digest.len]u8 = undefined; |
| 422 | const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { | 422 | const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { |
| 423 | log.debug("MachO LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); | 423 | log.debug("MachO LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); |
| 424 | // Handle this as a cache miss. | 424 | // Handle this as a cache miss. |
| 425 | break :blk prev_digest_buf[0..0]; | 425 | break :blk prev_digest_buf[0..0]; |
| 426 | }; | 426 | }; |
| ... | @@ -674,10 +674,10 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { | ... | @@ -674,10 +674,10 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { |
| 674 | } | 674 | } |
| 675 | 675 | ||
| 676 | if (!self.base.options.disable_lld_caching) { | 676 | if (!self.base.options.disable_lld_caching) { |
| 677 | // Update the dangling symlink with the digest. If it fails we can continue; it only | 677 | // Update the dangling file with the digest. If it fails we can continue; it only |
| 678 | // means that the next invocation will have an unnecessary cache miss. | 678 | // means that the next invocation will have an unnecessary cache miss. |
| 679 | directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { | 679 | directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { |
| 680 | std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); | 680 | std.log.warn("failed to save linking hash digest file: {}", .{@errorName(err)}); |
| 681 | }; | 681 | }; |
| 682 | // Again failure here only means an unnecessary cache miss. | 682 | // Again failure here only means an unnecessary cache miss. |
| 683 | man.writeManifest() catch |err| { | 683 | man.writeManifest() catch |err| { |
src/link/Wasm.zig+4-4
| ... | @@ -310,8 +310,8 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { | ... | @@ -310,8 +310,8 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { |
| 310 | digest = man.final(); | 310 | digest = man.final(); |
| 311 | 311 | ||
| 312 | var prev_digest_buf: [digest.len]u8 = undefined; | 312 | var prev_digest_buf: [digest.len]u8 = undefined; |
| 313 | const prev_digest: []u8 = directory.handle.readLink(id_symlink_basename, &prev_digest_buf) catch |err| blk: { | 313 | const prev_digest: []u8 = directory.handle.readFile(id_symlink_basename, &prev_digest_buf) catch |err| blk: { |
| 314 | log.debug("WASM LLD new_digest={} readlink error: {}", .{ digest, @errorName(err) }); | 314 | log.debug("WASM LLD new_digest={} readFile error: {}", .{ digest, @errorName(err) }); |
| 315 | // Handle this as a cache miss. | 315 | // Handle this as a cache miss. |
| 316 | break :blk prev_digest_buf[0..0]; | 316 | break :blk prev_digest_buf[0..0]; |
| 317 | }; | 317 | }; |
| ... | @@ -424,9 +424,9 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { | ... | @@ -424,9 +424,9 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { |
| 424 | } | 424 | } |
| 425 | 425 | ||
| 426 | if (!self.base.options.disable_lld_caching) { | 426 | if (!self.base.options.disable_lld_caching) { |
| 427 | // Update the dangling symlink with the digest. If it fails we can continue; it only | 427 | // Update the dangling file with the digest. If it fails we can continue; it only |
| 428 | // means that the next invocation will have an unnecessary cache miss. | 428 | // means that the next invocation will have an unnecessary cache miss. |
| 429 | directory.handle.symLink(&digest, id_symlink_basename, .{}) catch |err| { | 429 | directory.handle.writeFile(id_symlink_basename, &digest) catch |err| { |
| 430 | std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); | 430 | std.log.warn("failed to save linking hash digest symlink: {}", .{@errorName(err)}); |
| 431 | }; | 431 | }; |
| 432 | // Again failure here only means an unnecessary cache miss. | 432 | // Again failure here only means an unnecessary cache miss. |