authorgravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-08-18 00:43:33+02:00
committergravatar for robin@voetter.nlRobin Voetter <robin@voetter.nl> 2024-08-19 19:09:11+02:00
log43f73af3595c3174b8e67e9f2792c3774f2192e9
tree5b178f2316304780f9e84abb60a4286a730ec194
parent54e48f7b7dec96c8cdd1a0a0491554b118767817
signaturebadge-check Signed by SSH key SHA256:ZS52FNyUv2WUXvO4njmVaFVO46RHojFuOrxRc4LuKzg

fix various issues related to Path handling in the compiler and std

A compilation build step for which the binary is not required could not be compiled previously. There were 2 issues that caused this: - The compiler communicated only the results of the emitted binary and did not properly communicate the result if the binary was not emitted. This is fixed by communicating the final hash of the artifact path (the hash of the corresponding /o/<hash> directory) and communicating this instead of the entire path. This changes the zig build --listen protocol to communicate hashes instead of paths, and emit_bin_path is accordingly renamed to emit_digest. - There was an error related to the default llvm object path when CacheUse.Whole was selected. I'm not really sure why this didn't manifest when the binary is also emitted. This was fixed by improving the path handling related to flush() and emitLlvmObject(). In general, this commit also improves some of the path handling throughout the compiler and standard library.

16 files changed, 231 insertions(+), 220 deletions(-)

lib/compiler/objcopy.zig+4-3
...@@ -201,9 +201,10 @@ fn cmdObjCopy(...@@ -201,9 +201,10 @@ fn cmdObjCopy(
201 if (seen_update) fatal("zig objcopy only supports 1 update for now", .{});201 if (seen_update) fatal("zig objcopy only supports 1 update for now", .{});
202 seen_update = true;202 seen_update = true;
203203
204 try server.serveEmitBinPath(output, .{204 // The build system already knows what the output is at this point, we
205 .flags = .{ .cache_hit = false },205 // only need to communicate that the process has finished.
206 });206 // Use the empty error bundle to indicate that the update is done.
207 try server.serveErrorBundle(std.zig.ErrorBundle.empty);
207 },208 },
208 else => fatal("unsupported message: {s}", .{@tagName(hdr.tag)}),209 else => fatal("unsupported message: {s}", .{@tagName(hdr.tag)}),
209 }210 }
lib/std/Build.zig+1-1
...@@ -2373,7 +2373,7 @@ pub const LazyPath = union(enum) {...@@ -2373,7 +2373,7 @@ pub const LazyPath = union(enum) {
2373 // basis for not traversing up too many directories.2373 // basis for not traversing up too many directories.
23742374
2375 var file_path: Cache.Path = .{2375 var file_path: Cache.Path = .{
2376 .root_dir = gen.file.step.owner.build_root,2376 .root_dir = Cache.Directory.cwd(),
2377 .sub_path = gen.file.path orelse {2377 .sub_path = gen.file.path orelse {
2378 std.debug.lockStdErr();2378 std.debug.lockStdErr();
2379 const stderr = std.io.getStdErr();2379 const stderr = std.io.getStdErr();
lib/std/Build/Cache.zig+7-2
...@@ -896,8 +896,8 @@ pub const Manifest = struct {...@@ -896,8 +896,8 @@ pub const Manifest = struct {
896 }896 }
897 }897 }
898898
899 /// Returns a hex encoded hash of the inputs.899 /// Returns a binary hash of the inputs.
900 pub fn final(self: *Manifest) HexDigest {900 pub fn finalBin(self: *Manifest) BinDigest {
901 assert(self.manifest_file != null);901 assert(self.manifest_file != null);
902902
903 // We don't close the manifest file yet, because we want to903 // We don't close the manifest file yet, because we want to
...@@ -908,7 +908,12 @@ pub const Manifest = struct {...@@ -908,7 +908,12 @@ pub const Manifest = struct {
908908
909 var bin_digest: BinDigest = undefined;909 var bin_digest: BinDigest = undefined;
910 self.hash.hasher.final(&bin_digest);910 self.hash.hasher.final(&bin_digest);
911 return bin_digest;
912 }
911913
914 /// Returns a hex encoded hash of the inputs.
915 pub fn final(self: *Manifest) HexDigest {
916 const bin_digest = self.finalBin();
912 return binToHex(bin_digest);917 return binToHex(bin_digest);
913 }918 }
914919
lib/std/Build/Fuzz.zig+11-7
...@@ -100,6 +100,15 @@ pub fn start(...@@ -100,6 +100,15 @@ pub fn start(
100}100}
101101
102fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {102fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
103 rebuildTestsWorkerRunFallible(run, ttyconf, parent_prog_node) catch |err| {
104 const compile = run.producer.?;
105 log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
106 compile.step.name, @errorName(err),
107 });
108 };
109}
110
111fn rebuildTestsWorkerRunFallible(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) !void {
103 const gpa = run.step.owner.allocator;112 const gpa = run.step.owner.allocator;
104 const stderr = std.io.getStdErr();113 const stderr = std.io.getStdErr();
105114
...@@ -121,14 +130,9 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog...@@ -121,14 +130,9 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog
121130
122 const rebuilt_bin_path = result catch |err| switch (err) {131 const rebuilt_bin_path = result catch |err| switch (err) {
123 error.MakeFailed => return,132 error.MakeFailed => return,
124 else => {133 else => |other| return other,
125 log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
126 compile.step.name, @errorName(err),
127 });
128 return;
129 },
130 };134 };
131 run.rebuilt_executable = rebuilt_bin_path;135 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
132}136}
133137
134fn fuzzWorkerRun(138fn fuzzWorkerRun(
lib/std/Build/Fuzz/WebServer.zig+30-14
...@@ -8,6 +8,8 @@ const Coverage = std.debug.Coverage;...@@ -8,6 +8,8 @@ const Coverage = std.debug.Coverage;
8const abi = std.Build.Fuzz.abi;8const abi = std.Build.Fuzz.abi;
9const log = std.log;9const log = std.log;
10const assert = std.debug.assert;10const assert = std.debug.assert;
11const Cache = std.Build.Cache;
12const Path = Cache.Path;
1113
12const WebServer = @This();14const WebServer = @This();
1315
...@@ -31,6 +33,10 @@ coverage_mutex: std.Thread.Mutex,...@@ -31,6 +33,10 @@ coverage_mutex: std.Thread.Mutex,
31/// Signaled when `coverage_files` changes.33/// Signaled when `coverage_files` changes.
32coverage_condition: std.Thread.Condition,34coverage_condition: std.Thread.Condition,
3335
36const fuzzer_bin_name = "fuzzer";
37const fuzzer_arch_os_abi = "wasm32-freestanding";
38const fuzzer_cpu_features = "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext";
39
34const CoverageMap = struct {40const CoverageMap = struct {
35 mapped_memory: []align(std.mem.page_size) const u8,41 mapped_memory: []align(std.mem.page_size) const u8,
36 coverage: Coverage,42 coverage: Coverage,
...@@ -181,9 +187,18 @@ fn serveWasm(...@@ -181,9 +187,18 @@ fn serveWasm(
181187
182 // Do the compilation every request, so that the user can edit the files188 // Do the compilation every request, so that the user can edit the files
183 // and see the changes without restarting the server.189 // and see the changes without restarting the server.
184 const wasm_binary_path = try buildWasmBinary(ws, arena, optimize_mode);190 const wasm_base_path = try buildWasmBinary(ws, arena, optimize_mode);
191 const bin_name = try std.zig.binNameAlloc(arena, .{
192 .root_name = fuzzer_bin_name,
193 .target = std.zig.system.resolveTargetQuery(std.Build.parseTargetQuery(.{
194 .arch_os_abi = fuzzer_arch_os_abi,
195 .cpu_features = fuzzer_cpu_features,
196 }) catch unreachable) catch unreachable,
197 .output_mode = .Exe,
198 });
185 // std.http.Server does not have a sendfile API yet.199 // std.http.Server does not have a sendfile API yet.
186 const file_contents = try std.fs.cwd().readFileAlloc(gpa, wasm_binary_path, 10 * 1024 * 1024);200 const bin_path = try wasm_base_path.join(arena, bin_name);
201 const file_contents = try bin_path.root_dir.handle.readFileAlloc(gpa, bin_path.sub_path, 10 * 1024 * 1024);
187 defer gpa.free(file_contents);202 defer gpa.free(file_contents);
188 try request.respond(file_contents, .{203 try request.respond(file_contents, .{
189 .extra_headers = &.{204 .extra_headers = &.{
...@@ -197,7 +212,7 @@ fn buildWasmBinary(...@@ -197,7 +212,7 @@ fn buildWasmBinary(
197 ws: *WebServer,212 ws: *WebServer,
198 arena: Allocator,213 arena: Allocator,
199 optimize_mode: std.builtin.OptimizeMode,214 optimize_mode: std.builtin.OptimizeMode,
200) ![]const u8 {215) !Path {
201 const gpa = ws.gpa;216 const gpa = ws.gpa;
202217
203 const main_src_path: Build.Cache.Path = .{218 const main_src_path: Build.Cache.Path = .{
...@@ -219,11 +234,11 @@ fn buildWasmBinary(...@@ -219,11 +234,11 @@ fn buildWasmBinary(
219 ws.zig_exe_path, "build-exe", //234 ws.zig_exe_path, "build-exe", //
220 "-fno-entry", //235 "-fno-entry", //
221 "-O", @tagName(optimize_mode), //236 "-O", @tagName(optimize_mode), //
222 "-target", "wasm32-freestanding", //237 "-target", fuzzer_arch_os_abi, //
223 "-mcpu", "baseline+atomics+bulk_memory+multivalue+mutable_globals+nontrapping_fptoint+reference_types+sign_ext", //238 "-mcpu", fuzzer_cpu_features, //
224 "--cache-dir", ws.global_cache_directory.path orelse ".", //239 "--cache-dir", ws.global_cache_directory.path orelse ".", //
225 "--global-cache-dir", ws.global_cache_directory.path orelse ".", //240 "--global-cache-dir", ws.global_cache_directory.path orelse ".", //
226 "--name", "fuzzer", //241 "--name", fuzzer_bin_name, //
227 "-rdynamic", //242 "-rdynamic", //
228 "-fsingle-threaded", //243 "-fsingle-threaded", //
229 "--dep", "Walk", //244 "--dep", "Walk", //
...@@ -251,7 +266,7 @@ fn buildWasmBinary(...@@ -251,7 +266,7 @@ fn buildWasmBinary(
251 try sendMessage(child.stdin.?, .exit);266 try sendMessage(child.stdin.?, .exit);
252267
253 const Header = std.zig.Server.Message.Header;268 const Header = std.zig.Server.Message.Header;
254 var result: ?[]const u8 = null;269 var result: ?Path = null;
255 var result_error_bundle = std.zig.ErrorBundle.empty;270 var result_error_bundle = std.zig.ErrorBundle.empty;
256271
257 const stdout = poller.fifo(.stdout);272 const stdout = poller.fifo(.stdout);
...@@ -288,13 +303,17 @@ fn buildWasmBinary(...@@ -288,13 +303,17 @@ fn buildWasmBinary(
288 .extra = extra_array,303 .extra = extra_array,
289 };304 };
290 },305 },
291 .emit_bin_path => {306 .emit_digest => {
292 const EbpHdr = std.zig.Server.Message.EmitBinPath;307 const EbpHdr = std.zig.Server.Message.EmitDigest;
293 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));308 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
294 if (!ebp_hdr.flags.cache_hit) {309 if (!ebp_hdr.flags.cache_hit) {
295 log.info("source changes detected; rebuilt wasm component", .{});310 log.info("source changes detected; rebuilt wasm component", .{});
296 }311 }
297 result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);312 const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
313 result = Path{
314 .root_dir = ws.global_cache_directory,
315 .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
316 };
298 },317 },
299 else => {}, // ignore other messages318 else => {}, // ignore other messages
300 }319 }
...@@ -568,10 +587,7 @@ fn prepareTables(...@@ -568,10 +587,7 @@ fn prepareTables(
568 };587 };
569 errdefer gop.value_ptr.coverage.deinit(gpa);588 errdefer gop.value_ptr.coverage.deinit(gpa);
570589
571 const rebuilt_exe_path: Build.Cache.Path = .{590 const rebuilt_exe_path = run_step.rebuilt_executable.?;
572 .root_dir = Build.Cache.Directory.cwd(),
573 .sub_path = run_step.rebuilt_executable.?,
574 };
575 var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {591 var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
576 log.err("step '{s}': failed to load debug information for '{}': {s}", .{592 log.err("step '{s}': failed to load debug information for '{}': {s}", .{
577 run_step.step.name, rebuilt_exe_path, @errorName(err),593 run_step.step.name, rebuilt_exe_path, @errorName(err),
lib/std/Build/Step.zig+12-11
...@@ -317,6 +317,8 @@ const Build = std.Build;...@@ -317,6 +317,8 @@ const Build = std.Build;
317const Allocator = std.mem.Allocator;317const Allocator = std.mem.Allocator;
318const assert = std.debug.assert;318const assert = std.debug.assert;
319const builtin = @import("builtin");319const builtin = @import("builtin");
320const Cache = Build.Cache;
321const Path = Cache.Path;
320322
321pub fn evalChildProcess(s: *Step, argv: []const []const u8) ![]u8 {323pub fn evalChildProcess(s: *Step, argv: []const []const u8) ![]u8 {
322 const run_result = try captureChildProcess(s, std.Progress.Node.none, argv);324 const run_result = try captureChildProcess(s, std.Progress.Node.none, argv);
...@@ -373,7 +375,7 @@ pub fn evalZigProcess(...@@ -373,7 +375,7 @@ pub fn evalZigProcess(
373 argv: []const []const u8,375 argv: []const []const u8,
374 prog_node: std.Progress.Node,376 prog_node: std.Progress.Node,
375 watch: bool,377 watch: bool,
376) !?[]const u8 {378) !?Path {
377 if (s.getZigProcess()) |zp| update: {379 if (s.getZigProcess()) |zp| update: {
378 assert(watch);380 assert(watch);
379 if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd);381 if (std.Progress.have_ipc) if (zp.progress_ipc_fd) |fd| prog_node.setIpcFd(fd);
...@@ -477,7 +479,7 @@ pub fn evalZigProcess(...@@ -477,7 +479,7 @@ pub fn evalZigProcess(
477 return result;479 return result;
478}480}
479481
480fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {482fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?Path {
481 const b = s.owner;483 const b = s.owner;
482 const arena = b.allocator;484 const arena = b.allocator;
483485
...@@ -487,7 +489,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {...@@ -487,7 +489,7 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
487 if (!watch) try sendMessage(zp.child.stdin.?, .exit);489 if (!watch) try sendMessage(zp.child.stdin.?, .exit);
488490
489 const Header = std.zig.Server.Message.Header;491 const Header = std.zig.Server.Message.Header;
490 var result: ?[]const u8 = null;492 var result: ?Path = null;
491493
492 const stdout = zp.poller.fifo(.stdout);494 const stdout = zp.poller.fifo(.stdout);
493495
...@@ -531,16 +533,15 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {...@@ -531,16 +533,15 @@ fn zigProcessUpdate(s: *Step, zp: *ZigProcess, watch: bool) !?[]const u8 {
531 break;533 break;
532 }534 }
533 },535 },
534 .emit_bin_path => {536 .emit_digest => {
535 const EbpHdr = std.zig.Server.Message.EmitBinPath;537 const EbpHdr = std.zig.Server.Message.EmitDigest;
536 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));538 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
537 s.result_cached = ebp_hdr.flags.cache_hit;539 s.result_cached = ebp_hdr.flags.cache_hit;
538 result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);540 const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
539 if (watch) {541 result = Path{
540 // This message indicates the end of the update.542 .root_dir = b.cache_root,
541 stdout.discard(body.len);543 .sub_path = try arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*)),
542 break;544 };
543 }
544 },545 },
545 .file_system_inputs => {546 .file_system_inputs => {
546 s.clearWatchInputs();547 s.clearWatchInputs();
lib/std/Build/Step/Compile.zig+14-15
...@@ -17,6 +17,7 @@ const Module = std.Build.Module;...@@ -17,6 +17,7 @@ const Module = std.Build.Module;
17const InstallDir = std.Build.InstallDir;17const InstallDir = std.Build.InstallDir;
18const GeneratedFile = std.Build.GeneratedFile;18const GeneratedFile = std.Build.GeneratedFile;
19const Compile = @This();19const Compile = @This();
20const Path = std.Build.Cache.Path;
2021
21pub const base_id: Step.Id = .compile;22pub const base_id: Step.Id = .compile;
2223
...@@ -1765,7 +1766,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -1765,7 +1766,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
17651766
1766 const zig_args = try getZigArgs(compile, false);1767 const zig_args = try getZigArgs(compile, false);
17671768
1768 const maybe_output_bin_path = step.evalZigProcess(1769 const maybe_output_dir = step.evalZigProcess(
1769 zig_args,1770 zig_args,
1770 options.progress_node,1771 options.progress_node,
1771 (b.graph.incremental == true) and options.watch,1772 (b.graph.incremental == true) and options.watch,
...@@ -1779,53 +1780,51 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -1779,53 +1780,51 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
1779 };1780 };
17801781
1781 // Update generated files1782 // Update generated files
1782 if (maybe_output_bin_path) |output_bin_path| {1783 if (maybe_output_dir) |output_dir| {
1783 const output_dir = fs.path.dirname(output_bin_path).?;
1784
1785 if (compile.emit_directory) |lp| {1784 if (compile.emit_directory) |lp| {
1786 lp.path = output_dir;1785 lp.path = b.fmt("{}", .{output_dir});
1787 }1786 }
17881787
1789 // -femit-bin[=path] (default) Output machine code1788 // -femit-bin[=path] (default) Output machine code
1790 if (compile.generated_bin) |bin| {1789 if (compile.generated_bin) |bin| {
1791 bin.path = b.pathJoin(&.{ output_dir, compile.out_filename });1790 bin.path = output_dir.joinString(b.allocator, compile.out_filename) catch @panic("OOM");
1792 }1791 }
17931792
1794 const sep = std.fs.path.sep;1793 const sep = std.fs.path.sep_str;
17951794
1796 // output PDB if someone requested it1795 // output PDB if someone requested it
1797 if (compile.generated_pdb) |pdb| {1796 if (compile.generated_pdb) |pdb| {
1798 pdb.path = b.fmt("{s}{c}{s}.pdb", .{ output_dir, sep, compile.name });1797 pdb.path = b.fmt("{}" ++ sep ++ "{s}.pdb", .{ output_dir, compile.name });
1799 }1798 }
18001799
1801 // -femit-implib[=path] (default) Produce an import .lib when building a Windows DLL1800 // -femit-implib[=path] (default) Produce an import .lib when building a Windows DLL
1802 if (compile.generated_implib) |implib| {1801 if (compile.generated_implib) |implib| {
1803 implib.path = b.fmt("{s}{c}{s}.lib", .{ output_dir, sep, compile.name });1802 implib.path = b.fmt("{}" ++ sep ++ "{s}.lib", .{ output_dir, compile.name });
1804 }1803 }
18051804
1806 // -femit-h[=path] Generate a C header file (.h)1805 // -femit-h[=path] Generate a C header file (.h)
1807 if (compile.generated_h) |lp| {1806 if (compile.generated_h) |lp| {
1808 lp.path = b.fmt("{s}{c}{s}.h", .{ output_dir, sep, compile.name });1807 lp.path = b.fmt("{}" ++ sep ++ "{s}.h", .{ output_dir, compile.name });
1809 }1808 }
18101809
1811 // -femit-docs[=path] Create a docs/ dir with html documentation1810 // -femit-docs[=path] Create a docs/ dir with html documentation
1812 if (compile.generated_docs) |generated_docs| {1811 if (compile.generated_docs) |generated_docs| {
1813 generated_docs.path = b.pathJoin(&.{ output_dir, "docs" });1812 generated_docs.path = output_dir.joinString(b.allocator, "docs") catch @panic("OOM");
1814 }1813 }
18151814
1816 // -femit-asm[=path] Output .s (assembly code)1815 // -femit-asm[=path] Output .s (assembly code)
1817 if (compile.generated_asm) |lp| {1816 if (compile.generated_asm) |lp| {
1818 lp.path = b.fmt("{s}{c}{s}.s", .{ output_dir, sep, compile.name });1817 lp.path = b.fmt("{}" ++ sep ++ "{s}.s", .{ output_dir, compile.name });
1819 }1818 }
18201819
1821 // -femit-llvm-ir[=path] Produce a .ll file with optimized LLVM IR (requires LLVM extensions)1820 // -femit-llvm-ir[=path] Produce a .ll file with optimized LLVM IR (requires LLVM extensions)
1822 if (compile.generated_llvm_ir) |lp| {1821 if (compile.generated_llvm_ir) |lp| {
1823 lp.path = b.fmt("{s}{c}{s}.ll", .{ output_dir, sep, compile.name });1822 lp.path = b.fmt("{}" ++ sep ++ "{s}.ll", .{ output_dir, compile.name });
1824 }1823 }
18251824
1826 // -femit-llvm-bc[=path] Produce an optimized LLVM module as a .bc file (requires LLVM extensions)1825 // -femit-llvm-bc[=path] Produce an optimized LLVM module as a .bc file (requires LLVM extensions)
1827 if (compile.generated_llvm_bc) |lp| {1826 if (compile.generated_llvm_bc) |lp| {
1828 lp.path = b.fmt("{s}{c}{s}.bc", .{ output_dir, sep, compile.name });1827 lp.path = b.fmt("{}" ++ sep ++ "{s}.bc", .{ output_dir, compile.name });
1829 }1828 }
1830 }1829 }
18311830
...@@ -1841,7 +1840,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -1841,7 +1840,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
1841 }1840 }
1842}1841}
18431842
1844pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 {1843pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) !Path {
1845 const gpa = c.step.owner.allocator;1844 const gpa = c.step.owner.allocator;
18461845
1847 c.step.result_error_msgs.clearRetainingCapacity();1846 c.step.result_error_msgs.clearRetainingCapacity();
lib/std/Build/Step/InstallArtifact.zig+25-24
...@@ -125,10 +125,10 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -125,10 +125,10 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
125125
126 if (install_artifact.dest_dir) |dest_dir| {126 if (install_artifact.dest_dir) |dest_dir| {
127 const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path);127 const full_dest_path = b.getInstallPath(dest_dir, install_artifact.dest_sub_path);
128 const full_src_path = install_artifact.emitted_bin.?.getPath2(b, step);128 const src_path = install_artifact.emitted_bin.?.getPath3(b, step);
129 const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_dest_path, .{}) catch |err| {129 const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_dest_path, .{}) catch |err| {
130 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{130 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
131 full_src_path, full_dest_path, @errorName(err),131 src_path.sub_path, full_dest_path, @errorName(err),
132 });132 });
133 };133 };
134 all_cached = all_cached and p == .fresh;134 all_cached = all_cached and p == .fresh;
...@@ -141,22 +141,22 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -141,22 +141,22 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
141 }141 }
142142
143 if (install_artifact.implib_dir) |implib_dir| {143 if (install_artifact.implib_dir) |implib_dir| {
144 const full_src_path = install_artifact.emitted_implib.?.getPath2(b, step);144 const src_path = install_artifact.emitted_implib.?.getPath3(b, step);
145 const full_implib_path = b.getInstallPath(implib_dir, fs.path.basename(full_src_path));145 const full_implib_path = b.getInstallPath(implib_dir, fs.path.basename(src_path.sub_path));
146 const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_implib_path, .{}) catch |err| {146 const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_implib_path, .{}) catch |err| {
147 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{147 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
148 full_src_path, full_implib_path, @errorName(err),148 src_path.sub_path, full_implib_path, @errorName(err),
149 });149 });
150 };150 };
151 all_cached = all_cached and p == .fresh;151 all_cached = all_cached and p == .fresh;
152 }152 }
153153
154 if (install_artifact.pdb_dir) |pdb_dir| {154 if (install_artifact.pdb_dir) |pdb_dir| {
155 const full_src_path = install_artifact.emitted_pdb.?.getPath2(b, step);155 const src_path = install_artifact.emitted_pdb.?.getPath3(b, step);
156 const full_pdb_path = b.getInstallPath(pdb_dir, fs.path.basename(full_src_path));156 const full_pdb_path = b.getInstallPath(pdb_dir, fs.path.basename(src_path.sub_path));
157 const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_pdb_path, .{}) catch |err| {157 const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_pdb_path, .{}) catch |err| {
158 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{158 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
159 full_src_path, full_pdb_path, @errorName(err),159 src_path.sub_path, full_pdb_path, @errorName(err),
160 });160 });
161 };161 };
162 all_cached = all_cached and p == .fresh;162 all_cached = all_cached and p == .fresh;
...@@ -164,11 +164,11 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -164,11 +164,11 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
164164
165 if (install_artifact.h_dir) |h_dir| {165 if (install_artifact.h_dir) |h_dir| {
166 if (install_artifact.emitted_h) |emitted_h| {166 if (install_artifact.emitted_h) |emitted_h| {
167 const full_src_path = emitted_h.getPath2(b, step);167 const src_path = emitted_h.getPath3(b, step);
168 const full_h_path = b.getInstallPath(h_dir, fs.path.basename(full_src_path));168 const full_h_path = b.getInstallPath(h_dir, fs.path.basename(src_path.sub_path));
169 const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_h_path, .{}) catch |err| {169 const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_h_path, .{}) catch |err| {
170 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{170 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
171 full_src_path, full_h_path, @errorName(err),171 src_path.sub_path, full_h_path, @errorName(err),
172 });172 });
173 };173 };
174 all_cached = all_cached and p == .fresh;174 all_cached = all_cached and p == .fresh;
...@@ -176,22 +176,22 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -176,22 +176,22 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
176176
177 for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) {177 for (install_artifact.artifact.installed_headers.items) |installation| switch (installation) {
178 .file => |file| {178 .file => |file| {
179 const full_src_path = file.source.getPath2(b, step);179 const src_path = file.source.getPath3(b, step);
180 const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path);180 const full_h_path = b.getInstallPath(h_dir, file.dest_rel_path);
181 const p = fs.Dir.updateFile(cwd, full_src_path, cwd, full_h_path, .{}) catch |err| {181 const p = fs.Dir.updateFile(src_path.root_dir.handle, src_path.sub_path, cwd, full_h_path, .{}) catch |err| {
182 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{182 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
183 full_src_path, full_h_path, @errorName(err),183 src_path.sub_path, full_h_path, @errorName(err),
184 });184 });
185 };185 };
186 all_cached = all_cached and p == .fresh;186 all_cached = all_cached and p == .fresh;
187 },187 },
188 .directory => |dir| {188 .directory => |dir| {
189 const full_src_dir_path = dir.source.getPath2(b, step);189 const src_dir_path = dir.source.getPath3(b, step);
190 const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path);190 const full_h_prefix = b.getInstallPath(h_dir, dir.dest_rel_path);
191191
192 var src_dir = b.build_root.handle.openDir(full_src_dir_path, .{ .iterate = true }) catch |err| {192 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.sub_path, .{ .iterate = true }) catch |err| {
193 return step.fail("unable to open source directory '{s}': {s}", .{193 return step.fail("unable to open source directory '{s}': {s}", .{
194 full_src_dir_path, @errorName(err),194 src_dir_path.sub_path, @errorName(err),
195 });195 });
196 };196 };
197 defer src_dir.close();197 defer src_dir.close();
...@@ -208,14 +208,15 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -208,14 +208,15 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
208 continue :next_entry;208 continue :next_entry;
209 }209 }
210 }210 }
211 const full_src_entry_path = b.pathJoin(&.{ full_src_dir_path, entry.path });211
212 const src_entry_path = src_dir_path.join(b.allocator, entry.path) catch @panic("OOM");
212 const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path });213 const full_dest_path = b.pathJoin(&.{ full_h_prefix, entry.path });
213 switch (entry.kind) {214 switch (entry.kind) {
214 .directory => try cwd.makePath(full_dest_path),215 .directory => try cwd.makePath(full_dest_path),
215 .file => {216 .file => {
216 const p = fs.Dir.updateFile(cwd, full_src_entry_path, cwd, full_dest_path, .{}) catch |err| {217 const p = fs.Dir.updateFile(src_entry_path.root_dir.handle, src_entry_path.sub_path, cwd, full_dest_path, .{}) catch |err| {
217 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{218 return step.fail("unable to update file from '{s}' to '{s}': {s}", .{
218 full_src_entry_path, full_dest_path, @errorName(err),219 src_entry_path.sub_path, full_dest_path, @errorName(err),
219 });220 });
220 };221 };
221 all_cached = all_cached and p == .fresh;222 all_cached = all_cached and p == .fresh;
lib/std/Build/Step/Run.zig+3-2
...@@ -7,6 +7,7 @@ const mem = std.mem;...@@ -7,6 +7,7 @@ const mem = std.mem;
7const process = std.process;7const process = std.process;
8const EnvMap = process.EnvMap;8const EnvMap = process.EnvMap;
9const assert = std.debug.assert;9const assert = std.debug.assert;
10const Path = Build.Cache.Path;
1011
11const Run = @This();12const Run = @This();
1213
...@@ -93,7 +94,7 @@ cached_test_metadata: ?CachedTestMetadata = null,...@@ -93,7 +94,7 @@ cached_test_metadata: ?CachedTestMetadata = null,
9394
94/// Populated during the fuzz phase if this run step corresponds to a unit test95/// Populated during the fuzz phase if this run step corresponds to a unit test
95/// executable that contains fuzz tests.96/// executable that contains fuzz tests.
96rebuilt_executable: ?[]const u8,97rebuilt_executable: ?Path,
9798
98/// If this Run step was produced by a Compile step, it is tracked here.99/// If this Run step was produced by a Compile step, it is tracked here.
99producer: ?*Step.Compile,100producer: ?*Step.Compile,
...@@ -872,7 +873,7 @@ pub fn rerunInFuzzMode(...@@ -872,7 +873,7 @@ pub fn rerunInFuzzMode(
872 .artifact => |pa| {873 .artifact => |pa| {
873 const artifact = pa.artifact;874 const artifact = pa.artifact;
874 const file_path = if (artifact == run.producer.?)875 const file_path = if (artifact == run.producer.?)
875 run.rebuilt_executable.?876 b.fmt("{}", .{run.rebuilt_executable.?})
876 else877 else
877 (artifact.installed_path orelse artifact.generated_bin.?.path.?);878 (artifact.installed_path orelse artifact.generated_bin.?.path.?);
878 try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));879 try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path }));
lib/std/Build/Step/TranslateC.zig+6-6
...@@ -153,12 +153,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -153,12 +153,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
153 try argv_list.append(c_macro);153 try argv_list.append(c_macro);
154 }154 }
155155
156 try argv_list.append(translate_c.source.getPath2(b, step));156 const c_source_path = translate_c.source.getPath2(b, step);
157 try argv_list.append(c_source_path);
157158
158 const output_path = try step.evalZigProcess(argv_list.items, prog_node, false);159 const output_dir = try step.evalZigProcess(argv_list.items, prog_node, false);
159160
160 translate_c.out_basename = fs.path.basename(output_path.?);161 const basename = std.fs.path.stem(std.fs.path.basename(c_source_path));
161 const output_dir = fs.path.dirname(output_path.?).?;162 translate_c.out_basename = b.fmt("{s}.zig", .{basename});
162163 translate_c.output_file.path = output_dir.?.joinString(b.allocator, translate_c.out_basename) catch @panic("OOM");
163 translate_c.output_file.path = b.pathJoin(&.{ output_dir, translate_c.out_basename });
164}164}
lib/std/zig/Server.zig+11-10
...@@ -14,8 +14,8 @@ pub const Message = struct {...@@ -14,8 +14,8 @@ pub const Message = struct {
14 zig_version,14 zig_version,
15 /// Body is an ErrorBundle.15 /// Body is an ErrorBundle.
16 error_bundle,16 error_bundle,
17 /// Body is a EmitBinPath.17 /// Body is a EmitDigest.
18 emit_bin_path,18 emit_digest,
19 /// Body is a TestMetadata19 /// Body is a TestMetadata
20 test_metadata,20 test_metadata,
21 /// Body is a TestResults21 /// Body is a TestResults
...@@ -82,8 +82,8 @@ pub const Message = struct {...@@ -82,8 +82,8 @@ pub const Message = struct {
82 };82 };
8383
84 /// Trailing:84 /// Trailing:
85 /// * file system path where the emitted binary can be found85 /// * the hex digest of the cache directory within the /o/ subdirectory.
86 pub const EmitBinPath = extern struct {86 pub const EmitDigest = extern struct {
87 flags: Flags,87 flags: Flags,
8888
89 pub const Flags = packed struct(u8) {89 pub const Flags = packed struct(u8) {
...@@ -196,17 +196,17 @@ pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void {...@@ -196,17 +196,17 @@ pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void {
196 }, &.{std.mem.asBytes(&msg_le)});196 }, &.{std.mem.asBytes(&msg_le)});
197}197}
198198
199pub fn serveEmitBinPath(199pub fn serveEmitDigest(
200 s: *Server,200 s: *Server,
201 fs_path: []const u8,201 digest: *const [Cache.bin_digest_len]u8,
202 header: OutMessage.EmitBinPath,202 header: OutMessage.EmitDigest,
203) !void {203) !void {
204 try s.serveMessage(.{204 try s.serveMessage(.{
205 .tag = .emit_bin_path,205 .tag = .emit_digest,
206 .bytes_len = @intCast(fs_path.len + @sizeOf(OutMessage.EmitBinPath)),206 .bytes_len = @intCast(digest.len + @sizeOf(OutMessage.EmitDigest)),
207 }, &.{207 }, &.{
208 std.mem.asBytes(&header),208 std.mem.asBytes(&header),
209 fs_path,209 digest,
210 });210 });
211}211}
212212
...@@ -328,3 +328,4 @@ const Allocator = std.mem.Allocator;...@@ -328,3 +328,4 @@ const Allocator = std.mem.Allocator;
328const assert = std.debug.assert;328const assert = std.debug.assert;
329const native_endian = builtin.target.cpu.arch.endian();329const native_endian = builtin.target.cpu.arch.endian();
330const need_bswap = native_endian != .little;330const need_bswap = native_endian != .little;
331const Cache = std.Build.Cache;
src/Compilation.zig+49-46
...@@ -39,6 +39,8 @@ const Air = @import("Air.zig");...@@ -39,6 +39,8 @@ const Air = @import("Air.zig");
39const Builtin = @import("Builtin.zig");39const Builtin = @import("Builtin.zig");
40const LlvmObject = @import("codegen/llvm.zig").Object;40const LlvmObject = @import("codegen/llvm.zig").Object;
41const dev = @import("dev.zig");41const dev = @import("dev.zig");
42pub const Directory = Cache.Directory;
43const Path = Cache.Path;
4244
43pub const Config = @import("Compilation/Config.zig");45pub const Config = @import("Compilation/Config.zig");
4446
...@@ -269,6 +271,11 @@ llvm_opt_bisect_limit: c_int,...@@ -269,6 +271,11 @@ llvm_opt_bisect_limit: c_int,
269271
270file_system_inputs: ?*std.ArrayListUnmanaged(u8),272file_system_inputs: ?*std.ArrayListUnmanaged(u8),
271273
274/// This is the digest of the cache for the current compilation.
275/// This digest will be known after update() is called.
276digest: ?[Cache.bin_digest_len]u8 = null,
277
278/// TODO(robin): Remove because it is the same as Cache.Path
272pub const Emit = struct {279pub const Emit = struct {
273 /// Where the output will go.280 /// Where the output will go.
274 directory: Directory,281 directory: Directory,
...@@ -868,8 +875,6 @@ pub const LldError = struct {...@@ -868,8 +875,6 @@ pub const LldError = struct {
868 }875 }
869};876};
870877
871pub const Directory = Cache.Directory;
872
873pub const EmitLoc = struct {878pub const EmitLoc = struct {
874 /// If this is `null` it means the file will be output to the cache directory.879 /// If this is `null` it means the file will be output to the cache directory.
875 /// When provided, both the open file handle and the path name must outlive the `Compilation`.880 /// When provided, both the open file handle and the path name must outlive the `Compilation`.
...@@ -1672,7 +1677,9 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -1672,7 +1677,9 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
1672 // In the case of incremental cache mode, this `artifact_directory`1677 // In the case of incremental cache mode, this `artifact_directory`
1673 // is computed based on a hash of non-linker inputs, and it is where all1678 // is computed based on a hash of non-linker inputs, and it is where all
1674 // build artifacts are stored (even while in-progress).1679 // build artifacts are stored (even while in-progress).
1680 comp.digest = hash.peekBin();
1675 const digest = hash.final();1681 const digest = hash.final();
1682
1676 const artifact_sub_dir = "o" ++ std.fs.path.sep_str ++ digest;1683 const artifact_sub_dir = "o" ++ std.fs.path.sep_str ++ digest;
1677 var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{});1684 var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{});
1678 errdefer artifact_dir.close();1685 errdefer artifact_dir.close();
...@@ -2121,9 +2128,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2121,9 +2128,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21212128
2122 comp.last_update_was_cache_hit = true;2129 comp.last_update_was_cache_hit = true;
2123 log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name});2130 log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name});
2124 const digest = man.final();2131 const bin_digest = man.finalBin();
2132 const hex_digest = Cache.binToHex(bin_digest);
21252133
2126 comp.wholeCacheModeSetBinFilePath(whole, &digest);2134 comp.digest = bin_digest;
2135 comp.wholeCacheModeSetBinFilePath(whole, &hex_digest);
21272136
2128 assert(whole.lock == null);2137 assert(whole.lock == null);
2129 whole.lock = man.toOwnedLock();2138 whole.lock = man.toOwnedLock();
...@@ -2329,7 +2338,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2329,7 +2338,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2329 try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);2338 try man.populateOtherManifest(pwc.manifest, pwc.prefix_map);
2330 }2339 }
23312340
2332 const digest = man.final();2341 const bin_digest = man.finalBin();
2342 const hex_digest = Cache.binToHex(bin_digest);
23332343
2334 // Rename the temporary directory into place.2344 // Rename the temporary directory into place.
2335 // Close tmp dir and link.File to avoid open handle during rename.2345 // Close tmp dir and link.File to avoid open handle during rename.
...@@ -2341,7 +2351,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2341,7 +2351,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
23412351
2342 const s = std.fs.path.sep_str;2352 const s = std.fs.path.sep_str;
2343 const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(tmp_dir_rand_int);2353 const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(tmp_dir_rand_int);
2344 const o_sub_path = "o" ++ s ++ digest;2354 const o_sub_path = "o" ++ s ++ hex_digest;
23452355
2346 // Work around windows `AccessDenied` if any files within this2356 // Work around windows `AccessDenied` if any files within this
2347 // directory are open by closing and reopening the file handles.2357 // directory are open by closing and reopening the file handles.
...@@ -2376,7 +2386,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2376,7 +2386,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2376 },2386 },
2377 );2387 );
2378 };2388 };
2379 comp.wholeCacheModeSetBinFilePath(whole, &digest);2389 comp.digest = bin_digest;
2390 comp.wholeCacheModeSetBinFilePath(whole, &hex_digest);
23802391
2381 // The linker flush functions need to know the final output path2392 // The linker flush functions need to know the final output path
2382 // for debug info purposes because executable debug info contains2393 // for debug info purposes because executable debug info contains
...@@ -2393,9 +2404,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2393,9 +2404,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2393 }2404 }
2394 }2405 }
23952406
2396 try flush(comp, arena, .main, main_progress_node);2407 try flush(comp, arena, .{
23972408 .root_dir = comp.local_cache_directory,
2398 if (try comp.totalErrorCount() != 0) return;2409 .sub_path = o_sub_path,
2410 }, .main, main_progress_node);
23992411
2400 // Failure here only means an unnecessary cache miss.2412 // Failure here only means an unnecessary cache miss.
2401 man.writeManifest() catch |err| {2413 man.writeManifest() catch |err| {
...@@ -2410,8 +2422,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {...@@ -2410,8 +2422,10 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
2410 assert(whole.lock == null);2422 assert(whole.lock == null);
2411 whole.lock = man.toOwnedLock();2423 whole.lock = man.toOwnedLock();
2412 },2424 },
2413 .incremental => {2425 .incremental => |incremental| {
2414 try flush(comp, arena, .main, main_progress_node);2426 try flush(comp, arena, .{
2427 .root_dir = incremental.artifact_directory,
2428 }, .main, main_progress_node);
2415 },2429 },
2416 }2430 }
2417}2431}
...@@ -2440,7 +2454,13 @@ pub fn appendFileSystemInput(...@@ -2440,7 +2454,13 @@ pub fn appendFileSystemInput(
2440 std.debug.panic("missing prefix directory: {}, {s}", .{ root, sub_file_path });2454 std.debug.panic("missing prefix directory: {}, {s}", .{ root, sub_file_path });
2441}2455}
24422456
2443fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: std.Progress.Node) !void {2457fn flush(
2458 comp: *Compilation,
2459 arena: Allocator,
2460 default_artifact_directory: Path,
2461 tid: Zcu.PerThread.Id,
2462 prog_node: std.Progress.Node,
2463) !void {
2444 if (comp.bin_file) |lf| {2464 if (comp.bin_file) |lf| {
2445 // This is needed before reading the error flags.2465 // This is needed before reading the error flags.
2446 lf.flush(arena, tid, prog_node) catch |err| switch (err) {2466 lf.flush(arena, tid, prog_node) catch |err| switch (err) {
...@@ -2454,17 +2474,7 @@ fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node:...@@ -2454,17 +2474,7 @@ fn flush(comp: *Compilation, arena: Allocator, tid: Zcu.PerThread.Id, prog_node:
2454 try link.File.C.flushEmitH(zcu);2474 try link.File.C.flushEmitH(zcu);
24552475
2456 if (zcu.llvm_object) |llvm_object| {2476 if (zcu.llvm_object) |llvm_object| {
2457 const default_emit = switch (comp.cache_use) {2477 try emitLlvmObject(comp, arena, default_artifact_directory, null, llvm_object, prog_node);
2458 .whole => |whole| .{
2459 .directory = whole.tmp_artifact_directory.?,
2460 .sub_path = "dummy",
2461 },
2462 .incremental => |incremental| .{
2463 .directory = incremental.artifact_directory,
2464 .sub_path = "dummy",
2465 },
2466 };
2467 try emitLlvmObject(comp, arena, default_emit, null, llvm_object, prog_node);
2468 }2478 }
2469 }2479 }
2470}2480}
...@@ -2745,7 +2755,7 @@ fn emitOthers(comp: *Compilation) void {...@@ -2745,7 +2755,7 @@ fn emitOthers(comp: *Compilation) void {
2745pub fn emitLlvmObject(2755pub fn emitLlvmObject(
2746 comp: *Compilation,2756 comp: *Compilation,
2747 arena: Allocator,2757 arena: Allocator,
2748 default_emit: Emit,2758 default_artifact_directory: Path,
2749 bin_emit_loc: ?EmitLoc,2759 bin_emit_loc: ?EmitLoc,
2750 llvm_object: LlvmObject.Ptr,2760 llvm_object: LlvmObject.Ptr,
2751 prog_node: std.Progress.Node,2761 prog_node: std.Progress.Node,
...@@ -2756,10 +2766,10 @@ pub fn emitLlvmObject(...@@ -2756,10 +2766,10 @@ pub fn emitLlvmObject(
2756 try llvm_object.emit(.{2766 try llvm_object.emit(.{
2757 .pre_ir_path = comp.verbose_llvm_ir,2767 .pre_ir_path = comp.verbose_llvm_ir,
2758 .pre_bc_path = comp.verbose_llvm_bc,2768 .pre_bc_path = comp.verbose_llvm_bc,
2759 .bin_path = try resolveEmitLoc(arena, default_emit, bin_emit_loc),2769 .bin_path = try resolveEmitLoc(arena, default_artifact_directory, bin_emit_loc),
2760 .asm_path = try resolveEmitLoc(arena, default_emit, comp.emit_asm),2770 .asm_path = try resolveEmitLoc(arena, default_artifact_directory, comp.emit_asm),
2761 .post_ir_path = try resolveEmitLoc(arena, default_emit, comp.emit_llvm_ir),2771 .post_ir_path = try resolveEmitLoc(arena, default_artifact_directory, comp.emit_llvm_ir),
2762 .post_bc_path = try resolveEmitLoc(arena, default_emit, comp.emit_llvm_bc),2772 .post_bc_path = try resolveEmitLoc(arena, default_artifact_directory, comp.emit_llvm_bc),
27632773
2764 .is_debug = comp.root_mod.optimize_mode == .Debug,2774 .is_debug = comp.root_mod.optimize_mode == .Debug,
2765 .is_small = comp.root_mod.optimize_mode == .ReleaseSmall,2775 .is_small = comp.root_mod.optimize_mode == .ReleaseSmall,
...@@ -2772,14 +2782,14 @@ pub fn emitLlvmObject(...@@ -2772,14 +2782,14 @@ pub fn emitLlvmObject(
27722782
2773fn resolveEmitLoc(2783fn resolveEmitLoc(
2774 arena: Allocator,2784 arena: Allocator,
2775 default_emit: Emit,2785 default_artifact_directory: Path,
2776 opt_loc: ?EmitLoc,2786 opt_loc: ?EmitLoc,
2777) Allocator.Error!?[*:0]const u8 {2787) Allocator.Error!?[*:0]const u8 {
2778 const loc = opt_loc orelse return null;2788 const loc = opt_loc orelse return null;
2779 const slice = if (loc.directory) |directory|2789 const slice = if (loc.directory) |directory|
2780 try directory.joinZ(arena, &.{loc.basename})2790 try directory.joinZ(arena, &.{loc.basename})
2781 else2791 else
2782 try default_emit.basenamePath(arena, loc.basename);2792 try default_artifact_directory.joinStringZ(arena, loc.basename);
2783 return slice.ptr;2793 return slice.ptr;
2784}2794}
27852795
...@@ -4403,7 +4413,7 @@ pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest...@@ -4403,7 +4413,7 @@ pub fn obtainWin32ResourceCacheManifest(comp: *const Compilation) Cache.Manifest
4403}4413}
44044414
4405pub const CImportResult = struct {4415pub const CImportResult = struct {
4406 out_zig_path: []u8,4416 digest: [Cache.bin_digest_len]u8,
4407 cache_hit: bool,4417 cache_hit: bool,
4408 errors: std.zig.ErrorBundle,4418 errors: std.zig.ErrorBundle,
44094419
...@@ -4413,8 +4423,6 @@ pub const CImportResult = struct {...@@ -4413,8 +4423,6 @@ pub const CImportResult = struct {
4413};4423};
44144424
4415/// Caller owns returned memory.4425/// Caller owns returned memory.
4416/// This API is currently coupled pretty tightly to stage1's needs; it will need to be reworked
4417/// a bit when we want to start using it from self-hosted.
4418pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module) !CImportResult {4426pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module) !CImportResult {
4419 dev.check(.translate_c_command);4427 dev.check(.translate_c_command);
44204428
...@@ -4503,7 +4511,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module...@@ -4503,7 +4511,7 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
4503 error.OutOfMemory => return error.OutOfMemory,4511 error.OutOfMemory => return error.OutOfMemory,
4504 error.SemanticAnalyzeFail => {4512 error.SemanticAnalyzeFail => {
4505 return CImportResult{4513 return CImportResult{
4506 .out_zig_path = "",4514 .digest = undefined,
4507 .cache_hit = actual_hit,4515 .cache_hit = actual_hit,
4508 .errors = errors,4516 .errors = errors,
4509 };4517 };
...@@ -4528,8 +4536,9 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module...@@ -4528,8 +4536,9 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
4528 .incremental => {},4536 .incremental => {},
4529 }4537 }
45304538
4531 const digest = man.final();4539 const bin_digest = man.finalBin();
4532 const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest });4540 const hex_digest = Cache.binToHex(bin_digest);
4541 const o_sub_path = "o" ++ std.fs.path.sep_str ++ hex_digest;
4533 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});4542 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
4534 defer o_dir.close();4543 defer o_dir.close();
45354544
...@@ -4541,8 +4550,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module...@@ -4541,8 +4550,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
45414550
4542 try out_zig_file.writeAll(formatted);4551 try out_zig_file.writeAll(formatted);
45434552
4544 break :digest digest;4553 break :digest bin_digest;
4545 } else man.final();4554 } else man.finalBin();
45464555
4547 if (man.have_exclusive_lock) {4556 if (man.have_exclusive_lock) {
4548 // Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is4557 // Write the updated manifest. This is a no-op if the manifest is not dirty. Note that it is
...@@ -4554,14 +4563,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module...@@ -4554,14 +4563,8 @@ pub fn cImport(comp: *Compilation, c_src: []const u8, owner_mod: *Package.Module
4554 };4563 };
4555 }4564 }
45564565
4557 const out_zig_path = try comp.local_cache_directory.join(comp.arena, &.{
4558 "o", &digest, cimport_zig_basename,
4559 });
4560 if (comp.verbose_cimport) {
4561 log.info("C import output: {s}", .{out_zig_path});
4562 }
4563 return CImportResult{4566 return CImportResult{
4564 .out_zig_path = out_zig_path,4567 .digest = digest,
4565 .cache_hit = actual_hit,4568 .cache_hit = actual_hit,
4566 .errors = std.zig.ErrorBundle.empty,4569 .errors = std.zig.ErrorBundle.empty,
4567 };4570 };
src/Sema.zig+7-4
...@@ -183,6 +183,7 @@ const InternPool = @import("InternPool.zig");...@@ -183,6 +183,7 @@ const InternPool = @import("InternPool.zig");
183const Alignment = InternPool.Alignment;183const Alignment = InternPool.Alignment;
184const AnalUnit = InternPool.AnalUnit;184const AnalUnit = InternPool.AnalUnit;
185const ComptimeAllocIndex = InternPool.ComptimeAllocIndex;185const ComptimeAllocIndex = InternPool.ComptimeAllocIndex;
186const Cache = std.Build.Cache;
186187
187pub const default_branch_quota = 1000;188pub const default_branch_quota = 1000;
188pub const default_reference_trace_len = 2;189pub const default_reference_trace_len = 2;
...@@ -5871,16 +5872,18 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr...@@ -5871,16 +5872,18 @@ fn zirCImport(sema: *Sema, parent_block: *Block, inst: Zir.Inst.Index) CompileEr
5871 return sema.failWithOwnedErrorMsg(&child_block, msg);5872 return sema.failWithOwnedErrorMsg(&child_block, msg);
5872 }5873 }
5873 const parent_mod = parent_block.ownerModule();5874 const parent_mod = parent_block.ownerModule();
5875 const digest = Cache.binToHex(c_import_res.digest);
5876 const c_import_zig_path = try comp.arena.dupe(u8, "o" ++ std.fs.path.sep_str ++ digest);
5874 const c_import_mod = Package.Module.create(comp.arena, .{5877 const c_import_mod = Package.Module.create(comp.arena, .{
5875 .global_cache_directory = comp.global_cache_directory,5878 .global_cache_directory = comp.global_cache_directory,
5876 .paths = .{5879 .paths = .{
5877 .root = .{5880 .root = .{
5878 .root_dir = Compilation.Directory.cwd(),5881 .root_dir = comp.local_cache_directory,
5879 .sub_path = std.fs.path.dirname(c_import_res.out_zig_path) orelse "",5882 .sub_path = c_import_zig_path,
5880 },5883 },
5881 .root_src_path = std.fs.path.basename(c_import_res.out_zig_path),5884 .root_src_path = "cimport.zig",
5882 },5885 },
5883 .fully_qualified_name = c_import_res.out_zig_path,5886 .fully_qualified_name = c_import_zig_path,
5884 .cc_argv = parent_mod.cc_argv,5887 .cc_argv = parent_mod.cc_argv,
5885 .inherited = .{},5888 .inherited = .{},
5886 .global = comp.config,5889 .global = comp.config,
src/link.zig+4-1
...@@ -1029,7 +1029,10 @@ pub const File = struct {...@@ -1029,7 +1029,10 @@ pub const File = struct {
1029 llvm_object: LlvmObject.Ptr,1029 llvm_object: LlvmObject.Ptr,
1030 prog_node: std.Progress.Node,1030 prog_node: std.Progress.Node,
1031 ) !void {1031 ) !void {
1032 return base.comp.emitLlvmObject(arena, base.emit, .{1032 return base.comp.emitLlvmObject(arena, .{
1033 .root_dir = base.emit.directory,
1034 .sub_path = std.fs.path.dirname(base.emit.sub_path) orelse "",
1035 }, .{
1033 .directory = null,1036 .directory = null,
1034 .basename = base.zcu_object_sub_path.?,1037 .basename = base.zcu_object_sub_path.?,
1035 }, llvm_object, prog_node);1038 }, llvm_object, prog_node);
src/main.zig+14-64
...@@ -4142,7 +4142,7 @@ fn serve(...@@ -4142,7 +4142,7 @@ fn serve(
4142 if (output.errors.errorMessageCount() != 0) {4142 if (output.errors.errorMessageCount() != 0) {
4143 try server.serveErrorBundle(output.errors);4143 try server.serveErrorBundle(output.errors);
4144 } else {4144 } else {
4145 try server.serveEmitBinPath(output.out_zig_path, .{4145 try server.serveEmitDigest(&output.digest, .{
4146 .flags = .{ .cache_hit = output.cache_hit },4146 .flags = .{ .cache_hit = output.cache_hit },
4147 });4147 });
4148 }4148 }
...@@ -4229,62 +4229,10 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {...@@ -4229,62 +4229,10 @@ fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
4229 return;4229 return;
4230 }4230 }
42314231
4232 // This logic is counter-intuitive because the protocol accounts for each4232 if (comp.digest) |digest| {
4233 // emitted artifact possibly being in a different location, which correctly4233 try s.serveEmitDigest(&digest, .{
4234 // matches the behavior of the compiler, however, the build system
4235 // currently always passes flags that makes all build artifacts output to
4236 // the same local cache directory, and relies on them all being in the same
4237 // directory.
4238 //
4239 // So, until the build system and protocol are changed to reflect this,
4240 // this logic must ensure that emit_bin_path is emitted for at least one
4241 // thing, if there are any artifacts.
4242
4243 switch (comp.cache_use) {
4244 .incremental => if (comp.bin_file) |lf| {
4245 const full_path = try lf.emit.directory.join(gpa, &.{lf.emit.sub_path});
4246 defer gpa.free(full_path);
4247 try s.serveEmitBinPath(full_path, .{
4248 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
4249 });
4250 return;
4251 },
4252 .whole => |whole| if (whole.bin_sub_path) |sub_path| {
4253 const full_path = try comp.local_cache_directory.join(gpa, &.{sub_path});
4254 defer gpa.free(full_path);
4255 try s.serveEmitBinPath(full_path, .{
4256 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
4257 });
4258 return;
4259 },
4260 }
4261
4262 for ([_]?Compilation.Emit{
4263 comp.docs_emit,
4264 comp.implib_emit,
4265 }) |opt_emit| {
4266 const emit = opt_emit orelse continue;
4267 const full_path = try emit.directory.join(gpa, &.{emit.sub_path});
4268 defer gpa.free(full_path);
4269 try s.serveEmitBinPath(full_path, .{
4270 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },4234 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
4271 });4235 });
4272 return;
4273 }
4274
4275 for ([_]?Compilation.EmitLoc{
4276 comp.emit_asm,
4277 comp.emit_llvm_ir,
4278 comp.emit_llvm_bc,
4279 }) |opt_emit_loc| {
4280 const emit_loc = opt_emit_loc orelse continue;
4281 const directory = emit_loc.directory orelse continue;
4282 const full_path = try directory.join(gpa, &.{emit_loc.basename});
4283 defer gpa.free(full_path);
4284 try s.serveEmitBinPath(full_path, .{
4285 .flags = .{ .cache_hit = comp.last_update_was_cache_hit },
4286 });
4287 return;
4288 }4236 }
42894237
4290 // Serve empty error bundle to indicate the update is done.4238 // Serve empty error bundle to indicate the update is done.
...@@ -4539,9 +4487,11 @@ fn cmdTranslateC(...@@ -4539,9 +4487,11 @@ fn cmdTranslateC(
4539 };4487 };
45404488
4541 if (fancy_output) |p| p.cache_hit = true;4489 if (fancy_output) |p| p.cache_hit = true;
4542 const digest = if (try man.hit()) digest: {4490 const bin_digest, const hex_digest = if (try man.hit()) digest: {
4543 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);4491 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
4544 break :digest man.final();4492 const bin_digest = man.finalBin();
4493 const hex_digest = Cache.binToHex(bin_digest);
4494 break :digest .{ bin_digest, hex_digest };
4545 } else digest: {4495 } else digest: {
4546 if (fancy_output) |p| p.cache_hit = false;4496 if (fancy_output) |p| p.cache_hit = false;
4547 var argv = std.ArrayList([]const u8).init(arena);4497 var argv = std.ArrayList([]const u8).init(arena);
...@@ -4639,8 +4589,10 @@ fn cmdTranslateC(...@@ -4639,8 +4589,10 @@ fn cmdTranslateC(
4639 };4589 };
4640 }4590 }
46414591
4642 const digest = man.final();4592 const bin_digest = man.finalBin();
4643 const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest });4593 const hex_digest = Cache.binToHex(bin_digest);
4594
4595 const o_sub_path = try fs.path.join(arena, &[_][]const u8{ "o", &hex_digest });
46444596
4645 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});4597 var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{});
4646 defer o_dir.close();4598 defer o_dir.close();
...@@ -4656,16 +4608,14 @@ fn cmdTranslateC(...@@ -4656,16 +4608,14 @@ fn cmdTranslateC(
46564608
4657 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);4609 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
46584610
4659 break :digest digest;4611 break :digest .{ bin_digest, hex_digest };
4660 };4612 };
46614613
4662 if (fancy_output) |p| {4614 if (fancy_output) |p| {
4663 p.out_zig_path = try comp.local_cache_directory.join(comp.gpa, &[_][]const u8{4615 p.digest = bin_digest;
4664 "o", &digest, translated_zig_basename,
4665 });
4666 p.errors = std.zig.ErrorBundle.empty;4616 p.errors = std.zig.ErrorBundle.empty;
4667 } else {4617 } else {
4668 const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &digest, translated_zig_basename });4618 const out_zig_path = try fs.path.join(arena, &[_][]const u8{ "o", &hex_digest, translated_zig_basename });
4669 const zig_file = comp.local_cache_directory.handle.openFile(out_zig_path, .{}) catch |err| {4619 const zig_file = comp.local_cache_directory.handle.openFile(out_zig_path, .{}) catch |err| {
4670 const path = comp.local_cache_directory.path orelse ".";4620 const path = comp.local_cache_directory.path orelse ".";
4671 fatal("unable to open cached translated zig file '{s}{s}{s}': {s}", .{ path, fs.path.sep_str, out_zig_path, @errorName(err) });4621 fatal("unable to open cached translated zig file '{s}{s}{s}': {s}", .{ path, fs.path.sep_str, out_zig_path, @errorName(err) });
tools/incr-check.zig+33-10
...@@ -1,6 +1,7 @@...@@ -1,6 +1,7 @@
1const std = @import("std");1const std = @import("std");
2const fatal = std.process.fatal;2const fatal = std.process.fatal;
3const Allocator = std.mem.Allocator;3const Allocator = std.mem.Allocator;
4const Cache = std.Build.Cache;
45
5const usage = "usage: incr-check <zig binary path> <input file> [--zig-lib-dir lib] [--debug-zcu] [--emit none|bin|c] [--zig-cc-binary /path/to/zig]";6const usage = "usage: incr-check <zig binary path> <input file> [--zig-lib-dir lib] [--debug-zcu] [--emit none|bin|c] [--zig-cc-binary /path/to/zig]";
67
...@@ -233,30 +234,52 @@ const Eval = struct {...@@ -233,30 +234,52 @@ const Eval = struct {
233 fatal("error_bundle included unexpected stderr:\n{s}", .{stderr_data});234 fatal("error_bundle included unexpected stderr:\n{s}", .{stderr_data});
234 }235 }
235 }236 }
236 if (result_error_bundle.errorMessageCount() == 0) {237 if (result_error_bundle.errorMessageCount() != 0) {
237 // Empty bundle indicates successful update in a `-fno-emit-bin` build.
238 try eval.checkSuccessOutcome(update, null, prog_node);
239 } else {
240 try eval.checkErrorOutcome(update, result_error_bundle);238 try eval.checkErrorOutcome(update, result_error_bundle);
241 }239 }
242 // This message indicates the end of the update.240 // This message indicates the end of the update.
243 stdout.discard(body.len);241 stdout.discard(body.len);
244 return;242 return;
245 },243 },
246 .emit_bin_path => {244 .emit_digest => {
247 const EbpHdr = std.zig.Server.Message.EmitBinPath;245 const EbpHdr = std.zig.Server.Message.EmitDigest;
248 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));246 const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body));
249 _ = ebp_hdr;247 _ = ebp_hdr;
250 const result_binary = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
251 if (stderr.readableLength() > 0) {248 if (stderr.readableLength() > 0) {
252 const stderr_data = try stderr.toOwnedSlice();249 const stderr_data = try stderr.toOwnedSlice();
253 if (eval.allow_stderr) {250 if (eval.allow_stderr) {
254 std.log.info("emit_bin_path included stderr:\n{s}", .{stderr_data});251 std.log.info("emit_digest included stderr:\n{s}", .{stderr_data});
255 } else {252 } else {
256 fatal("emit_bin_path included unexpected stderr:\n{s}", .{stderr_data});253 fatal("emit_digest included unexpected stderr:\n{s}", .{stderr_data});
257 }254 }
258 }255 }
259 try eval.checkSuccessOutcome(update, result_binary, prog_node);256
257 if (eval.emit == .none) {
258 try eval.checkSuccessOutcome(update, null, prog_node);
259 // This message indicates the end of the update.
260 stdout.discard(body.len);
261 return;
262 }
263
264 const digest = body[@sizeOf(EbpHdr)..][0..Cache.bin_digest_len];
265 const result_dir = ".local-cache" ++ std.fs.path.sep_str ++ "o" ++ std.fs.path.sep_str ++ Cache.binToHex(digest.*);
266
267 const name = std.fs.path.stem(std.fs.path.basename(eval.case.root_source_file));
268 const bin_name = try std.zig.binNameAlloc(arena, .{
269 .root_name = name,
270 .target = try std.zig.system.resolveTargetQuery(try std.Build.parseTargetQuery(.{
271 .arch_os_abi = eval.case.target_query,
272 .object_format = switch (eval.emit) {
273 .none => unreachable,
274 .bin => null,
275 .c => "c",
276 },
277 })),
278 .output_mode = .Exe,
279 });
280 const bin_path = try std.fs.path.join(arena, &.{ result_dir, bin_name });
281
282 try eval.checkSuccessOutcome(update, bin_path, prog_node);
260 // This message indicates the end of the update.283 // This message indicates the end of the update.
261 stdout.discard(body.len);284 stdout.discard(body.len);
262 return;285 return;