authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-11 16:26:04-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:14:08-07:00
loga3c20dffaed77727494d34f7b4b03c0d10771270
tree20863dac0a78dd8739109f26e53b9818469a71b6
parentfd4d366009e92c79137ee681334f216bbfc9b5f5

integrate Compile steps with file watching

Updates the build runner to unconditionally require a zig lib directory parameter. This parameter is needed in order to correctly understand file system inputs from zig compiler subprocesses, since they will refer to "the zig lib directory", and the build runner needs to place file system watches on directories in there. The build runner's fanotify file watching implementation now accounts for when two or more Cache.Path instances compare unequal but ultimately refer to the same directory in the file system. Breaking change: std.Build no longer has a zig_lib_dir field. Instead, there is the Graph zig_lib_directory field, and individual Compile steps can still have their zig lib directories overridden. I think this is unlikely to break anyone's build in practice. The compiler now sends a "file_system_inputs" message to the build runner which shares the full set of files that were added to the cache system with the build system, so that the build runner can watch properly and redo the Compile step. This is implemented for whole cache mode but not yet for incremental cache mode.

8 files changed, 152 insertions(+), 44 deletions(-)

build.zig+4-5
......@@ -1261,7 +1261,9 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath {
12611261 });
12621262
12631263 var dir = b.build_root.handle.openDir("doc/langref", .{ .iterate = true }) catch |err| {
1264 std.debug.panic("unable to open 'doc/langref' directory: {s}", .{@errorName(err)});
1264 std.debug.panic("unable to open '{}doc/langref' directory: {s}", .{
1265 b.build_root, @errorName(err),
1266 });
12651267 };
12661268 defer dir.close();
12671269
......@@ -1280,10 +1282,7 @@ fn generateLangRef(b: *std.Build) std.Build.LazyPath {
12801282 // in a temporary directory
12811283 "--cache-root", b.cache_root.path orelse ".",
12821284 });
1283 if (b.zig_lib_dir) |p| {
1284 cmd.addArg("--zig-lib-dir");
1285 cmd.addDirectoryArg(p);
1286 }
1285 cmd.addArgs(&.{ "--zig-lib-dir", b.fmt("{}", .{b.graph.zig_lib_directory}) });
12871286 cmd.addArgs(&.{"-i"});
12881287 cmd.addFileArg(b.path(b.fmt("doc/langref/{s}", .{entry.name})));
12891288
lib/compiler/build_runner.zig+30-25
......@@ -31,21 +31,15 @@ pub fn main() !void {
3131 // skip my own exe name
3232 var arg_idx: usize = 1;
3333
34 const zig_exe = nextArg(args, &arg_idx) orelse {
35 std.debug.print("Expected path to zig compiler\n", .{});
36 return error.InvalidArgs;
37 };
38 const build_root = nextArg(args, &arg_idx) orelse {
39 std.debug.print("Expected build root directory path\n", .{});
40 return error.InvalidArgs;
41 };
42 const cache_root = nextArg(args, &arg_idx) orelse {
43 std.debug.print("Expected cache root directory path\n", .{});
44 return error.InvalidArgs;
45 };
46 const global_cache_root = nextArg(args, &arg_idx) orelse {
47 std.debug.print("Expected global cache root directory path\n", .{});
48 return error.InvalidArgs;
34 const zig_exe = nextArg(args, &arg_idx) orelse fatal("missing zig compiler path", .{});
35 const zig_lib_dir = nextArg(args, &arg_idx) orelse fatal("missing zig lib directory path", .{});
36 const build_root = nextArg(args, &arg_idx) orelse fatal("missing build root directory path", .{});
37 const cache_root = nextArg(args, &arg_idx) orelse fatal("missing cache root directory path", .{});
38 const global_cache_root = nextArg(args, &arg_idx) orelse fatal("missing global cache root directory path", .{});
39
40 const zig_lib_directory: std.Build.Cache.Directory = .{
41 .path = zig_lib_dir,
42 .handle = try std.fs.cwd().openDir(zig_lib_dir, .{}),
4943 };
5044
5145 const build_root_directory: std.Build.Cache.Directory = .{
......@@ -72,6 +66,7 @@ pub fn main() !void {
7266 .zig_exe = zig_exe,
7367 .env_map = try process.getEnvMap(arena),
7468 .global_cache_root = global_cache_directory,
69 .zig_lib_directory = zig_lib_directory,
7570 .host = .{
7671 .query = .{},
7772 .result = try std.zig.system.resolveTargetQuery(.{}),
......@@ -189,8 +184,6 @@ pub fn main() !void {
189184 arg, next_arg,
190185 });
191186 };
192 } else if (mem.eql(u8, arg, "--zig-lib-dir")) {
193 builder.zig_lib_dir = .{ .cwd_relative = nextArgOrFatal(args, &arg_idx) };
194187 } else if (mem.eql(u8, arg, "--seed")) {
195188 const next_arg = nextArg(args, &arg_idx) orelse
196189 fatalWithHint("expected u32 after '{s}'", .{arg});
......@@ -416,15 +409,27 @@ pub fn main() !void {
416409 const reaction_set = rs: {
417410 const gop = try w.dir_table.getOrPut(gpa, path);
418411 if (!gop.found_existing) {
419 std.posix.fanotify_mark(w.fan_fd, .{
420 .ADD = true,
421 .ONLYDIR = true,
422 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| {
423 fatal("unable to watch {}: {s}", .{ path, @errorName(err) });
424 };
425
426412 const dir_handle = try Watch.getDirHandle(gpa, path);
427 try w.handle_table.putNoClobber(gpa, dir_handle, .{});
413 // `dir_handle` may already be present in the table in
414 // the case that we have multiple Cache.Path instances
415 // that compare inequal but ultimately point to the same
416 // directory on the file system.
417 // In such case, we must revert adding this directory, but keep
418 // the additions to the step set.
419 const dh_gop = try w.handle_table.getOrPut(gpa, dir_handle);
420 if (dh_gop.found_existing) {
421 _ = w.dir_table.pop();
422 } else {
423 assert(dh_gop.index == gop.index);
424 dh_gop.value_ptr.* = .{};
425 std.posix.fanotify_mark(w.fan_fd, .{
426 .ADD = true,
427 .ONLYDIR = true,
428 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOrDot()) catch |err| {
429 fatal("unable to watch {}: {s}", .{ path, @errorName(err) });
430 };
431 }
432 break :rs dh_gop.value_ptr;
428433 }
429434 break :rs &w.handle_table.values()[gop.index];
430435 };
lib/std/Build.zig+7-9
......@@ -54,7 +54,6 @@ libc_file: ?[]const u8 = null,
5454/// Path to the directory containing build.zig.
5555build_root: Cache.Directory,
5656cache_root: Cache.Directory,
57zig_lib_dir: ?LazyPath,
5857pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
5958args: ?[]const []const u8 = null,
6059debug_log_scopes: []const []const u8 = &.{},
......@@ -117,6 +116,7 @@ pub const Graph = struct {
117116 zig_exe: [:0]const u8,
118117 env_map: EnvMap,
119118 global_cache_root: Cache.Directory,
119 zig_lib_directory: Cache.Directory,
120120 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
121121 /// Information about the native target. Computed before build() is invoked.
122122 host: ResolvedTarget,
......@@ -293,7 +293,6 @@ pub fn create(
293293 }),
294294 .description = "Remove build artifacts from prefix path",
295295 },
296 .zig_lib_dir = null,
297296 .install_path = undefined,
298297 .args = null,
299298 .host = graph.host,
......@@ -379,7 +378,6 @@ fn createChildOnly(
379378 .libc_file = parent.libc_file,
380379 .build_root = build_root,
381380 .cache_root = parent.cache_root,
382 .zig_lib_dir = parent.zig_lib_dir,
383381 .debug_log_scopes = parent.debug_log_scopes,
384382 .debug_compile_errors = parent.debug_compile_errors,
385383 .debug_pkg_config = parent.debug_pkg_config,
......@@ -687,7 +685,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile {
687685 .max_rss = options.max_rss,
688686 .use_llvm = options.use_llvm,
689687 .use_lld = options.use_lld,
690 .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
688 .zig_lib_dir = options.zig_lib_dir,
691689 .win32_manifest = options.win32_manifest,
692690 });
693691}
......@@ -735,7 +733,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile {
735733 .max_rss = options.max_rss,
736734 .use_llvm = options.use_llvm,
737735 .use_lld = options.use_lld,
738 .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
736 .zig_lib_dir = options.zig_lib_dir,
739737 });
740738}
741739
......@@ -791,7 +789,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile
791789 .max_rss = options.max_rss,
792790 .use_llvm = options.use_llvm,
793791 .use_lld = options.use_lld,
794 .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
792 .zig_lib_dir = options.zig_lib_dir,
795793 .win32_manifest = options.win32_manifest,
796794 });
797795}
......@@ -842,7 +840,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile
842840 .max_rss = options.max_rss,
843841 .use_llvm = options.use_llvm,
844842 .use_lld = options.use_lld,
845 .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
843 .zig_lib_dir = options.zig_lib_dir,
846844 });
847845}
848846
......@@ -905,7 +903,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile {
905903 .test_runner = options.test_runner,
906904 .use_llvm = options.use_llvm,
907905 .use_lld = options.use_lld,
908 .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
906 .zig_lib_dir = options.zig_lib_dir,
909907 });
910908}
911909
......@@ -929,7 +927,7 @@ pub fn addAssembly(b: *Build, options: AssemblyOptions) *Step.Compile {
929927 .optimize = options.optimize,
930928 },
931929 .max_rss = options.max_rss,
932 .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir,
930 .zig_lib_dir = options.zig_lib_dir,
933931 });
934932 obj_step.addAssemblyFile(options.source_file);
935933 return obj_step;
lib/std/Build/Cache.zig+16
......@@ -1007,6 +1007,22 @@ pub const Manifest = struct {
10071007 }
10081008 self.files.deinit(self.cache.gpa);
10091009 }
1010
1011 pub fn populateFileSystemInputs(man: *Manifest, buf: *std.ArrayListUnmanaged(u8)) Allocator.Error!void {
1012 assert(@typeInfo(std.zig.Server.Message.PathPrefix).Enum.fields.len == man.cache.prefixes_len);
1013 const gpa = man.cache.gpa;
1014 const files = man.files.keys();
1015 if (files.len > 0) {
1016 for (files) |file| {
1017 try buf.ensureUnusedCapacity(gpa, file.prefixed_path.sub_path.len + 2);
1018 buf.appendAssumeCapacity(file.prefixed_path.prefix + 1);
1019 buf.appendSliceAssumeCapacity(file.prefixed_path.sub_path);
1020 buf.appendAssumeCapacity(0);
1021 }
1022 // The null byte is a separator, not a terminator.
1023 buf.items.len -= 1;
1024 }
1025 }
10101026};
10111027
10121028/// On operating systems that support symlinks, does a readlink. On other operating systems,
lib/std/Build/Step.zig+38
......@@ -435,6 +435,44 @@ pub fn evalZigProcess(
435435 s.result_cached = ebp_hdr.flags.cache_hit;
436436 result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]);
437437 },
438 .file_system_inputs => {
439 s.clearWatchInputs();
440 var it = std.mem.splitScalar(u8, body, 0);
441 while (it.next()) |prefixed_path| {
442 const prefix_index: std.zig.Server.Message.PathPrefix = @enumFromInt(prefixed_path[0] - 1);
443 const sub_path = try arena.dupe(u8, prefixed_path[1..]);
444 const sub_path_dirname = std.fs.path.dirname(sub_path) orelse "";
445 switch (prefix_index) {
446 .cwd => {
447 const path: Build.Cache.Path = .{
448 .root_dir = Build.Cache.Directory.cwd(),
449 .sub_path = sub_path_dirname,
450 };
451 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
452 },
453 .zig_lib => zl: {
454 if (s.cast(Step.Compile)) |compile| {
455 if (compile.zig_lib_dir) |lp| {
456 try addWatchInput(s, lp);
457 break :zl;
458 }
459 }
460 const path: Build.Cache.Path = .{
461 .root_dir = s.owner.graph.zig_lib_directory,
462 .sub_path = sub_path_dirname,
463 };
464 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
465 },
466 .local_cache => {
467 const path: Build.Cache.Path = .{
468 .root_dir = b.cache_root,
469 .sub_path = sub_path_dirname,
470 };
471 try addWatchInputFromPath(s, path, std.fs.path.basename(sub_path));
472 },
473 }
474 }
475 },
438476 else => {}, // ignore other messages
439477 }
440478
lib/std/zig/Server.zig+15-1
......@@ -20,10 +20,24 @@ pub const Message = struct {
2020 test_metadata,
2121 /// Body is a TestResults
2222 test_results,
23 /// Body is a series of strings, delimited by null bytes.
24 /// Each string is a prefixed file path.
25 /// The first byte indicates the file prefix path (see prefixes fields
26 /// of Cache). This byte is sent over the wire incremented so that null
27 /// bytes are not confused with string terminators.
28 /// The remaining bytes is the file path relative to that prefix.
29 /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
30 file_system_inputs,
2331
2432 _,
2533 };
2634
35 pub const PathPrefix = enum(u8) {
36 cwd,
37 zig_lib,
38 local_cache,
39 };
40
2741 /// Trailing:
2842 /// * extra: [extra_len]u32,
2943 /// * string_bytes: [string_bytes_len]u8,
......@@ -58,7 +72,7 @@ pub const Message = struct {
5872 };
5973
6074 /// Trailing:
61 /// * the file system path the emitted binary can be found
75 /// * file system path where the emitted binary can be found
6276 pub const EmitBinPath = extern struct {
6377 flags: Flags,
6478
src/Compilation.zig+14
......@@ -235,6 +235,8 @@ astgen_wait_group: WaitGroup = .{},
235235
236236llvm_opt_bisect_limit: c_int,
237237
238file_system_inputs: ?*std.ArrayListUnmanaged(u8),
239
238240pub const Emit = struct {
239241 /// Where the output will go.
240242 directory: Directory,
......@@ -1157,6 +1159,9 @@ pub const CreateOptions = struct {
11571159 error_limit: ?Zcu.ErrorInt = null,
11581160 global_cc_argv: []const []const u8 = &.{},
11591161
1162 /// Tracks all files that can cause the Compilation to be invalidated and need a rebuild.
1163 file_system_inputs: ?*std.ArrayListUnmanaged(u8) = null,
1164
11601165 pub const Entry = link.File.OpenOptions.Entry;
11611166};
11621167
......@@ -1332,6 +1337,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
13321337 .gpa = gpa,
13331338 .manifest_dir = try options.local_cache_directory.handle.makeOpenPath("h", .{}),
13341339 };
1340 // These correspond to std.zig.Server.Message.PathPrefix.
13351341 cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
13361342 cache.addPrefix(options.zig_lib_directory);
13371343 cache.addPrefix(options.local_cache_directory);
......@@ -1508,6 +1514,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
15081514 .force_undefined_symbols = options.force_undefined_symbols,
15091515 .link_eh_frame_hdr = link_eh_frame_hdr,
15101516 .global_cc_argv = options.global_cc_argv,
1517 .file_system_inputs = options.file_system_inputs,
15111518 };
15121519
15131520 // Prevent some footguns by making the "any" fields of config reflect
......@@ -2044,6 +2051,8 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
20442051 );
20452052 };
20462053 if (is_hit) {
2054 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
2055
20472056 comp.last_update_was_cache_hit = true;
20482057 log.debug("CacheMode.whole cache hit for {s}", .{comp.root_name});
20492058 const digest = man.final();
......@@ -2170,6 +2179,11 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
21702179
21712180 try comp.performAllTheWork(main_progress_node);
21722181
2182 switch (comp.cache_use) {
2183 .whole => if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf),
2184 .incremental => {},
2185 }
2186
21732187 if (comp.module) |zcu| {
21742188 const pt: Zcu.PerThread = .{ .zcu = zcu, .tid = .main };
21752189
src/main.zig+28-4
......@@ -3227,6 +3227,9 @@ fn buildOutputType(
32273227
32283228 process.raiseFileDescriptorLimit();
32293229
3230 var file_system_inputs: std.ArrayListUnmanaged(u8) = .{};
3231 defer file_system_inputs.deinit(gpa);
3232
32303233 const comp = Compilation.create(gpa, arena, .{
32313234 .zig_lib_directory = zig_lib_directory,
32323235 .local_cache_directory = local_cache_directory,
......@@ -3350,6 +3353,7 @@ fn buildOutputType(
33503353 // than to any particular module. This feature can greatly reduce CLI
33513354 // noise when --search-prefix and --mod are combined.
33523355 .global_cc_argv = try cc_argv.toOwnedSlice(arena),
3356 .file_system_inputs = &file_system_inputs,
33533357 }) catch |err| switch (err) {
33543358 error.LibCUnavailable => {
33553359 const triple_name = try target.zigTriple(arena);
......@@ -3433,7 +3437,7 @@ fn buildOutputType(
34333437 defer root_prog_node.end();
34343438
34353439 if (arg_mode == .translate_c) {
3436 return cmdTranslateC(comp, arena, null, root_prog_node);
3440 return cmdTranslateC(comp, arena, null, null, root_prog_node);
34373441 }
34383442
34393443 updateModule(comp, color, root_prog_node) catch |err| switch (err) {
......@@ -4059,6 +4063,7 @@ fn serve(
40594063 var child_pid: ?std.process.Child.Id = null;
40604064
40614065 const main_progress_node = std.Progress.start(.{});
4066 const file_system_inputs = comp.file_system_inputs.?;
40624067
40634068 while (true) {
40644069 const hdr = try server.receiveMessage();
......@@ -4067,14 +4072,16 @@ fn serve(
40674072 .exit => return cleanExit(),
40684073 .update => {
40694074 tracy.frameMark();
4075 file_system_inputs.clearRetainingCapacity();
40704076
40714077 if (arg_mode == .translate_c) {
40724078 var arena_instance = std.heap.ArenaAllocator.init(gpa);
40734079 defer arena_instance.deinit();
40744080 const arena = arena_instance.allocator();
40754081 var output: Compilation.CImportResult = undefined;
4076 try cmdTranslateC(comp, arena, &output, main_progress_node);
4082 try cmdTranslateC(comp, arena, &output, file_system_inputs, main_progress_node);
40774083 defer output.deinit(gpa);
4084 try server.serveStringMessage(.file_system_inputs, file_system_inputs.items);
40784085 if (output.errors.errorMessageCount() != 0) {
40794086 try server.serveErrorBundle(output.errors);
40804087 } else {
......@@ -4116,6 +4123,7 @@ fn serve(
41164123 },
41174124 .hot_update => {
41184125 tracy.frameMark();
4126 file_system_inputs.clearRetainingCapacity();
41194127 if (child_pid) |pid| {
41204128 try comp.hotCodeSwap(main_progress_node, pid);
41214129 try serveUpdateResults(&server, comp);
......@@ -4147,6 +4155,12 @@ fn serve(
41474155
41484156fn serveUpdateResults(s: *Server, comp: *Compilation) !void {
41494157 const gpa = comp.gpa;
4158
4159 if (comp.file_system_inputs) |file_system_inputs| {
4160 assert(file_system_inputs.items.len > 0);
4161 try s.serveStringMessage(.file_system_inputs, file_system_inputs.items);
4162 }
4163
41504164 var error_bundle = try comp.getAllErrorsAlloc();
41514165 defer error_bundle.deinit(gpa);
41524166 if (error_bundle.errorMessageCount() > 0) {
......@@ -4434,6 +4448,7 @@ fn cmdTranslateC(
44344448 comp: *Compilation,
44354449 arena: Allocator,
44364450 fancy_output: ?*Compilation.CImportResult,
4451 file_system_inputs: ?*std.ArrayListUnmanaged(u8),
44374452 prog_node: std.Progress.Node,
44384453) !void {
44394454 if (build_options.only_core_functionality) @panic("@translate-c is not available in a zig2.c build");
......@@ -4454,7 +4469,10 @@ fn cmdTranslateC(
44544469 };
44554470
44564471 if (fancy_output) |p| p.cache_hit = true;
4457 const digest = if (try man.hit()) man.final() else digest: {
4472 const digest = if (try man.hit()) digest: {
4473 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
4474 break :digest man.final();
4475 } else digest: {
44584476 if (fancy_output) |p| p.cache_hit = false;
44594477 var argv = std.ArrayList([]const u8).init(arena);
44604478 switch (comp.config.c_frontend) {
......@@ -4566,6 +4584,8 @@ fn cmdTranslateC(
45664584 @errorName(err),
45674585 });
45684586
4587 if (file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
4588
45694589 break :digest digest;
45704590 };
45714591
......@@ -4678,6 +4698,9 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
46784698 const self_exe_path = try introspect.findZigExePath(arena);
46794699 try child_argv.append(self_exe_path);
46804700
4701 const argv_index_zig_lib_dir = child_argv.items.len;
4702 _ = try child_argv.addOne();
4703
46814704 const argv_index_build_file = child_argv.items.len;
46824705 _ = try child_argv.addOne();
46834706
......@@ -4727,7 +4750,6 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
47274750 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
47284751 i += 1;
47294752 override_lib_dir = args[i];
4730 try child_argv.appendSlice(&.{ arg, args[i] });
47314753 continue;
47324754 } else if (mem.eql(u8, arg, "--build-runner")) {
47334755 if (i + 1 >= args.len) fatal("expected argument after '{s}'", .{arg});
......@@ -4865,6 +4887,8 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
48654887 defer zig_lib_directory.handle.close();
48664888
48674889 const cwd_path = try process.getCwdAlloc(arena);
4890 child_argv.items[argv_index_zig_lib_dir] = zig_lib_directory.path orelse cwd_path;
4891
48684892 const build_root = try findBuildRoot(arena, .{
48694893 .cwd_path = cwd_path,
48704894 .build_file = build_file,