authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-08 23:42:20-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:14:07-07:00
logbbd90a562efd6e802ed41df2649a05fad763a4de
treec5276879ad6ade65af80393019ac649f9e6c1734
parentdeea36250ffe458d92b32b1ad090b8a958ba8082

build runner: implement --watch (work-in-progress)

I'm still learning how the fanotify API works but I think after playing with it in this commit, I finally know how to implement it, at least on Linux. This commit does not accomplish the goal but I want to take the code in a different direction and still be able to reference this point in time by viewing a source control diff. I think the move is going to be saving the file_handle for the parent directory, which combined with the dirent names is how we can correlate the events back to the Step instances that have registered file system inputs. I predict this to be similar to implementations on other operating systems.

4 files changed, 407 insertions(+), 144 deletions(-)

lib/compiler/build_runner.zig+352-66
......@@ -8,6 +8,7 @@ const process = std.process;
88const ArrayList = std.ArrayList;
99const File = std.fs.File;
1010const Step = std.Build.Step;
11const Allocator = std.mem.Allocator;
1112
1213pub const root = @import("@build");
1314pub const dependencies = @import("@dependencies");
......@@ -74,7 +75,6 @@ pub fn main() !void {
7475 .query = .{},
7576 .result = try std.zig.system.resolveTargetQuery(.{}),
7677 },
77 .watch = null,
7878 };
7979
8080 graph.cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
......@@ -105,6 +105,7 @@ pub fn main() !void {
105105 var help_menu = false;
106106 var steps_menu = false;
107107 var output_tmp_nonce: ?[16]u8 = null;
108 var watch = false;
108109
109110 while (nextArg(args, &arg_idx)) |arg| {
110111 if (mem.startsWith(u8, arg, "-Z")) {
......@@ -229,9 +230,7 @@ pub fn main() !void {
229230 } else if (mem.eql(u8, arg, "--prominent-compile-errors")) {
230231 prominent_compile_errors = true;
231232 } else if (mem.eql(u8, arg, "--watch")) {
232 const watch = try arena.create(std.Build.Watch);
233 watch.* = std.Build.Watch.init;
234 graph.watch = watch;
233 watch = true;
235234 } else if (mem.eql(u8, arg, "-fwine")) {
236235 builder.enable_wine = true;
237236 } else if (mem.eql(u8, arg, "-fno-wine")) {
......@@ -297,6 +296,7 @@ pub fn main() !void {
297296 const main_progress_node = std.Progress.start(.{
298297 .disable_printing = (color == .off),
299298 });
299 defer main_progress_node.end();
300300
301301 builder.debug_log_scopes = debug_log_scopes.items;
302302 builder.resolveInstallPrefix(install_prefix, dir_list);
......@@ -345,13 +345,16 @@ pub fn main() !void {
345345 .max_rss_is_default = false,
346346 .max_rss_mutex = .{},
347347 .skip_oom_steps = skip_oom_steps,
348 .watch = watch,
348349 .memory_blocked_steps = std.ArrayList(*Step).init(arena),
350 .step_stack = .{},
349351 .prominent_compile_errors = prominent_compile_errors,
350352
351353 .claimed_rss = 0,
352 .summary = summary orelse if (graph.watch != null) .new else .failures,
354 .summary = summary orelse if (watch) .new else .failures,
353355 .ttyconf = ttyconf,
354356 .stderr = stderr,
357 .thread_pool = undefined,
355358 };
356359
357360 if (run.max_rss == 0) {
......@@ -359,30 +362,311 @@ pub fn main() !void {
359362 run.max_rss_is_default = true;
360363 }
361364
362 runStepNames(
363 arena,
364 builder,
365 targets.items,
366 main_progress_node,
367 thread_pool_options,
368 &run,
369 seed,
370 ) catch |err| switch (err) {
371 error.UncleanExit => {
372 if (graph.watch == null)
373 process.exit(1);
374 },
365 const gpa = arena;
366 prepare(gpa, arena, builder, targets.items, &run, seed) catch |err| switch (err) {
367 error.UncleanExit => process.exit(1),
375368 else => return err,
376369 };
370
371 var w = Watch.init;
372 if (watch) {
373 w.fan_fd = try std.posix.fanotify_init(.{
374 .CLASS = .NOTIF,
375 .CLOEXEC = true,
376 .NONBLOCK = true,
377 .REPORT_NAME = true,
378 .REPORT_DIR_FID = true,
379 .REPORT_FID = true,
380 .REPORT_TARGET_FID = true,
381 }, 0);
382 }
383
384 try run.thread_pool.init(thread_pool_options);
385 defer run.thread_pool.deinit();
386
387 rebuild: while (true) {
388 runStepNames(
389 gpa,
390 builder,
391 targets.items,
392 main_progress_node,
393 &run,
394 ) catch |err| switch (err) {
395 error.UncleanExit => {
396 assert(!run.watch);
397 process.exit(1);
398 },
399 else => return err,
400 };
401 if (!watch) return cleanExit();
402
403 // Clear all file handles.
404 for (w.handle_table.keys(), w.handle_table.values()) |lfh, *step_set| {
405 lfh.destroy(gpa);
406 step_set.clearAndFree(gpa);
407 }
408 w.handle_table.clearRetainingCapacity();
409
410 // Add missing marks and note persisted ones.
411 for (run.step_stack.keys()) |step| {
412 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
413 {
414 const gop = try w.dir_table.getOrPut(gpa, path);
415 gop.value_ptr.* = w.generation;
416 if (!gop.found_existing) {
417 try std.posix.fanotify_mark(w.fan_fd, .{
418 .ADD = true,
419 .ONLYDIR = true,
420 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt());
421 }
422 }
423 for (files.items) |basename| {
424 const file_handle = try Watch.getFileHandle(gpa, path, basename);
425 std.debug.print("watching file_handle '{}{s}' = {}\n", .{
426 path, basename, std.fmt.fmtSliceHexLower(file_handle.slice()),
427 });
428 const gop = try w.handle_table.getOrPut(gpa, file_handle);
429 if (!gop.found_existing) gop.value_ptr.* = .{};
430 try gop.value_ptr.put(gpa, step, {});
431 }
432 }
433 }
434
435 {
436 // Remove marks for files that are no longer inputs.
437 var i: usize = 0;
438 while (i < w.dir_table.entries.len) {
439 const generations = w.dir_table.values();
440 if (generations[i] == w.generation) {
441 i += 1;
442 continue;
443 }
444
445 const path = w.dir_table.keys()[i];
446
447 try std.posix.fanotify_mark(w.fan_fd, .{
448 .REMOVE = true,
449 .ONLYDIR = true,
450 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt());
451
452 w.dir_table.swapRemoveAt(i);
453 }
454 w.generation +%= 1;
455 }
456
457 // Wait until a file system notification arrives. Read all such events
458 // until the buffer is empty. Then wait for a debounce interval, resetting
459 // if any more events come in. After the debounce interval has passed,
460 // trigger a rebuild on all steps with modified inputs, as well as their
461 // recursive dependants.
462 const debounce_interval_ms = 10;
463 var poll_fds: [1]std.posix.pollfd = .{
464 .{
465 .fd = w.fan_fd,
466 .events = std.posix.POLL.IN,
467 .revents = undefined,
468 },
469 };
470 var caption_buf: [40]u8 = undefined;
471 const caption = std.fmt.bufPrint(&caption_buf, "Watching {d} Directories", .{
472 w.dir_table.entries.len,
473 }) catch &caption_buf;
474 var debouncing_node = main_progress_node.start(caption, 0);
475 var debouncing = false;
476 while (true) {
477 const timeout: i32 = if (debouncing) debounce_interval_ms else -1;
478 const events_len = try std.posix.poll(&poll_fds, timeout);
479 if (events_len == 0) {
480 debouncing_node.end();
481 continue :rebuild;
482 }
483 if (try markDirtySteps(&w)) {
484 if (!debouncing) {
485 debouncing = true;
486 debouncing_node.end();
487 debouncing_node = main_progress_node.start("Debouncing (Change Detected)", 0);
488 }
489 }
490 }
491 }
377492}
378493
494fn markDirtySteps(w: *Watch) !bool {
495 const fanotify = std.os.linux.fanotify;
496 const M = fanotify.event_metadata;
497 var events_buf: [256 + 4096]u8 = undefined;
498 var any_dirty = false;
499 while (true) {
500 var len = std.posix.read(w.fan_fd, &events_buf) catch |err| switch (err) {
501 error.WouldBlock => return any_dirty,
502 else => |e| return e,
503 };
504 //std.debug.dump_hex(events_buf[0..len]);
505 var meta: [*]align(1) M = @ptrCast(&events_buf);
506 while (len >= @sizeOf(M) and meta[0].event_len >= @sizeOf(M) and meta[0].event_len <= len) : ({
507 len -= meta[0].event_len;
508 meta = @ptrCast(@as([*]u8, @ptrCast(meta)) + meta[0].event_len);
509 }) {
510 assert(meta[0].vers == M.VERSION);
511 std.debug.print("meta = {any}\n", .{meta[0]});
512 const fid: *align(1) fanotify.event_info_fid = @ptrCast(meta + 1);
513 switch (fid.hdr.info_type) {
514 .DFID_NAME => {
515 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
516 const file_name_z: [*:0]u8 = @ptrCast((&file_handle.f_handle).ptr + file_handle.handle_bytes);
517 const file_name = mem.span(file_name_z);
518 std.debug.print("DFID_NAME file_handle = {any}, found: '{s}'\n", .{ file_handle.*, file_name });
519 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
520 if (w.handle_table.get(lfh)) |step_set| {
521 for (step_set.keys()) |step| {
522 std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name});
523 step.state = .precheck_done;
524 any_dirty = true;
525 }
526 } else {
527 std.debug.print("DFID_NAME changed file did not match any steps: '{}'\n", .{
528 std.fmt.fmtSliceHexLower(lfh.slice()),
529 });
530 }
531 },
532 .FID => {
533 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
534 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
535 if (w.handle_table.get(lfh)) |step_set| {
536 for (step_set.keys()) |step| {
537 std.debug.print("FID marking step '{s}' dirty\n", .{step.name});
538 step.state = .precheck_done;
539 any_dirty = true;
540 }
541 } else {
542 std.debug.print("FID changed file did not match any steps: '{}'\n", .{
543 std.fmt.fmtSliceHexLower(lfh.slice()),
544 });
545 }
546 },
547 .DFID => {
548 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
549 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
550 if (w.handle_table.get(lfh)) |step_set| {
551 for (step_set.keys()) |step| {
552 std.debug.print("DFID marking step '{s}' dirty\n", .{step.name});
553 step.state = .precheck_done;
554 any_dirty = true;
555 }
556 } else {
557 std.debug.print("DFID changed file did not match any steps\n", .{});
558 }
559 },
560 else => |t| {
561 std.debug.panic("TODO: received event type '{s}'", .{@tagName(t)});
562 },
563 }
564 }
565 }
566}
567
568const Watch = struct {
569 dir_table: DirTable,
570 handle_table: HandleTable,
571 fan_fd: std.posix.fd_t,
572 generation: u8,
573
574 const fan_mask: std.os.linux.fanotify.MarkMask = .{
575 .CLOSE_WRITE = true,
576 .DELETE = true,
577 .MOVED_FROM = true,
578 .MOVED_TO = true,
579 .EVENT_ON_CHILD = true,
580 };
581
582 const init: Watch = .{
583 .dir_table = .{},
584 .handle_table = .{},
585 .fan_fd = -1,
586 .generation = 0,
587 };
588
589 /// Key is the directory to watch which contains one or more files we are
590 /// interested in noticing changes to.
591 ///
592 /// Value is generation.
593 const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, u8, Cache.Path.TableAdapter, false);
594
595 const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, StepSet, LinuxFileHandle.Adapter, false);
596 const StepSet = std.AutoArrayHashMapUnmanaged(*Step, void);
597
598 const Hash = std.hash.Wyhash;
599 const Cache = std.Build.Cache;
600
601 const LinuxFileHandle = struct {
602 handle: *align(1) std.os.linux.file_handle,
603
604 fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle {
605 const bytes = lfh.slice();
606 const new_ptr = try gpa.alignedAlloc(
607 u8,
608 @alignOf(std.os.linux.file_handle),
609 @sizeOf(std.os.linux.file_handle) + bytes.len,
610 );
611 const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr);
612 new_header.* = lfh.handle.*;
613 const new: LinuxFileHandle = .{ .handle = new_header };
614 @memcpy(new.slice(), lfh.slice());
615 return new;
616 }
617
618 fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void {
619 const ptr: [*]u8 = @ptrCast(lfh.handle);
620 const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes];
621 return gpa.free(allocated_slice);
622 }
623
624 fn slice(lfh: LinuxFileHandle) []u8 {
625 const ptr: [*]u8 = &lfh.handle.f_handle;
626 return ptr[0..lfh.handle.handle_bytes];
627 }
628
629 const Adapter = struct {
630 pub fn hash(self: Adapter, a: LinuxFileHandle) u32 {
631 _ = self;
632 const unsigned_type: u32 = @bitCast(a.handle.handle_type);
633 return @truncate(Hash.hash(unsigned_type, a.slice()));
634 }
635 pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool {
636 _ = self;
637 _ = b_index;
638 return a.handle.handle_type == b.handle.handle_type and mem.eql(u8, a.slice(), b.slice());
639 }
640 };
641 };
642
643 fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle {
644 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
645 var mount_id: i32 = undefined;
646 var buf: [std.fs.max_path_bytes]u8 = undefined;
647 const joined_path = if (path.sub_path.len == 0) basename else path: {
648 break :path std.fmt.bufPrint(&buf, "{s}" ++ std.fs.path.sep_str ++ "{s}", .{
649 path.sub_path, basename,
650 }) catch return error.NameTooLong;
651 };
652 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
653 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
654 try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0);
655 const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr };
656 return stack_lfh.clone(gpa);
657 }
658};
659
379660const Run = struct {
380661 max_rss: u64,
381662 max_rss_is_default: bool,
382663 max_rss_mutex: std.Thread.Mutex,
383664 skip_oom_steps: bool,
665 watch: bool,
384666 memory_blocked_steps: std.ArrayList(*Step),
667 step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
385668 prominent_compile_errors: bool,
669 thread_pool: std.Thread.Pool,
386670
387671 claimed_rss: usize,
388672 summary: Summary,
......@@ -390,18 +674,15 @@ const Run = struct {
390674 stderr: File,
391675};
392676
393fn runStepNames(
394 arena: std.mem.Allocator,
677fn prepare(
678 gpa: Allocator,
679 arena: Allocator,
395680 b: *std.Build,
396681 step_names: []const []const u8,
397 parent_prog_node: std.Progress.Node,
398 thread_pool_options: std.Thread.Pool.Options,
399682 run: *Run,
400683 seed: u32,
401684) !void {
402 const gpa = b.allocator;
403 var step_stack: std.AutoArrayHashMapUnmanaged(*Step, void) = .{};
404 defer step_stack.deinit(gpa);
685 const step_stack = &run.step_stack;
405686
406687 if (step_names.len == 0) {
407688 try step_stack.put(gpa, b.default_step, {});
......@@ -424,7 +705,7 @@ fn runStepNames(
424705 rand.shuffle(*Step, starting_steps);
425706
426707 for (starting_steps) |s| {
427 constructGraphAndCheckForDependencyLoop(b, s, &step_stack, rand) catch |err| switch (err) {
708 constructGraphAndCheckForDependencyLoop(b, s, &run.step_stack, rand) catch |err| switch (err) {
428709 error.DependencyLoopDetected => return uncleanExit(),
429710 else => |e| return e,
430711 };
......@@ -453,14 +734,19 @@ fn runStepNames(
453734 return uncleanExit();
454735 }
455736 }
737}
456738
457 var thread_pool: std.Thread.Pool = undefined;
458 try thread_pool.init(thread_pool_options);
459 defer thread_pool.deinit();
739fn runStepNames(
740 gpa: Allocator,
741 b: *std.Build,
742 step_names: []const []const u8,
743 parent_prog_node: std.Progress.Node,
744 run: *Run,
745) !void {
746 const step_stack = &run.step_stack;
747 const thread_pool = &run.thread_pool;
460748
461749 {
462 defer parent_prog_node.end();
463
464750 const step_prog = parent_prog_node.start("steps", step_stack.count());
465751 defer step_prog.end();
466752
......@@ -476,7 +762,7 @@ fn runStepNames(
476762 if (step.state == .skipped_oom) continue;
477763
478764 thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{
479 &wait_group, &thread_pool, b, step, step_prog, run,
765 &wait_group, b, step, step_prog, run,
480766 });
481767 }
482768 }
......@@ -493,8 +779,6 @@ fn runStepNames(
493779 var failure_count: usize = 0;
494780 var pending_count: usize = 0;
495781 var total_compile_errors: usize = 0;
496 var compile_error_steps: std.ArrayListUnmanaged(*Step) = .{};
497 defer compile_error_steps.deinit(gpa);
498782
499783 for (step_stack.keys()) |s| {
500784 test_fail_count += s.test_results.fail_count;
......@@ -524,7 +808,6 @@ fn runStepNames(
524808 const compile_errors_len = s.result_error_bundle.errorMessageCount();
525809 if (compile_errors_len > 0) {
526810 total_compile_errors += compile_errors_len;
527 try compile_error_steps.append(gpa, s);
528811 }
529812 },
530813 }
......@@ -537,8 +820,8 @@ fn runStepNames(
537820 else => false,
538821 };
539822 if (failure_count == 0 and failures_only) {
540 if (b.graph.watch != null) return;
541 return cleanExit();
823 if (!run.watch) cleanExit();
824 return;
542825 }
543826
544827 const ttyconf = run.ttyconf;
......@@ -561,10 +844,13 @@ fn runStepNames(
561844 stderr.writeAll("\n") catch {};
562845
563846 // Print a fancy tree with build results.
847 var step_stack_copy = try step_stack.clone(gpa);
848 defer step_stack_copy.deinit(gpa);
849
564850 var print_node: PrintNode = .{ .parent = null };
565851 if (step_names.len == 0) {
566852 print_node.last = true;
567 printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack) catch {};
853 printTreeStep(b, b.default_step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {};
568854 } else {
569855 const last_index = if (run.summary == .all) b.top_level_steps.count() else blk: {
570856 var i: usize = step_names.len;
......@@ -583,44 +869,34 @@ fn runStepNames(
583869 for (step_names, 0..) |step_name, i| {
584870 const tls = b.top_level_steps.get(step_name).?;
585871 print_node.last = i + 1 == last_index;
586 printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack) catch {};
872 printTreeStep(b, &tls.step, run, stderr, ttyconf, &print_node, &step_stack_copy) catch {};
587873 }
588874 }
589875 }
590876
591877 if (failure_count == 0) {
592 if (b.graph.watch != null) return;
593 return cleanExit();
878 if (!run.watch) cleanExit();
879 return;
594880 }
595881
596882 // Finally, render compile errors at the bottom of the terminal.
597 // We use a separate compile_error_steps array list because step_stack is destructively
598 // mutated in printTreeStep above.
599883 if (run.prominent_compile_errors and total_compile_errors > 0) {
600 for (compile_error_steps.items) |s| {
884 for (step_stack.keys()) |s| {
601885 if (s.result_error_bundle.errorMessageCount() > 0) {
602886 s.result_error_bundle.renderToStdErr(renderOptions(ttyconf));
603887 }
604888 }
605889
606 if (b.graph.watch != null) return uncleanExit();
607
608 // Signal to parent process that we have printed compile errors. The
609 // parent process may choose to omit the "following command failed"
610 // line in this case.
611 process.exit(2);
890 if (!run.watch) {
891 // Signal to parent process that we have printed compile errors. The
892 // parent process may choose to omit the "following command failed"
893 // line in this case.
894 std.debug.lockStdErr();
895 process.exit(2);
896 }
612897 }
613898
614 return uncleanExit();
615}
616
617fn uncleanExit() error{UncleanExit}!void {
618 if (builtin.mode == .Debug) {
619 return error.UncleanExit;
620 } else {
621 std.debug.lockStdErr();
622 process.exit(1);
623 }
899 if (!run.watch) return uncleanExit();
624900}
625901
626902const PrintNode = struct {
......@@ -912,12 +1188,13 @@ fn constructGraphAndCheckForDependencyLoop(
9121188
9131189fn workerMakeOneStep(
9141190 wg: *std.Thread.WaitGroup,
915 thread_pool: *std.Thread.Pool,
9161191 b: *std.Build,
9171192 s: *Step,
9181193 prog_node: std.Progress.Node,
9191194 run: *Run,
9201195) void {
1196 const thread_pool = &run.thread_pool;
1197
9211198 // First, check the conditions for running this step. If they are not met,
9221199 // then we return without doing the step, relying on another worker to
9231200 // queue this step up again when dependencies are met.
......@@ -997,7 +1274,7 @@ fn workerMakeOneStep(
9971274 // Successful completion of a step, so we queue up its dependants as well.
9981275 for (s.dependants.items) |dep| {
9991276 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1000 wg, thread_pool, b, dep, prog_node, run,
1277 wg, b, dep, prog_node, run,
10011278 });
10021279 }
10031280 }
......@@ -1022,7 +1299,7 @@ fn workerMakeOneStep(
10221299 remaining -= dep.max_rss;
10231300
10241301 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1025 wg, thread_pool, b, dep, prog_node, run,
1302 wg, b, dep, prog_node, run,
10261303 });
10271304 } else {
10281305 run.memory_blocked_steps.items[i] = dep;
......@@ -1242,13 +1519,22 @@ fn argsRest(args: [][:0]const u8, idx: usize) ?[][:0]const u8 {
12421519 return args[idx..];
12431520}
12441521
1522/// Perhaps in the future there could be an Advanced Options flag such as
1523/// --debug-build-runner-leaks which would make this function return instead of
1524/// calling exit.
12451525fn cleanExit() void {
1246 // Perhaps in the future there could be an Advanced Options flag such as
1247 // --debug-build-runner-leaks which would make this function return instead
1248 // of calling exit.
1526 std.debug.lockStdErr();
12491527 process.exit(0);
12501528}
12511529
1530/// Perhaps in the future there could be an Advanced Options flag such as
1531/// --debug-build-runner-leaks which would make this function return instead of
1532/// calling exit.
1533fn uncleanExit() error{UncleanExit} {
1534 std.debug.lockStdErr();
1535 process.exit(1);
1536}
1537
12521538const Color = std.zig.Color;
12531539const Summary = enum { all, new, failures, none };
12541540
lib/std/Build.zig-55
......@@ -119,61 +119,6 @@ pub const Graph = struct {
119119 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
120120 /// Information about the native target. Computed before build() is invoked.
121121 host: ResolvedTarget,
122 /// When `--watch` is provided, collects the set of files that should be
123 /// watched and the state to required to poll the system for changes.
124 watch: ?*Watch,
125};
126
127pub const Watch = struct {
128 table: Table,
129
130 pub const init: Watch = .{
131 .table = .{},
132 };
133
134 /// Key is the directory to watch which contains one or more files we are
135 /// interested in noticing changes to.
136 pub const Table = std.ArrayHashMapUnmanaged(Cache.Path, ReactionSet, TableContext, false);
137
138 const Hash = std.hash.Wyhash;
139
140 pub const TableContext = struct {
141 pub fn hash(self: TableContext, a: Cache.Path) u32 {
142 _ = self;
143 const seed: u32 = @bitCast(a.root_dir.handle.fd);
144 return @truncate(Hash.hash(seed, a.sub_path));
145 }
146 pub fn eql(self: TableContext, a: Cache.Path, b: Cache.Path, b_index: usize) bool {
147 _ = self;
148 _ = b_index;
149 return a.eql(b);
150 }
151 };
152
153 pub const ReactionSet = std.ArrayHashMapUnmanaged(Match, void, Match.Context, false);
154
155 pub const Match = struct {
156 /// Relative to the watched directory, the file path that triggers this
157 /// match.
158 basename: []const u8,
159 /// The step to re-run when file corresponding to `basename` is changed.
160 step: *Step,
161
162 pub const Context = struct {
163 pub fn hash(self: Context, a: Match) u32 {
164 _ = self;
165 var hasher = Hash.init(0);
166 std.hash.autoHash(&hasher, a.step);
167 hasher.update(a.basename);
168 return @truncate(hasher.final());
169 }
170 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
171 _ = self;
172 _ = b_index;
173 return a.step == b.step and mem.eql(u8, a.basename, b.basename);
174 }
175 };
176 };
177122};
178123
179124const AvailableDeps = []const struct { []const u8, []const u8 };
lib/std/Build/Step.zig+51-22
......@@ -7,6 +7,16 @@ dependencies: std.ArrayList(*Step),
77/// This field is empty during execution of the user's build script, and
88/// then populated during dependency loop checking in the build runner.
99dependants: std.ArrayListUnmanaged(*Step),
10/// Collects the set of files that retrigger this step to run.
11///
12/// This is used by the build system's implementation of `--watch` but it can
13/// also be potentially useful for IDEs to know what effects editing a
14/// particular file has.
15///
16/// Populated within `make`. Implementation may choose to clear and repopulate,
17/// retain previous value, or update.
18inputs: Inputs,
19
1020state: State,
1121/// Set this field to declare an upper bound on the amount of bytes of memory it will
1222/// take to run the step. Zero means no limit.
......@@ -63,6 +73,11 @@ pub const MakeFn = *const fn (step: *Step, prog_node: std.Progress.Node) anyerro
6373pub const State = enum {
6474 precheck_unstarted,
6575 precheck_started,
76 /// This is also used to indicate "dirty" steps that have been modified
77 /// after a previous build completed, in which case, the step may or may
78 /// not have been completed before. Either way, one or more of its direct
79 /// file system inputs have been modified, meaning that the step needs to
80 /// be re-evaluated.
6681 precheck_done,
6782 running,
6883 dependency_failure,
......@@ -134,6 +149,26 @@ pub const Run = @import("Step/Run.zig");
134149pub const TranslateC = @import("Step/TranslateC.zig");
135150pub const WriteFile = @import("Step/WriteFile.zig");
136151
152pub const Inputs = struct {
153 table: Table,
154
155 pub const init: Inputs = .{
156 .table = .{},
157 };
158
159 pub const Table = std.ArrayHashMapUnmanaged(Build.Cache.Path, Files, Build.Cache.Path.TableAdapter, false);
160 pub const Files = std.ArrayListUnmanaged([]const u8);
161
162 pub fn populated(inputs: *Inputs) bool {
163 return inputs.table.count() != 0;
164 }
165
166 pub fn clear(inputs: *Inputs, gpa: Allocator) void {
167 for (inputs.table.values()) |*files| files.deinit(gpa);
168 inputs.table.clearRetainingCapacity();
169 }
170};
171
137172pub const StepOptions = struct {
138173 id: Id,
139174 name: []const u8,
......@@ -153,6 +188,7 @@ pub fn init(options: StepOptions) Step {
153188 .makeFn = options.makeFn,
154189 .dependencies = std.ArrayList(*Step).init(arena),
155190 .dependants = .{},
191 .inputs = Inputs.init,
156192 .state = .precheck_unstarted,
157193 .max_rss = options.max_rss,
158194 .debug_stack_trace = blk: {
......@@ -542,19 +578,19 @@ pub fn allocPrintCmd2(
542578 return buf.toOwnedSlice(arena);
543579}
544580
545pub fn cacheHit(s: *Step, man: *std.Build.Cache.Manifest) !bool {
581pub fn cacheHit(s: *Step, man: *Build.Cache.Manifest) !bool {
546582 s.result_cached = man.hit() catch |err| return failWithCacheError(s, man, err);
547583 return s.result_cached;
548584}
549585
550fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyerror) anyerror {
586fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: anyerror) anyerror {
551587 const i = man.failed_file_index orelse return err;
552588 const pp = man.files.keys()[i].prefixed_path;
553589 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
554590 return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path });
555591}
556592
557pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void {
593pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void {
558594 if (s.test_results.isSuccess()) {
559595 man.writeManifest() catch |err| {
560596 try s.addError("unable to write cache manifest: {s}", .{@errorName(err)});
......@@ -568,44 +604,37 @@ fn oom(err: anytype) noreturn {
568604 }
569605}
570606
571pub fn addWatchInput(step: *Step, lazy_path: std.Build.LazyPath) void {
607pub fn addWatchInput(step: *Step, lazy_path: Build.LazyPath) void {
572608 errdefer |err| oom(err);
573 const w = step.owner.graph.watch orelse return;
574609 switch (lazy_path) {
575 .src_path => |src_path| try addWatchInputFromBuilder(step, w, src_path.owner, src_path.sub_path),
576 .dependency => |d| try addWatchInputFromBuilder(step, w, d.dependency.builder, d.sub_path),
610 .src_path => |src_path| try addWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
611 .dependency => |d| try addWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
577612 .cwd_relative => |path_string| {
578 try addWatchInputFromPath(w, .{
613 try addWatchInputFromPath(step, .{
579614 .root_dir = .{
580615 .path = null,
581616 .handle = std.fs.cwd(),
582617 },
583618 .sub_path = std.fs.path.dirname(path_string) orelse "",
584 }, .{
585 .step = step,
586 .basename = std.fs.path.basename(path_string),
587 });
619 }, std.fs.path.basename(path_string));
588620 },
589621 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
590622 .generated => {},
591623 }
592624}
593625
594fn addWatchInputFromBuilder(step: *Step, w: *std.Build.Watch, builder: *std.Build, sub_path: []const u8) !void {
595 return addWatchInputFromPath(w, .{
626fn addWatchInputFromBuilder(step: *Step, builder: *Build, sub_path: []const u8) !void {
627 return addWatchInputFromPath(step, .{
596628 .root_dir = builder.build_root,
597629 .sub_path = std.fs.path.dirname(sub_path) orelse "",
598 }, .{
599 .step = step,
600 .basename = std.fs.path.basename(sub_path),
601 });
630 }, std.fs.path.basename(sub_path));
602631}
603632
604fn addWatchInputFromPath(w: *std.Build.Watch, path: std.Build.Cache.Path, match: std.Build.Watch.Match) !void {
605 const gpa = match.step.owner.allocator;
606 const gop = try w.table.getOrPut(gpa, path);
633fn addWatchInputFromPath(step: *Step, path: Build.Cache.Path, basename: []const u8) !void {
634 const gpa = step.owner.allocator;
635 const gop = try step.inputs.table.getOrPut(gpa, path);
607636 if (!gop.found_existing) gop.value_ptr.* = .{};
608 try gop.value_ptr.put(gpa, match, {});
637 try gop.value_ptr.append(gpa, basename);
609638}
610639
611640test {
lib/std/Build/Step/InstallFile.zig+4-1
......@@ -39,7 +39,10 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
3939 _ = prog_node;
4040 const b = step.owner;
4141 const install_file: *InstallFile = @fieldParentPtr("step", step);
42 step.addWatchInput(install_file.source);
42
43 // Inputs never change when re-running `make`.
44 if (!step.inputs.populated()) step.addWatchInput(install_file.source);
45
4346 const full_src_path = install_file.source.getPath2(b, step);
4447 const full_dest_path = b.getInstallPath(install_file.dir, install_file.dest_rel_path);
4548 const cwd = std.fs.cwd();