authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-09 18:17:27-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:14:07-07:00
log5ee3971b1828468c89104cb01e19edc87edf35a6
tree88e1173279e20aa0087ca0a23a1c13bcfd6c457a
parentc5a4177140f417b80c3d2e86f247ee5af769a39a

proof-of-concept --watch implementation based on fanotify

So far, only implemented for InstallFile steps. Default debounce interval bumped to 50ms. I think it should be configurable. Next I have an idea to simplify the fanotify implementation, but other OS implementations might want to refer back to this commit before I make those changes.

3 files changed, 208 insertions(+), 148 deletions(-)

lib/compiler/build_runner.zig+72-148
......@@ -8,6 +8,7 @@ const process = std.process;
88const ArrayList = std.ArrayList;
99const File = std.fs.File;
1010const Step = std.Build.Step;
11const Watch = std.Build.Watch;
1112const Allocator = std.mem.Allocator;
1213
1314pub const root = @import("@build");
......@@ -400,34 +401,26 @@ pub fn main() !void {
400401 };
401402 if (!watch) return cleanExit();
402403
403 // Clear all file handles.
404 for (w.handle_table.keys(), w.handle_table.values()) |lfh, *step_set| {
405 lfh.destroy(gpa);
406 step_set.clearAndFree(gpa);
407 }
408 w.handle_table.clearRetainingCapacity();
409
410404 // Add missing marks and note persisted ones.
411405 for (run.step_stack.keys()) |step| {
412406 for (step.inputs.table.keys(), step.inputs.table.values()) |path, *files| {
413 {
407 const reaction_set = rs: {
414408 const gop = try w.dir_table.getOrPut(gpa, path);
415 gop.value_ptr.* = w.generation;
416409 if (!gop.found_existing) {
417410 try std.posix.fanotify_mark(w.fan_fd, .{
418411 .ADD = true,
419412 .ONLYDIR = true,
420413 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt());
414
415 const dir_handle = try Watch.getDirHandle(gpa, path);
416 try w.handle_table.putNoClobber(gpa, dir_handle, .{});
421417 }
422 }
418 break :rs &w.handle_table.values()[gop.index];
419 };
423420 for (files.items) |basename| {
424 const file_handle = try Watch.getFileHandle(gpa, path, basename);
425 std.debug.print("watching file_handle '{}{s}' = {}\n", .{
426 path, basename, std.fmt.fmtSliceHexLower(file_handle.slice()),
427 });
428 const gop = try w.handle_table.getOrPut(gpa, file_handle);
421 const gop = try reaction_set.getOrPut(gpa, basename);
429422 if (!gop.found_existing) gop.value_ptr.* = .{};
430 try gop.value_ptr.put(gpa, step, {});
423 try gop.value_ptr.put(gpa, step, w.generation);
431424 }
432425 }
433426 }
......@@ -435,11 +428,31 @@ pub fn main() !void {
435428 {
436429 // Remove marks for files that are no longer inputs.
437430 var i: usize = 0;
438 while (i < w.dir_table.entries.len) {
439 const generations = w.dir_table.values();
440 if (generations[i] == w.generation) {
441 i += 1;
442 continue;
431 while (i < w.handle_table.entries.len) {
432 {
433 const reaction_set = &w.handle_table.values()[i];
434 var step_set_i: usize = 0;
435 while (step_set_i < reaction_set.entries.len) {
436 const step_set = &reaction_set.values()[step_set_i];
437 var dirent_i: usize = 0;
438 while (dirent_i < step_set.entries.len) {
439 const generations = step_set.values();
440 if (generations[dirent_i] == w.generation) {
441 dirent_i += 1;
442 continue;
443 }
444 step_set.swapRemoveAt(dirent_i);
445 }
446 if (step_set.entries.len > 0) {
447 step_set_i += 1;
448 continue;
449 }
450 reaction_set.swapRemoveAt(step_set_i);
451 }
452 if (reaction_set.entries.len > 0) {
453 i += 1;
454 continue;
455 }
443456 }
444457
445458 const path = w.dir_table.keys()[i];
......@@ -450,6 +463,7 @@ pub fn main() !void {
450463 }, Watch.fan_mask, path.root_dir.handle.fd, path.subPathOpt());
451464
452465 w.dir_table.swapRemoveAt(i);
466 w.handle_table.swapRemoveAt(i);
453467 }
454468 w.generation +%= 1;
455469 }
......@@ -459,7 +473,7 @@ pub fn main() !void {
459473 // if any more events come in. After the debounce interval has passed,
460474 // trigger a rebuild on all steps with modified inputs, as well as their
461475 // recursive dependants.
462 const debounce_interval_ms = 10;
476 const debounce_interval_ms = 50;
463477 var poll_fds: [1]std.posix.pollfd = .{
464478 .{
465479 .fd = w.fan_fd,
......@@ -517,46 +531,48 @@ fn markDirtySteps(w: *Watch) !bool {
517531 const file_name = mem.span(file_name_z);
518532 std.debug.print("DFID_NAME file_handle = {any}, found: '{s}'\n", .{ file_handle.*, file_name });
519533 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
520 if (w.handle_table.get(lfh)) |step_set| {
521 for (step_set.keys()) |step| {
522 std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name});
523 step.state = .precheck_done;
524 any_dirty = true;
534 if (w.handle_table.getPtr(lfh)) |reaction_set| {
535 if (reaction_set.getPtr(file_name)) |step_set| {
536 for (step_set.keys()) |step| {
537 std.debug.print("DFID_NAME marking step '{s}' dirty\n", .{step.name});
538 step.state = .precheck_done;
539 any_dirty = true;
540 }
525541 }
526542 } else {
527 std.debug.print("DFID_NAME changed file did not match any steps: '{}'\n", .{
543 std.debug.print("DFID_NAME changed file did not match any directories: '{}'\n", .{
528544 std.fmt.fmtSliceHexLower(lfh.slice()),
529545 });
530546 }
531547 },
532 .FID => {
533 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
534 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
535 if (w.handle_table.get(lfh)) |step_set| {
536 for (step_set.keys()) |step| {
537 std.debug.print("FID marking step '{s}' dirty\n", .{step.name});
538 step.state = .precheck_done;
539 any_dirty = true;
540 }
541 } else {
542 std.debug.print("FID changed file did not match any steps: '{}'\n", .{
543 std.fmt.fmtSliceHexLower(lfh.slice()),
544 });
545 }
546 },
547 .DFID => {
548 const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
549 const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
550 if (w.handle_table.get(lfh)) |step_set| {
551 for (step_set.keys()) |step| {
552 std.debug.print("DFID marking step '{s}' dirty\n", .{step.name});
553 step.state = .precheck_done;
554 any_dirty = true;
555 }
556 } else {
557 std.debug.print("DFID changed file did not match any steps\n", .{});
558 }
559 },
548 //.FID => {
549 // const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
550 // const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
551 // if (w.handle_table.get(lfh)) |step_set| {
552 // for (step_set.keys()) |step| {
553 // std.debug.print("FID marking step '{s}' dirty\n", .{step.name});
554 // step.state = .precheck_done;
555 // any_dirty = true;
556 // }
557 // } else {
558 // std.debug.print("FID changed file did not match any steps: '{}'\n", .{
559 // std.fmt.fmtSliceHexLower(lfh.slice()),
560 // });
561 // }
562 //},
563 //.DFID => {
564 // const file_handle: *align(1) std.os.linux.file_handle = @ptrCast(&fid.handle);
565 // const lfh: Watch.LinuxFileHandle = .{ .handle = file_handle };
566 // if (w.handle_table.get(lfh)) |step_set| {
567 // for (step_set.keys()) |step| {
568 // std.debug.print("DFID marking step '{s}' dirty\n", .{step.name});
569 // step.state = .precheck_done;
570 // any_dirty = true;
571 // }
572 // } else {
573 // std.debug.print("DFID changed file did not match any steps\n", .{});
574 // }
575 //},
560576 else => |t| {
561577 std.debug.panic("TODO: received event type '{s}'", .{@tagName(t)});
562578 },
......@@ -565,98 +581,6 @@ fn markDirtySteps(w: *Watch) !bool {
565581 }
566582}
567583
568const Watch = struct {
569 dir_table: DirTable,
570 handle_table: HandleTable,
571 fan_fd: std.posix.fd_t,
572 generation: u8,
573
574 const fan_mask: std.os.linux.fanotify.MarkMask = .{
575 .CLOSE_WRITE = true,
576 .DELETE = true,
577 .MOVED_FROM = true,
578 .MOVED_TO = true,
579 .EVENT_ON_CHILD = true,
580 };
581
582 const init: Watch = .{
583 .dir_table = .{},
584 .handle_table = .{},
585 .fan_fd = -1,
586 .generation = 0,
587 };
588
589 /// Key is the directory to watch which contains one or more files we are
590 /// interested in noticing changes to.
591 ///
592 /// Value is generation.
593 const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, u8, Cache.Path.TableAdapter, false);
594
595 const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, StepSet, LinuxFileHandle.Adapter, false);
596 const StepSet = std.AutoArrayHashMapUnmanaged(*Step, void);
597
598 const Hash = std.hash.Wyhash;
599 const Cache = std.Build.Cache;
600
601 const LinuxFileHandle = struct {
602 handle: *align(1) std.os.linux.file_handle,
603
604 fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle {
605 const bytes = lfh.slice();
606 const new_ptr = try gpa.alignedAlloc(
607 u8,
608 @alignOf(std.os.linux.file_handle),
609 @sizeOf(std.os.linux.file_handle) + bytes.len,
610 );
611 const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr);
612 new_header.* = lfh.handle.*;
613 const new: LinuxFileHandle = .{ .handle = new_header };
614 @memcpy(new.slice(), lfh.slice());
615 return new;
616 }
617
618 fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void {
619 const ptr: [*]u8 = @ptrCast(lfh.handle);
620 const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes];
621 return gpa.free(allocated_slice);
622 }
623
624 fn slice(lfh: LinuxFileHandle) []u8 {
625 const ptr: [*]u8 = &lfh.handle.f_handle;
626 return ptr[0..lfh.handle.handle_bytes];
627 }
628
629 const Adapter = struct {
630 pub fn hash(self: Adapter, a: LinuxFileHandle) u32 {
631 _ = self;
632 const unsigned_type: u32 = @bitCast(a.handle.handle_type);
633 return @truncate(Hash.hash(unsigned_type, a.slice()));
634 }
635 pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool {
636 _ = self;
637 _ = b_index;
638 return a.handle.handle_type == b.handle.handle_type and mem.eql(u8, a.slice(), b.slice());
639 }
640 };
641 };
642
643 fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle {
644 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
645 var mount_id: i32 = undefined;
646 var buf: [std.fs.max_path_bytes]u8 = undefined;
647 const joined_path = if (path.sub_path.len == 0) basename else path: {
648 break :path std.fmt.bufPrint(&buf, "{s}" ++ std.fs.path.sep_str ++ "{s}", .{
649 path.sub_path, basename,
650 }) catch return error.NameTooLong;
651 };
652 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
653 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
654 try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0);
655 const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr };
656 return stack_lfh.clone(gpa);
657 }
658};
659
660584const Run = struct {
661585 max_rss: u64,
662586 max_rss_is_default: bool,
lib/std/Build.zig+1
......@@ -20,6 +20,7 @@ const Build = @This();
2020pub const Cache = @import("Build/Cache.zig");
2121pub const Step = @import("Build/Step.zig");
2222pub const Module = @import("Build/Module.zig");
23pub const Watch = @import("Build/Watch.zig");
2324
2425/// Shared state among all Build instances.
2526graph: *Graph,
lib/std/Build/Watch.zig created+135
......@@ -0,0 +1,135 @@
1const std = @import("../std.zig");
2const Watch = @This();
3const Step = std.Build.Step;
4const Allocator = std.mem.Allocator;
5
6dir_table: DirTable,
7/// Keyed differently but indexes correspond 1:1 with `dir_table`.
8handle_table: HandleTable,
9fan_fd: std.posix.fd_t,
10generation: Generation,
11
12pub const fan_mask: std.os.linux.fanotify.MarkMask = .{
13 .CLOSE_WRITE = true,
14 .DELETE = true,
15 .MOVED_FROM = true,
16 .MOVED_TO = true,
17 .EVENT_ON_CHILD = true,
18};
19
20pub const init: Watch = .{
21 .dir_table = .{},
22 .handle_table = .{},
23 .fan_fd = -1,
24 .generation = 0,
25};
26
27/// Key is the directory to watch which contains one or more files we are
28/// interested in noticing changes to.
29///
30/// Value is generation.
31const DirTable = std.ArrayHashMapUnmanaged(Cache.Path, void, Cache.Path.TableAdapter, false);
32
33const HandleTable = std.ArrayHashMapUnmanaged(LinuxFileHandle, ReactionSet, LinuxFileHandle.Adapter, false);
34const ReactionSet = std.StringArrayHashMapUnmanaged(StepSet);
35const StepSet = std.AutoArrayHashMapUnmanaged(*Step, Generation);
36
37const Generation = u8;
38
39const Hash = std.hash.Wyhash;
40const Cache = std.Build.Cache;
41
42pub const Match = struct {
43 /// Relative to the watched directory, the file path that triggers this
44 /// match.
45 basename: []const u8,
46 /// The step to re-run when file corresponding to `basename` is changed.
47 step: *Step,
48
49 pub const Context = struct {
50 pub fn hash(self: Context, a: Match) u32 {
51 _ = self;
52 var hasher = Hash.init(0);
53 std.hash.autoHash(&hasher, a.step);
54 hasher.update(a.basename);
55 return @truncate(hasher.final());
56 }
57 pub fn eql(self: Context, a: Match, b: Match, b_index: usize) bool {
58 _ = self;
59 _ = b_index;
60 return a.step == b.step and std.mem.eql(u8, a.basename, b.basename);
61 }
62 };
63};
64
65pub const LinuxFileHandle = struct {
66 handle: *align(1) std.os.linux.file_handle,
67
68 pub fn clone(lfh: LinuxFileHandle, gpa: Allocator) Allocator.Error!LinuxFileHandle {
69 const bytes = lfh.slice();
70 const new_ptr = try gpa.alignedAlloc(
71 u8,
72 @alignOf(std.os.linux.file_handle),
73 @sizeOf(std.os.linux.file_handle) + bytes.len,
74 );
75 const new_header: *std.os.linux.file_handle = @ptrCast(new_ptr);
76 new_header.* = lfh.handle.*;
77 const new: LinuxFileHandle = .{ .handle = new_header };
78 @memcpy(new.slice(), lfh.slice());
79 return new;
80 }
81
82 pub fn destroy(lfh: LinuxFileHandle, gpa: Allocator) void {
83 const ptr: [*]u8 = @ptrCast(lfh.handle);
84 const allocated_slice = ptr[0 .. @sizeOf(std.os.linux.file_handle) + lfh.handle.handle_bytes];
85 return gpa.free(allocated_slice);
86 }
87
88 pub fn slice(lfh: LinuxFileHandle) []u8 {
89 const ptr: [*]u8 = &lfh.handle.f_handle;
90 return ptr[0..lfh.handle.handle_bytes];
91 }
92
93 pub const Adapter = struct {
94 pub fn hash(self: Adapter, a: LinuxFileHandle) u32 {
95 _ = self;
96 const unsigned_type: u32 = @bitCast(a.handle.handle_type);
97 return @truncate(Hash.hash(unsigned_type, a.slice()));
98 }
99 pub fn eql(self: Adapter, a: LinuxFileHandle, b: LinuxFileHandle, b_index: usize) bool {
100 _ = self;
101 _ = b_index;
102 return a.handle.handle_type == b.handle.handle_type and std.mem.eql(u8, a.slice(), b.slice());
103 }
104 };
105};
106
107pub fn getFileHandle(gpa: Allocator, path: std.Build.Cache.Path, basename: []const u8) !LinuxFileHandle {
108 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
109 var mount_id: i32 = undefined;
110 var buf: [std.fs.max_path_bytes]u8 = undefined;
111 const joined_path = if (path.sub_path.len == 0) basename else path: {
112 break :path std.fmt.bufPrint(&buf, "{s}/{s}", .{
113 path.sub_path, basename,
114 }) catch return error.NameTooLong;
115 };
116 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
117 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
118 try std.posix.name_to_handle_at(path.root_dir.handle.fd, joined_path, stack_ptr, &mount_id, 0);
119 const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr };
120 return stack_lfh.clone(gpa);
121}
122
123pub fn getDirHandle(gpa: Allocator, path: std.Build.Cache.Path) !LinuxFileHandle {
124 var file_handle_buffer: [@sizeOf(std.os.linux.file_handle) + 128]u8 align(@alignOf(std.os.linux.file_handle)) = undefined;
125 var mount_id: i32 = undefined;
126 var buf: [std.fs.max_path_bytes]u8 = undefined;
127 const adjusted_path = if (path.sub_path.len == 0) "./" else std.fmt.bufPrint(&buf, "{s}/", .{
128 path.sub_path,
129 }) catch return error.NameTooLong;
130 const stack_ptr: *std.os.linux.file_handle = @ptrCast(&file_handle_buffer);
131 stack_ptr.handle_bytes = file_handle_buffer.len - @sizeOf(std.os.linux.file_handle);
132 try std.posix.name_to_handle_at(path.root_dir.handle.fd, adjusted_path, stack_ptr, &mount_id, std.os.linux.AT.HANDLE_FID);
133 const stack_lfh: LinuxFileHandle = .{ .handle = stack_ptr };
134 return stack_lfh.clone(gpa);
135}