authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-10 15:09:46-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-12 00:14:08-07:00
loga966eee090d55c7d61484333af675c80115bf188
tree9df97f8551ce59a4f163d4bd23b145c5bcb44858
parentf2856403c6997ff1317c968abed0871df9586c7c

std.Build.Step.WriteFile: fix handling of directories

and add file system watching integration. `addDirectoryWatchInput` now returns a `bool` which helps remind the caller to 1. call addDirectoryWatchInputFromPath on any derived paths 2. but only if the dependency is not already captured by a step dependency edge. The make function now recursively walks all directories and adds the found files to the cache hash rather than incorrectly only adding the directory name to the cache hash. closes #20571

3 files changed, 95 insertions(+), 43 deletions(-)

lib/std/Build/Step.zig+12-2
......@@ -634,7 +634,11 @@ pub fn addWatchInput(step: *Step, lazy_file: Build.LazyPath) Allocator.Error!voi
634634/// Any changes inside the directory will trigger invalidation.
635635///
636636/// See also `addDirectoryWatchInputFromPath` which takes a `Build.Cache.Path` instead.
637pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!void {
637///
638/// Paths derived from this directory should also be manually added via
639/// `addDirectoryWatchInputFromPath` if and only if this function returns
640/// `true`.
641pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Allocator.Error!bool {
638642 switch (lazy_directory) {
639643 .src_path => |src_path| try addDirectoryWatchInputFromBuilder(step, src_path.owner, src_path.sub_path),
640644 .dependency => |d| try addDirectoryWatchInputFromBuilder(step, d.dependency.builder, d.sub_path),
......@@ -648,13 +652,19 @@ pub fn addDirectoryWatchInput(step: *Step, lazy_directory: Build.LazyPath) Alloc
648652 });
649653 },
650654 // Nothing to watch because this dependency edge is modeled instead via `dependants`.
651 .generated => {},
655 .generated => return false,
652656 }
657 return true;
653658}
654659
655660/// Any changes inside the directory will trigger invalidation.
656661///
657662/// See also `addDirectoryWatchInput` which takes a `Build.LazyPath` instead.
663///
664/// This function should only be called when it has been verified that the
665/// dependency on `path` is not already accounted for by a `Step` dependency.
666/// In other words, before calling this function, first check that the
667/// `Build.LazyPath` which this `path` is derived from is not `generated`.
658668pub fn addDirectoryWatchInputFromPath(step: *Step, path: Build.Cache.Path) !void {
659669 return addWatchInputFromPath(step, path, ".");
660670}
lib/std/Build/Step/InstallDir.zig+3-4
......@@ -63,8 +63,8 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
6363 const arena = b.allocator;
6464 const dest_prefix = b.getInstallPath(install_dir.options.install_dir, install_dir.options.install_subdir);
6565 const src_dir_path = install_dir.options.source_dir.getPath3(b, step);
66 try step.addDirectoryWatchInput(install_dir.options.source_dir);
67 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOpt() orelse ".", .{ .iterate = true }) catch |err| {
66 const need_derived_inputs = try step.addDirectoryWatchInput(install_dir.options.source_dir);
67 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
6868 return step.fail("unable to open source directory '{}': {s}", .{
6969 src_dir_path, @errorName(err),
7070 });
......@@ -96,8 +96,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
9696
9797 switch (entry.kind) {
9898 .directory => {
99 const subdir_path = try src_dir_path.join(arena, entry.path);
100 try step.addDirectoryWatchInputFromPath(subdir_path);
99 if (need_derived_inputs) try step.addDirectoryWatchInputFromPath(src_sub_path);
101100 try cwd.makePath(dest_path);
102101 // TODO: set result_cached=false if the directory did not already exist.
103102 },
lib/std/Build/Step/WriteFile.zig+80-37
......@@ -40,6 +40,22 @@ pub const Directory = struct {
4040 .include_extensions = if (opts.include_extensions) |incs| b.dupeStrings(incs) else null,
4141 };
4242 }
43
44 pub fn pathIncluded(opts: Options, path: []const u8) bool {
45 for (opts.exclude_extensions) |ext| {
46 if (std.mem.endsWith(u8, path, ext))
47 return false;
48 }
49 if (opts.include_extensions) |incs| {
50 for (incs) |inc| {
51 if (std.mem.endsWith(u8, path, inc))
52 return true;
53 } else {
54 return false;
55 }
56 }
57 return true;
58 }
4359 };
4460};
4561
......@@ -158,7 +174,10 @@ fn maybeUpdateName(write_file: *WriteFile) void {
158174fn make(step: *Step, prog_node: std.Progress.Node) !void {
159175 _ = prog_node;
160176 const b = step.owner;
177 const arena = b.allocator;
178 const gpa = arena;
161179 const write_file: *WriteFile = @fieldParentPtr("step", step);
180 step.clearWatchInputs();
162181
163182 // The cache is used here not really as a way to speed things up - because writing
164183 // the data to a file would probably be very fast - but as a way to find a canonical
......@@ -173,29 +192,67 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
173192 // Random bytes to make WriteFile unique. Refresh this with
174193 // new random bytes when WriteFile implementation is modified
175194 // in a non-backwards-compatible way.
176 man.hash.add(@as(u32, 0xd767ee59));
195 man.hash.add(@as(u32, 0xc2a287d0));
177196
178197 for (write_file.files.items) |file| {
179198 man.hash.addBytes(file.sub_path);
199
180200 switch (file.contents) {
181201 .bytes => |bytes| {
182202 man.hash.addBytes(bytes);
183203 },
184 .copy => |file_source| {
185 _ = try man.addFile(file_source.getPath2(b, step), null);
204 .copy => |lazy_path| {
205 const path = lazy_path.getPath3(b, step);
206 _ = try man.addFilePath(path, null);
207 try step.addWatchInput(lazy_path);
186208 },
187209 }
188210 }
189 for (write_file.directories.items) |dir| {
190 man.hash.addBytes(dir.source.getPath2(b, step));
211
212 const open_dir_cache = try arena.alloc(fs.Dir, write_file.directories.items.len);
213 var open_dirs_count: usize = 0;
214 defer closeDirs(open_dir_cache[0..open_dirs_count]);
215
216 for (write_file.directories.items, open_dir_cache) |dir, *open_dir_cache_elem| {
191217 man.hash.addBytes(dir.sub_path);
192218 for (dir.options.exclude_extensions) |ext| man.hash.addBytes(ext);
193219 if (dir.options.include_extensions) |incs| for (incs) |inc| man.hash.addBytes(inc);
220
221 const need_derived_inputs = try step.addDirectoryWatchInput(dir.source);
222 const src_dir_path = dir.source.getPath3(b, step);
223
224 var src_dir = src_dir_path.root_dir.handle.openDir(src_dir_path.subPathOrDot(), .{ .iterate = true }) catch |err| {
225 return step.fail("unable to open source directory '{}': {s}", .{
226 src_dir_path, @errorName(err),
227 });
228 };
229 open_dir_cache_elem.* = src_dir;
230 open_dirs_count += 1;
231
232 var it = try src_dir.walk(gpa);
233 defer it.deinit();
234 while (try it.next()) |entry| {
235 if (!dir.options.pathIncluded(entry.path)) continue;
236
237 switch (entry.kind) {
238 .directory => {
239 if (need_derived_inputs) {
240 const entry_path = try src_dir_path.join(arena, entry.path);
241 try step.addDirectoryWatchInputFromPath(entry_path);
242 }
243 },
244 .file => {
245 const entry_path = try src_dir_path.join(arena, entry.path);
246 _ = try man.addFilePath(entry_path, null);
247 },
248 else => continue,
249 }
250 }
194251 }
195252
196253 if (try step.cacheHit(&man)) {
197254 const digest = man.final();
198 write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest });
255 write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest });
199256 step.result_cached = true;
200257 return;
201258 }
......@@ -203,7 +260,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
203260 const digest = man.final();
204261 const cache_path = "o" ++ fs.path.sep_str ++ digest;
205262
206 write_file.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest });
263 write_file.generated_directory.path = try b.cache_root.join(arena, &.{ "o", &digest });
207264
208265 var cache_dir = b.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {
209266 return step.fail("unable to make path '{}{s}': {s}", .{
......@@ -256,8 +313,9 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
256313 },
257314 }
258315 }
259 for (write_file.directories.items) |dir| {
260 const full_src_dir_path = dir.source.getPath2(b, step);
316
317 for (write_file.directories.items, open_dir_cache) |dir, already_open_dir| {
318 const src_dir_path = dir.source.getPath3(b, step);
261319 const dest_dirname = dir.sub_path;
262320
263321 if (dest_dirname.len != 0) {
......@@ -268,44 +326,25 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
268326 };
269327 }
270328
271 var src_dir = b.build_root.handle.openDir(full_src_dir_path, .{ .iterate = true }) catch |err| {
272 return step.fail("unable to open source directory '{s}': {s}", .{
273 full_src_dir_path, @errorName(err),
274 });
275 };
276 defer src_dir.close();
329 var it = try already_open_dir.walk(gpa);
330 defer it.deinit();
331 while (try it.next()) |entry| {
332 if (!dir.options.pathIncluded(entry.path)) continue;
277333
278 var it = try src_dir.walk(b.allocator);
279 next_entry: while (try it.next()) |entry| {
280 for (dir.options.exclude_extensions) |ext| {
281 if (std.mem.endsWith(u8, entry.path, ext)) continue :next_entry;
282 }
283 if (dir.options.include_extensions) |incs| {
284 for (incs) |inc| {
285 if (std.mem.endsWith(u8, entry.path, inc)) break;
286 } else {
287 continue :next_entry;
288 }
289 }
290 const full_src_entry_path = b.pathJoin(&.{ full_src_dir_path, entry.path });
334 const src_entry_path = try src_dir_path.join(arena, entry.path);
291335 const dest_path = b.pathJoin(&.{ dest_dirname, entry.path });
292336 switch (entry.kind) {
293337 .directory => try cache_dir.makePath(dest_path),
294338 .file => {
295339 const prev_status = fs.Dir.updateFile(
296 cwd,
297 full_src_entry_path,
340 src_entry_path.root_dir.handle,
341 src_entry_path.sub_path,
298342 cache_dir,
299343 dest_path,
300344 .{},
301345 ) catch |err| {
302 return step.fail("unable to update file from '{s}' to '{}{s}{c}{s}': {s}", .{
303 full_src_entry_path,
304 b.cache_root,
305 cache_path,
306 fs.path.sep,
307 dest_path,
308 @errorName(err),
346 return step.fail("unable to update file from '{}' to '{}{s}{c}{s}': {s}", .{
347 src_entry_path, b.cache_root, cache_path, fs.path.sep, dest_path, @errorName(err),
309348 });
310349 };
311350 _ = prev_status;
......@@ -317,3 +356,7 @@ fn make(step: *Step, prog_node: std.Progress.Node) !void {
317356
318357 try step.writeManifest(&man);
319358}
359
360fn closeDirs(dirs: []fs.Dir) void {
361 for (dirs) |*d| d.close();
362}