authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-02 14:42:29-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-02 20:43:01-07:00
log252f4ab2a509c5b8b2e570f4a0de6cb706dd9588
treee195a777f8c28b81041d925495d66ffa4dad680a
parent434a6a4f633f3f9b29bd80f4089e8f403aab4f9b

build system: implement lazy dependencies, part 1

Build manifest files support `lazy: true` for dependency sections. This causes the auto-generated dependencies.zig to have 2 more possibilities: 1. It communicates whether a dependency is lazy or not. 2. The dependency might be acknowledged, but missing due to being lazy and not fetched. Lazy dependencies are not fetched by default, but if they are already fetched then they are provided to the build script. The build runner reports the set of missing lazy dependenices that are required to the parent process via stdout and indicates the situation with exit code 3. std.Build now has a `lazyDependency` function. I'll let the doc comments speak for themselves: When this function is called, it means that the current build does, in fact, require this dependency. If the dependency is already fetched, it proceeds in the same manner as `dependency`. However if the dependency was not fetched, then when the build script is finished running, the build will not proceed to the make phase. Instead, the parent process will additionally fetch all the lazy dependencies that were actually required by running the build script, rebuild the build script, and then run it again. In other words, if this function returns `null` it means that the only purpose of completing the configure phase is to find out all the other lazy dependencies that are also required. It is allowed to use this function for non-lazy dependencies, in which case it will never return `null`. This allows toggling laziness via build.zig.zon without changing build.zig logic. The CLI for `zig build` detects this situation, but the logic for then redoing the build process with these extra dependencies fetched is not yet implemented.

5 files changed, 162 insertions(+), 16 deletions(-)

lib/build_runner.zig+14-4
......@@ -100,8 +100,6 @@ pub fn main() !void {
100100 var help_menu: bool = false;
101101 var steps_menu: bool = false;
102102
103 const stdout_stream = io.getStdOut().writer();
104
105103 while (nextArg(args, &arg_idx)) |arg| {
106104 if (mem.startsWith(u8, arg, "-D")) {
107105 const option_contents = arg[2..];
......@@ -308,17 +306,29 @@ pub fn main() !void {
308306 try builder.runBuild(root);
309307 }
310308
309 if (graph.needed_lazy_dependencies.entries.len != 0) {
310 var buffer: std.ArrayListUnmanaged(u8) = .{};
311 for (graph.needed_lazy_dependencies.keys()) |k| {
312 try buffer.appendSlice(arena, k);
313 try buffer.append(arena, '\n');
314 }
315 try io.getStdOut().writeAll(buffer.items);
316 process.exit(3); // Indicate configure phase failed with meaningful stdout.
317 }
318
311319 if (builder.validateUserInputDidItFail()) {
312320 fatal(" access the help menu with 'zig build -h'", .{});
313321 }
314322
315323 validateSystemLibraryOptions(builder);
316324
325 const stdout_writer = io.getStdOut().writer();
326
317327 if (help_menu)
318 return usage(builder, stdout_stream);
328 return usage(builder, stdout_writer);
319329
320330 if (steps_menu)
321 return steps(builder, stdout_stream);
331 return steps(builder, stdout_writer);
322332
323333 var run: Run = .{
324334 .max_rss = max_rss,
lib/std/Build.zig+51-8
......@@ -117,6 +117,7 @@ pub const Graph = struct {
117117 env_map: EnvMap,
118118 global_cache_root: Cache.Directory,
119119 host_query_options: std.Target.Query.ParseOptions = .{},
120 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
120121};
121122
122123const AvailableDeps = []const struct { []const u8, []const u8 };
......@@ -1802,21 +1803,63 @@ pub const Dependency = struct {
18021803 }
18031804};
18041805
1805pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
1806fn findPkgHashOrFatal(b: *Build, name: []const u8) []const u8 {
1807 for (b.available_deps) |dep| {
1808 if (mem.eql(u8, dep[0], name)) return dep[1];
1809 }
1810
1811 const full_path = b.pathFromRoot("build.zig.zon");
1812 std.debug.panic("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file", .{ name, full_path });
1813}
1814
1815fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {
1816 b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM");
1817}
1818
1819/// When this function is called, it means that the current build does, in
1820/// fact, require this dependency. If the dependency is already fetched, it
1821/// proceeds in the same manner as `dependency`. However if the dependency was
1822/// not fetched, then when the build script is finished running, the build will
1823/// not proceed to the make phase. Instead, the parent process will
1824/// additionally fetch all the lazy dependencies that were actually required by
1825/// running the build script, rebuild the build script, and then run it again.
1826/// In other words, if this function returns `null` it means that the only
1827/// purpose of completing the configure phase is to find out all the other lazy
1828/// dependencies that are also required.
1829/// It is allowed to use this function for non-lazy dependencies, in which case
1830/// it will never return `null`. This allows toggling laziness via
1831/// build.zig.zon without changing build.zig logic.
1832pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {
18061833 const build_runner = @import("root");
18071834 const deps = build_runner.dependencies;
1835 const pkg_hash = findPkgHashOrFatal(b, name);
18081836
1809 const pkg_hash = for (b.available_deps) |dep| {
1810 if (mem.eql(u8, dep[0], name)) break dep[1];
1811 } else {
1812 const full_path = b.pathFromRoot("build.zig.zon");
1813 std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });
1814 process.exit(1);
1815 };
1837 inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
1838 if (mem.eql(u8, decl.name, pkg_hash)) {
1839 const pkg = @field(deps.packages, decl.name);
1840 const available = !@hasDecl(pkg, "available") or pkg.available;
1841 if (!available) {
1842 markNeededLazyDep(b, pkg_hash);
1843 return null;
1844 }
1845 return dependencyInner(b, name, pkg.build_root, if (@hasDecl(pkg, "build_zig")) pkg.build_zig else null, pkg.deps, args);
1846 }
1847 }
1848
1849 unreachable; // Bad @dependencies source
1850}
1851
1852pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
1853 const build_runner = @import("root");
1854 const deps = build_runner.dependencies;
1855 const pkg_hash = findPkgHashOrFatal(b, name);
18161856
18171857 inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
18181858 if (mem.eql(u8, decl.name, pkg_hash)) {
18191859 const pkg = @field(deps.packages, decl.name);
1860 if (@hasDecl(pkg, "available")) {
1861 @compileError("dependency is marked as lazy in build.zig.zon which means it must use the lazyDependency function instead");
1862 }
18201863 return dependencyInner(b, name, pkg.build_root, if (@hasDecl(pkg, "build_zig")) pkg.build_zig else null, pkg.deps, args);
18211864 }
18221865 }
src/Package/Fetch.zig+50-2
......@@ -31,6 +31,8 @@ arena: std.heap.ArenaAllocator,
3131location: Location,
3232location_tok: std.zig.Ast.TokenIndex,
3333hash_tok: std.zig.Ast.TokenIndex,
34name_tok: std.zig.Ast.TokenIndex,
35lazy_status: LazyStatus,
3436parent_package_root: Package.Path,
3537parent_manifest_ast: ?*const std.zig.Ast,
3638prog_node: *std.Progress.Node,
......@@ -64,6 +66,15 @@ oom_flag: bool,
6466/// the root source file.
6567module: ?*Package.Module,
6668
69pub const LazyStatus = enum {
70 /// Not lazy.
71 eager,
72 /// Lazy, found.
73 available,
74 /// Lazy, not found.
75 unavailable,
76};
77
6778/// Contains shared state among all `Fetch` tasks.
6879pub const JobQueue = struct {
6980 mutex: std.Thread.Mutex = .{},
......@@ -150,11 +161,37 @@ pub const JobQueue = struct {
150161 // The first one is a dummy package for the current project.
151162 continue;
152163 }
164
153165 try buf.writer().print(
154166 \\ pub const {} = struct {{
167 \\
168 , .{std.zig.fmtId(&hash)});
169
170 lazy: {
171 switch (fetch.lazy_status) {
172 .eager => break :lazy,
173 .available => {
174 try buf.appendSlice(
175 \\ pub const available = true;
176 \\
177 );
178 break :lazy;
179 },
180 .unavailable => {
181 try buf.appendSlice(
182 \\ pub const available = false;
183 \\ };
184 \\
185 );
186 continue;
187 },
188 }
189 }
190
191 try buf.writer().print(
155192 \\ pub const build_root = "{q}";
156193 \\
157 , .{ std.zig.fmtId(&hash), fetch.package_root });
194 , .{fetch.package_root});
158195
159196 if (fetch.has_build_zig) {
160197 try buf.writer().print(
......@@ -325,6 +362,7 @@ pub fn run(f: *Fetch) RunError!void {
325362 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
326363 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
327364 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
365 assert(f.lazy_status != .unavailable);
328366 f.package_root = .{
329367 .root_dir = cache_root,
330368 .sub_path = try arena.dupe(u8, pkg_sub_path),
......@@ -335,8 +373,16 @@ pub fn run(f: *Fetch) RunError!void {
335373 return queueJobsForDeps(f);
336374 } else |err| switch (err) {
337375 error.FileNotFound => {
376 switch (f.lazy_status) {
377 .eager => {},
378 .available => {
379 f.lazy_status = .unavailable;
380 return;
381 },
382 .unavailable => unreachable,
383 }
338384 if (f.job_queue.read_only) return f.fail(
339 f.location_tok,
385 f.name_tok,
340386 try eb.printString("package not found at '{}{s}'", .{
341387 cache_root, pkg_sub_path,
342388 }),
......@@ -627,6 +673,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
627673 .location = location,
628674 .location_tok = dep.location_tok,
629675 .hash_tok = dep.hash_tok,
676 .name_tok = dep.name_tok,
677 .lazy_status = if (dep.lazy) .available else .eager,
630678 .parent_package_root = f.package_root,
631679 .parent_manifest_ast = &f.manifest_ast,
632680 .prog_node = f.prog_node,
src/Package/Manifest.zig+28
......@@ -12,6 +12,8 @@ pub const Dependency = struct {
1212 hash: ?[]const u8,
1313 hash_tok: Ast.TokenIndex,
1414 node: Ast.Node.Index,
15 name_tok: Ast.TokenIndex,
16 lazy: bool,
1517
1618 pub const Location = union(enum) {
1719 url: []const u8,
......@@ -303,11 +305,14 @@ const Parse = struct {
303305 .hash = null,
304306 .hash_tok = 0,
305307 .node = node,
308 .name_tok = 0,
309 .lazy = false,
306310 };
307311 var has_location = false;
308312
309313 for (struct_init.ast.fields) |field_init| {
310314 const name_token = ast.firstToken(field_init) - 2;
315 dep.name_tok = name_token;
311316 const field_name = try identifierTokenString(p, name_token);
312317 // We could get fancy with reflection and comptime logic here but doing
313318 // things manually provides an opportunity to do any additional verification
......@@ -342,6 +347,11 @@ const Parse = struct {
342347 else => |e| return e,
343348 };
344349 dep.hash_tok = main_tokens[field_init];
350 } else if (mem.eql(u8, field_name, "lazy")) {
351 dep.lazy = parseBool(p, field_init) catch |err| switch (err) {
352 error.ParseFailure => continue,
353 else => |e| return e,
354 };
345355 } else {
346356 // Ignore unknown fields so that we can add fields in future zig
347357 // versions without breaking older zig versions.
......@@ -374,6 +384,24 @@ const Parse = struct {
374384 }
375385 }
376386
387 fn parseBool(p: *Parse, node: Ast.Node.Index) !bool {
388 const ast = p.ast;
389 const node_tags = ast.nodes.items(.tag);
390 const main_tokens = ast.nodes.items(.main_token);
391 if (node_tags[node] != .identifier) {
392 return fail(p, main_tokens[node], "expected identifier", .{});
393 }
394 const ident_token = main_tokens[node];
395 const token_bytes = ast.tokenSlice(ident_token);
396 if (mem.eql(u8, token_bytes, "true")) {
397 return true;
398 } else if (mem.eql(u8, token_bytes, "false")) {
399 return false;
400 } else {
401 return fail(p, ident_token, "expected boolean", .{});
402 }
403 }
404
377405 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
378406 const ast = p.ast;
379407 const node_tags = ast.nodes.items(.tag);
src/main.zig+19-2
......@@ -5464,6 +5464,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
54645464 .location = .{ .relative_path = build_mod.root },
54655465 .location_tok = 0,
54665466 .hash_tok = 0,
5467 .name_tok = 0,
5468 .lazy_status = .eager,
54675469 .parent_package_root = build_mod.root,
54685470 .parent_manifest_ast = null,
54695471 .prog_node = root_prog_node,
......@@ -5618,10 +5620,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
56185620 if (process.can_spawn) {
56195621 var child = std.ChildProcess.init(child_argv, gpa);
56205622 child.stdin_behavior = .Inherit;
5621 child.stdout_behavior = .Inherit;
5623 child.stdout_behavior = .Pipe;
56225624 child.stderr_behavior = .Inherit;
56235625
5624 const term = try child.spawnAndWait();
5626 try child.spawn();
5627 // Since only one output stream is piped, we can simply do a blocking
5628 // read until the stream is finished.
5629 const stdout = try child.stdout.?.readToEndAlloc(arena, 50 * 1024 * 1024);
5630 const term = try child.wait();
56255631 switch (term) {
56265632 .Exited => |code| {
56275633 if (code == 0) return cleanExit();
......@@ -5630,6 +5636,15 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
56305636 // diagnostics.
56315637 if (code == 2) process.exit(2);
56325638
5639 if (code == 3) {
5640 // Indicates the configure phase failed due to missing lazy
5641 // dependencies and stdout contains the hashes of the ones
5642 // that are missing.
5643 std.debug.print("missing lazy dependencies: '{s}'\n", .{stdout});
5644 std.debug.print("TODO: fetch them and rebuild the build script\n", .{});
5645 process.exit(1);
5646 }
5647
56335648 const cmd = try std.mem.join(arena, " ", child_argv);
56345649 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
56355650 },
......@@ -7395,6 +7410,8 @@ fn cmdFetch(
73957410 .location = .{ .path_or_url = path_or_url },
73967411 .location_tok = 0,
73977412 .hash_tok = 0,
7413 .name_tok = 0,
7414 .lazy_status = .eager,
73987415 .parent_package_root = undefined,
73997416 .parent_manifest_ast = null,
74007417 .prog_node = root_prog_node,