authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-02 14:42:29-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-02-02 20:43:01-07:00
log252f4ab2a509c5b8b2e570f4a0de6cb706dd9588
treee195a777f8c28b81041d925495d66ffa4dad680a
parent434a6a4f633f3f9b29bd80f4089e8f403aab4f9b

build system: implement lazy dependencies, part 1

Build manifest files support `lazy: true` for dependency sections. This causes the auto-generated dependencies.zig to have 2 more possibilities: 1. It communicates whether a dependency is lazy or not. 2. The dependency might be acknowledged, but missing due to being lazy and not fetched. Lazy dependencies are not fetched by default, but if they are already fetched then they are provided to the build script. The build runner reports the set of missing lazy dependenices that are required to the parent process via stdout and indicates the situation with exit code 3. std.Build now has a `lazyDependency` function. I'll let the doc comments speak for themselves: When this function is called, it means that the current build does, in fact, require this dependency. If the dependency is already fetched, it proceeds in the same manner as `dependency`. However if the dependency was not fetched, then when the build script is finished running, the build will not proceed to the make phase. Instead, the parent process will additionally fetch all the lazy dependencies that were actually required by running the build script, rebuild the build script, and then run it again. In other words, if this function returns `null` it means that the only purpose of completing the configure phase is to find out all the other lazy dependencies that are also required. It is allowed to use this function for non-lazy dependencies, in which case it will never return `null`. This allows toggling laziness via build.zig.zon without changing build.zig logic. The CLI for `zig build` detects this situation, but the logic for then redoing the build process with these extra dependencies fetched is not yet implemented.

5 files changed, 162 insertions(+), 16 deletions(-)

lib/build_runner.zig+14-4
...@@ -100,8 +100,6 @@ pub fn main() !void {...@@ -100,8 +100,6 @@ pub fn main() !void {
100 var help_menu: bool = false;100 var help_menu: bool = false;
101 var steps_menu: bool = false;101 var steps_menu: bool = false;
102102
103 const stdout_stream = io.getStdOut().writer();
104
105 while (nextArg(args, &arg_idx)) |arg| {103 while (nextArg(args, &arg_idx)) |arg| {
106 if (mem.startsWith(u8, arg, "-D")) {104 if (mem.startsWith(u8, arg, "-D")) {
107 const option_contents = arg[2..];105 const option_contents = arg[2..];
...@@ -308,17 +306,29 @@ pub fn main() !void {...@@ -308,17 +306,29 @@ pub fn main() !void {
308 try builder.runBuild(root);306 try builder.runBuild(root);
309 }307 }
310308
309 if (graph.needed_lazy_dependencies.entries.len != 0) {
310 var buffer: std.ArrayListUnmanaged(u8) = .{};
311 for (graph.needed_lazy_dependencies.keys()) |k| {
312 try buffer.appendSlice(arena, k);
313 try buffer.append(arena, '\n');
314 }
315 try io.getStdOut().writeAll(buffer.items);
316 process.exit(3); // Indicate configure phase failed with meaningful stdout.
317 }
318
311 if (builder.validateUserInputDidItFail()) {319 if (builder.validateUserInputDidItFail()) {
312 fatal(" access the help menu with 'zig build -h'", .{});320 fatal(" access the help menu with 'zig build -h'", .{});
313 }321 }
314322
315 validateSystemLibraryOptions(builder);323 validateSystemLibraryOptions(builder);
316324
325 const stdout_writer = io.getStdOut().writer();
326
317 if (help_menu)327 if (help_menu)
318 return usage(builder, stdout_stream);328 return usage(builder, stdout_writer);
319329
320 if (steps_menu)330 if (steps_menu)
321 return steps(builder, stdout_stream);331 return steps(builder, stdout_writer);
322332
323 var run: Run = .{333 var run: Run = .{
324 .max_rss = max_rss,334 .max_rss = max_rss,
lib/std/Build.zig+51-8
...@@ -117,6 +117,7 @@ pub const Graph = struct {...@@ -117,6 +117,7 @@ pub const Graph = struct {
117 env_map: EnvMap,117 env_map: EnvMap,
118 global_cache_root: Cache.Directory,118 global_cache_root: Cache.Directory,
119 host_query_options: std.Target.Query.ParseOptions = .{},119 host_query_options: std.Target.Query.ParseOptions = .{},
120 needed_lazy_dependencies: std.StringArrayHashMapUnmanaged(void) = .{},
120};121};
121122
122const AvailableDeps = []const struct { []const u8, []const u8 };123const AvailableDeps = []const struct { []const u8, []const u8 };
...@@ -1802,21 +1803,63 @@ pub const Dependency = struct {...@@ -1802,21 +1803,63 @@ pub const Dependency = struct {
1802 }1803 }
1803};1804};
18041805
1805pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {1806fn findPkgHashOrFatal(b: *Build, name: []const u8) []const u8 {
1807 for (b.available_deps) |dep| {
1808 if (mem.eql(u8, dep[0], name)) return dep[1];
1809 }
1810
1811 const full_path = b.pathFromRoot("build.zig.zon");
1812 std.debug.panic("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file", .{ name, full_path });
1813}
1814
1815fn markNeededLazyDep(b: *Build, pkg_hash: []const u8) void {
1816 b.graph.needed_lazy_dependencies.put(b.graph.arena, pkg_hash, {}) catch @panic("OOM");
1817}
1818
1819/// When this function is called, it means that the current build does, in
1820/// fact, require this dependency. If the dependency is already fetched, it
1821/// proceeds in the same manner as `dependency`. However if the dependency was
1822/// not fetched, then when the build script is finished running, the build will
1823/// not proceed to the make phase. Instead, the parent process will
1824/// additionally fetch all the lazy dependencies that were actually required by
1825/// running the build script, rebuild the build script, and then run it again.
1826/// In other words, if this function returns `null` it means that the only
1827/// purpose of completing the configure phase is to find out all the other lazy
1828/// dependencies that are also required.
1829/// It is allowed to use this function for non-lazy dependencies, in which case
1830/// it will never return `null`. This allows toggling laziness via
1831/// build.zig.zon without changing build.zig logic.
1832pub fn lazyDependency(b: *Build, name: []const u8, args: anytype) ?*Dependency {
1806 const build_runner = @import("root");1833 const build_runner = @import("root");
1807 const deps = build_runner.dependencies;1834 const deps = build_runner.dependencies;
1835 const pkg_hash = findPkgHashOrFatal(b, name);
18081836
1809 const pkg_hash = for (b.available_deps) |dep| {1837 inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
1810 if (mem.eql(u8, dep[0], name)) break dep[1];1838 if (mem.eql(u8, decl.name, pkg_hash)) {
1811 } else {1839 const pkg = @field(deps.packages, decl.name);
1812 const full_path = b.pathFromRoot("build.zig.zon");1840 const available = !@hasDecl(pkg, "available") or pkg.available;
1813 std.debug.print("no dependency named '{s}' in '{s}'. All packages used in build.zig must be declared in this file.\n", .{ name, full_path });1841 if (!available) {
1814 process.exit(1);1842 markNeededLazyDep(b, pkg_hash);
1815 };1843 return null;
1844 }
1845 return dependencyInner(b, name, pkg.build_root, if (@hasDecl(pkg, "build_zig")) pkg.build_zig else null, pkg.deps, args);
1846 }
1847 }
1848
1849 unreachable; // Bad @dependencies source
1850}
1851
1852pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
1853 const build_runner = @import("root");
1854 const deps = build_runner.dependencies;
1855 const pkg_hash = findPkgHashOrFatal(b, name);
18161856
1817 inline for (@typeInfo(deps.packages).Struct.decls) |decl| {1857 inline for (@typeInfo(deps.packages).Struct.decls) |decl| {
1818 if (mem.eql(u8, decl.name, pkg_hash)) {1858 if (mem.eql(u8, decl.name, pkg_hash)) {
1819 const pkg = @field(deps.packages, decl.name);1859 const pkg = @field(deps.packages, decl.name);
1860 if (@hasDecl(pkg, "available")) {
1861 @compileError("dependency is marked as lazy in build.zig.zon which means it must use the lazyDependency function instead");
1862 }
1820 return dependencyInner(b, name, pkg.build_root, if (@hasDecl(pkg, "build_zig")) pkg.build_zig else null, pkg.deps, args);1863 return dependencyInner(b, name, pkg.build_root, if (@hasDecl(pkg, "build_zig")) pkg.build_zig else null, pkg.deps, args);
1821 }1864 }
1822 }1865 }
src/Package/Fetch.zig+50-2
...@@ -31,6 +31,8 @@ arena: std.heap.ArenaAllocator,...@@ -31,6 +31,8 @@ arena: std.heap.ArenaAllocator,
31location: Location,31location: Location,
32location_tok: std.zig.Ast.TokenIndex,32location_tok: std.zig.Ast.TokenIndex,
33hash_tok: std.zig.Ast.TokenIndex,33hash_tok: std.zig.Ast.TokenIndex,
34name_tok: std.zig.Ast.TokenIndex,
35lazy_status: LazyStatus,
34parent_package_root: Package.Path,36parent_package_root: Package.Path,
35parent_manifest_ast: ?*const std.zig.Ast,37parent_manifest_ast: ?*const std.zig.Ast,
36prog_node: *std.Progress.Node,38prog_node: *std.Progress.Node,
...@@ -64,6 +66,15 @@ oom_flag: bool,...@@ -64,6 +66,15 @@ oom_flag: bool,
64/// the root source file.66/// the root source file.
65module: ?*Package.Module,67module: ?*Package.Module,
6668
69pub const LazyStatus = enum {
70 /// Not lazy.
71 eager,
72 /// Lazy, found.
73 available,
74 /// Lazy, not found.
75 unavailable,
76};
77
67/// Contains shared state among all `Fetch` tasks.78/// Contains shared state among all `Fetch` tasks.
68pub const JobQueue = struct {79pub const JobQueue = struct {
69 mutex: std.Thread.Mutex = .{},80 mutex: std.Thread.Mutex = .{},
...@@ -150,11 +161,37 @@ pub const JobQueue = struct {...@@ -150,11 +161,37 @@ pub const JobQueue = struct {
150 // The first one is a dummy package for the current project.161 // The first one is a dummy package for the current project.
151 continue;162 continue;
152 }163 }
164
153 try buf.writer().print(165 try buf.writer().print(
154 \\ pub const {} = struct {{166 \\ pub const {} = struct {{
167 \\
168 , .{std.zig.fmtId(&hash)});
169
170 lazy: {
171 switch (fetch.lazy_status) {
172 .eager => break :lazy,
173 .available => {
174 try buf.appendSlice(
175 \\ pub const available = true;
176 \\
177 );
178 break :lazy;
179 },
180 .unavailable => {
181 try buf.appendSlice(
182 \\ pub const available = false;
183 \\ };
184 \\
185 );
186 continue;
187 },
188 }
189 }
190
191 try buf.writer().print(
155 \\ pub const build_root = "{q}";192 \\ pub const build_root = "{q}";
156 \\193 \\
157 , .{ std.zig.fmtId(&hash), fetch.package_root });194 , .{fetch.package_root});
158195
159 if (fetch.has_build_zig) {196 if (fetch.has_build_zig) {
160 try buf.writer().print(197 try buf.writer().print(
...@@ -325,6 +362,7 @@ pub fn run(f: *Fetch) RunError!void {...@@ -325,6 +362,7 @@ pub fn run(f: *Fetch) RunError!void {
325 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;362 const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
326 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];363 const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
327 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {364 if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
365 assert(f.lazy_status != .unavailable);
328 f.package_root = .{366 f.package_root = .{
329 .root_dir = cache_root,367 .root_dir = cache_root,
330 .sub_path = try arena.dupe(u8, pkg_sub_path),368 .sub_path = try arena.dupe(u8, pkg_sub_path),
...@@ -335,8 +373,16 @@ pub fn run(f: *Fetch) RunError!void {...@@ -335,8 +373,16 @@ pub fn run(f: *Fetch) RunError!void {
335 return queueJobsForDeps(f);373 return queueJobsForDeps(f);
336 } else |err| switch (err) {374 } else |err| switch (err) {
337 error.FileNotFound => {375 error.FileNotFound => {
376 switch (f.lazy_status) {
377 .eager => {},
378 .available => {
379 f.lazy_status = .unavailable;
380 return;
381 },
382 .unavailable => unreachable,
383 }
338 if (f.job_queue.read_only) return f.fail(384 if (f.job_queue.read_only) return f.fail(
339 f.location_tok,385 f.name_tok,
340 try eb.printString("package not found at '{}{s}'", .{386 try eb.printString("package not found at '{}{s}'", .{
341 cache_root, pkg_sub_path,387 cache_root, pkg_sub_path,
342 }),388 }),
...@@ -627,6 +673,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {...@@ -627,6 +673,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
627 .location = location,673 .location = location,
628 .location_tok = dep.location_tok,674 .location_tok = dep.location_tok,
629 .hash_tok = dep.hash_tok,675 .hash_tok = dep.hash_tok,
676 .name_tok = dep.name_tok,
677 .lazy_status = if (dep.lazy) .available else .eager,
630 .parent_package_root = f.package_root,678 .parent_package_root = f.package_root,
631 .parent_manifest_ast = &f.manifest_ast,679 .parent_manifest_ast = &f.manifest_ast,
632 .prog_node = f.prog_node,680 .prog_node = f.prog_node,
src/Package/Manifest.zig+28
...@@ -12,6 +12,8 @@ pub const Dependency = struct {...@@ -12,6 +12,8 @@ pub const Dependency = struct {
12 hash: ?[]const u8,12 hash: ?[]const u8,
13 hash_tok: Ast.TokenIndex,13 hash_tok: Ast.TokenIndex,
14 node: Ast.Node.Index,14 node: Ast.Node.Index,
15 name_tok: Ast.TokenIndex,
16 lazy: bool,
1517
16 pub const Location = union(enum) {18 pub const Location = union(enum) {
17 url: []const u8,19 url: []const u8,
...@@ -303,11 +305,14 @@ const Parse = struct {...@@ -303,11 +305,14 @@ const Parse = struct {
303 .hash = null,305 .hash = null,
304 .hash_tok = 0,306 .hash_tok = 0,
305 .node = node,307 .node = node,
308 .name_tok = 0,
309 .lazy = false,
306 };310 };
307 var has_location = false;311 var has_location = false;
308312
309 for (struct_init.ast.fields) |field_init| {313 for (struct_init.ast.fields) |field_init| {
310 const name_token = ast.firstToken(field_init) - 2;314 const name_token = ast.firstToken(field_init) - 2;
315 dep.name_tok = name_token;
311 const field_name = try identifierTokenString(p, name_token);316 const field_name = try identifierTokenString(p, name_token);
312 // We could get fancy with reflection and comptime logic here but doing317 // We could get fancy with reflection and comptime logic here but doing
313 // things manually provides an opportunity to do any additional verification318 // things manually provides an opportunity to do any additional verification
...@@ -342,6 +347,11 @@ const Parse = struct {...@@ -342,6 +347,11 @@ const Parse = struct {
342 else => |e| return e,347 else => |e| return e,
343 };348 };
344 dep.hash_tok = main_tokens[field_init];349 dep.hash_tok = main_tokens[field_init];
350 } else if (mem.eql(u8, field_name, "lazy")) {
351 dep.lazy = parseBool(p, field_init) catch |err| switch (err) {
352 error.ParseFailure => continue,
353 else => |e| return e,
354 };
345 } else {355 } else {
346 // Ignore unknown fields so that we can add fields in future zig356 // Ignore unknown fields so that we can add fields in future zig
347 // versions without breaking older zig versions.357 // versions without breaking older zig versions.
...@@ -374,6 +384,24 @@ const Parse = struct {...@@ -374,6 +384,24 @@ const Parse = struct {
374 }384 }
375 }385 }
376386
387 fn parseBool(p: *Parse, node: Ast.Node.Index) !bool {
388 const ast = p.ast;
389 const node_tags = ast.nodes.items(.tag);
390 const main_tokens = ast.nodes.items(.main_token);
391 if (node_tags[node] != .identifier) {
392 return fail(p, main_tokens[node], "expected identifier", .{});
393 }
394 const ident_token = main_tokens[node];
395 const token_bytes = ast.tokenSlice(ident_token);
396 if (mem.eql(u8, token_bytes, "true")) {
397 return true;
398 } else if (mem.eql(u8, token_bytes, "false")) {
399 return false;
400 } else {
401 return fail(p, ident_token, "expected boolean", .{});
402 }
403 }
404
377 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {405 fn parseString(p: *Parse, node: Ast.Node.Index) ![]const u8 {
378 const ast = p.ast;406 const ast = p.ast;
379 const node_tags = ast.nodes.items(.tag);407 const node_tags = ast.nodes.items(.tag);
src/main.zig+19-2
...@@ -5464,6 +5464,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -5464,6 +5464,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
5464 .location = .{ .relative_path = build_mod.root },5464 .location = .{ .relative_path = build_mod.root },
5465 .location_tok = 0,5465 .location_tok = 0,
5466 .hash_tok = 0,5466 .hash_tok = 0,
5467 .name_tok = 0,
5468 .lazy_status = .eager,
5467 .parent_package_root = build_mod.root,5469 .parent_package_root = build_mod.root,
5468 .parent_manifest_ast = null,5470 .parent_manifest_ast = null,
5469 .prog_node = root_prog_node,5471 .prog_node = root_prog_node,
...@@ -5618,10 +5620,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -5618,10 +5620,14 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
5618 if (process.can_spawn) {5620 if (process.can_spawn) {
5619 var child = std.ChildProcess.init(child_argv, gpa);5621 var child = std.ChildProcess.init(child_argv, gpa);
5620 child.stdin_behavior = .Inherit;5622 child.stdin_behavior = .Inherit;
5621 child.stdout_behavior = .Inherit;5623 child.stdout_behavior = .Pipe;
5622 child.stderr_behavior = .Inherit;5624 child.stderr_behavior = .Inherit;
56235625
5624 const term = try child.spawnAndWait();5626 try child.spawn();
5627 // Since only one output stream is piped, we can simply do a blocking
5628 // read until the stream is finished.
5629 const stdout = try child.stdout.?.readToEndAlloc(arena, 50 * 1024 * 1024);
5630 const term = try child.wait();
5625 switch (term) {5631 switch (term) {
5626 .Exited => |code| {5632 .Exited => |code| {
5627 if (code == 0) return cleanExit();5633 if (code == 0) return cleanExit();
...@@ -5630,6 +5636,15 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi...@@ -5630,6 +5636,15 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi
5630 // diagnostics.5636 // diagnostics.
5631 if (code == 2) process.exit(2);5637 if (code == 2) process.exit(2);
56325638
5639 if (code == 3) {
5640 // Indicates the configure phase failed due to missing lazy
5641 // dependencies and stdout contains the hashes of the ones
5642 // that are missing.
5643 std.debug.print("missing lazy dependencies: '{s}'\n", .{stdout});
5644 std.debug.print("TODO: fetch them and rebuild the build script\n", .{});
5645 process.exit(1);
5646 }
5647
5633 const cmd = try std.mem.join(arena, " ", child_argv);5648 const cmd = try std.mem.join(arena, " ", child_argv);
5634 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });5649 fatal("the following build command failed with exit code {d}:\n{s}", .{ code, cmd });
5635 },5650 },
...@@ -7395,6 +7410,8 @@ fn cmdFetch(...@@ -7395,6 +7410,8 @@ fn cmdFetch(
7395 .location = .{ .path_or_url = path_or_url },7410 .location = .{ .path_or_url = path_or_url },
7396 .location_tok = 0,7411 .location_tok = 0,
7397 .hash_tok = 0,7412 .hash_tok = 0,
7413 .name_tok = 0,
7414 .lazy_status = .eager,
7398 .parent_package_root = undefined,7415 .parent_package_root = undefined,
7399 .parent_manifest_ast = null,7416 .parent_manifest_ast = null,
7400 .prog_node = root_prog_node,7417 .prog_node = root_prog_node,