authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-02-13 15:14:49-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-03-15 10:48:12-07:00
logcff86cf7a17e038db44fa1f72ee5919eea6a6cae
tree185b4c8a3daae10afcfc0e5d9aee5143878ee384
parent658de75500871f28015aa2ff14872eed0410dddf

build_runner now executes the step graph in parallel

2 files changed, 103 insertions(+), 38 deletions(-)

lib/build_runner.zig+93-33
......@@ -14,10 +14,14 @@ pub fn main() !void {
1414 // Here we use an ArenaAllocator backed by a DirectAllocator because a build is a short-lived,
1515 // one shot program. We don't need to waste time freeing memory and finding places to squish
1616 // bytes into. So we free everything all at once at the very end.
17 var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
18 defer arena.deinit();
17 var single_threaded_arena = std.heap.ArenaAllocator.init(std.heap.page_allocator);
18 defer single_threaded_arena.deinit();
19
20 var thread_safe_arena: std.heap.ThreadSafeAllocator = .{
21 .child_allocator = single_threaded_arena.allocator(),
22 };
23 const allocator = thread_safe_arena.allocator();
1924
20 const allocator = arena.allocator();
2125 var args = try process.argsAlloc(allocator);
2226 defer process.argsFree(allocator, args);
2327
......@@ -245,71 +249,127 @@ pub fn main() !void {
245249 if (builder.validateUserInputDidItFail())
246250 usageAndErr(builder, true, stderr_stream);
247251
248 make(builder, targets.items) catch |err| {
252 runStepNames(builder, targets.items) catch |err| {
249253 switch (err) {
250254 error.UncleanExit => process.exit(1),
251 // This error is intended to indicate that the step has already
252 // logged an error message and so printing the error return trace
253 // here would be unwanted extra information, unless the user opts
254 // into it with a debug flag.
255 error.StepFailed => process.exit(1),
256255 else => return err,
257256 }
258257 };
259258}
260259
261fn make(b: *std.Build, step_names: []const []const u8) !void {
262 var wanted_steps = ArrayList(*std.Build.Step).init(b.allocator);
263 defer wanted_steps.deinit();
260fn runStepNames(b: *std.Build, step_names: []const []const u8) !void {
261 var step_stack = ArrayList(*std.Build.Step).init(b.allocator);
262 defer step_stack.deinit();
264263
265264 if (step_names.len == 0) {
266 try wanted_steps.append(b.default_step);
265 try step_stack.append(b.default_step);
267266 } else {
268 for (step_names) |step_name| {
267 try step_stack.resize(step_names.len);
268
269 for (step_names) |step_name, i| {
269270 const s = b.top_level_steps.get(step_name) orelse {
270271 std.debug.print("no step named '{s}'. Access the help menu with 'zig build -h'\n", .{step_name});
271272 process.exit(1);
272273 };
273 try wanted_steps.append(&s.step);
274 step_stack.items[step_names.len - i - 1] = &s.step;
274275 }
275276 }
276277
277 for (wanted_steps.items) |s| {
278 checkForDependencyLoop(b, s) catch |err| switch (err) {
278 const starting_steps = step_stack.items;
279 for (starting_steps) |s| {
280 checkForDependencyLoop(b, s, &step_stack) catch |err| switch (err) {
279281 error.DependencyLoopDetected => return error.UncleanExit,
280282 else => |e| return e,
281283 };
282284 }
283285
284 for (wanted_steps.items) |s| {
285 try makeOneStep(b, s);
286 var thread_pool: std.Thread.Pool = undefined;
287 try thread_pool.init(b.allocator);
288 defer thread_pool.deinit();
289
290 {
291 var wait_group: std.Thread.WaitGroup = .{};
292 defer wait_group.wait();
293 var i = step_stack.items.len;
294
295 while (i > 0) {
296 i -= 1;
297 const step = step_stack.items[i];
298
299 wait_group.start();
300 thread_pool.spawn(workerMakeOneStep, .{ &wait_group, b, step }) catch
301 @panic("unhandled error");
302 }
286303 }
287}
288304
289fn checkForDependencyLoop(b: *std.Build, s: *std.Build.Step) !void {
290 if (s.loop_tag == .started) {
291 std.debug.print("dependency loop detected:\n {s}\n", .{s.name});
292 return error.DependencyLoopDetected;
305 var any_failed = false;
306
307 for (step_stack.items) |s| {
308 switch (s.result) {
309 .not_done => unreachable,
310 .success => continue,
311 .failure => |f| {
312 any_failed = true;
313 std.debug.print("{s}: {s}\n", .{
314 s.name, @errorName(f.err_code),
315 });
316 },
317 }
293318 }
294 s.loop_tag = .started;
295319
296 for (s.dependencies.items) |dep| {
297 checkForDependencyLoop(b, dep) catch |err| {
298 if (err == error.DependencyLoopDetected) {
299 std.debug.print(" {s}\n", .{s.name});
320 if (any_failed) {
321 process.exit(1);
322 }
323}
324
325fn checkForDependencyLoop(
326 b: *std.Build,
327 s: *std.Build.Step,
328 step_stack: *ArrayList(*std.Build.Step),
329) !void {
330 switch (s.loop_tag) {
331 .started => {
332 std.debug.print("dependency loop detected:\n {s}\n", .{s.name});
333 return error.DependencyLoopDetected;
334 },
335 .unstarted => {
336 s.loop_tag = .started;
337
338 try step_stack.append(s);
339
340 for (s.dependencies.items) |dep| {
341 checkForDependencyLoop(b, dep, step_stack) catch |err| {
342 if (err == error.DependencyLoopDetected) {
343 std.debug.print(" {s}\n", .{s.name});
344 }
345 return err;
346 };
300347 }
301 return err;
302 };
348
349 s.loop_tag = .done;
350 },
351 .done => {},
303352 }
353}
354
355fn workerMakeOneStep(wg: *std.Thread.WaitGroup, b: *std.Build, s: *std.Build.Step) void {
356 defer wg.finish();
357
358 _ = b;
304359
305 s.loop_tag = .done;
360 if (s.make()) |_| {
361 s.result = .success;
362 } else |err| {
363 s.result = .{ .failure = .{
364 .err_code = err,
365 } };
366 }
306367}
307368
308369fn makeOneStep(b: *std.Build, s: *std.Build.Step) anyerror!void {
309370 for (s.dependencies.items) |dep| {
310371 try makeOneStep(b, dep);
311372 }
312
313373 try s.make();
314374}
315375
lib/std/Build/Step.zig+10-5
......@@ -4,7 +4,13 @@ makeFn: *const fn (self: *Step) anyerror!void,
44dependencies: std.ArrayList(*Step),
55/// Used only during a pre-check for dependency loops.
66loop_tag: enum { unstarted, started, done },
7done_flag: bool,
7result: union(enum) {
8 not_done,
9 success,
10 failure: struct {
11 err_code: anyerror,
12 },
13},
814
915pub const Id = enum {
1016 top_level,
......@@ -62,7 +68,7 @@ pub fn init(
6268 .makeFn = makeFn,
6369 .dependencies = std.ArrayList(*Step).init(allocator),
6470 .loop_tag = .unstarted,
65 .done_flag = false,
71 .result = .not_done,
6672 };
6773}
6874
......@@ -71,10 +77,8 @@ pub fn initNoOp(id: Id, name: []const u8, allocator: Allocator) Step {
7177}
7278
7379pub fn make(self: *Step) !void {
74 if (self.done_flag) return;
75
80 assert(self.result == .not_done);
7681 try self.makeFn(self);
77 self.done_flag = true;
7882}
7983
8084pub fn dependOn(self: *Step, other: *Step) void {
......@@ -96,3 +100,4 @@ const Step = @This();
96100const std = @import("../std.zig");
97101const Build = std.Build;
98102const Allocator = std.mem.Allocator;
103const assert = std.debug.assert;