authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-01-08 19:00:11+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2026-01-09 03:16:39+01:00
logaebd84b525d8ea91f9be5b6562e1fe7abf749220
tree040876b7a3444e29296ec5675481fa006a0efc98
parent04e73d03bd4cd7c584c059c7cee02faf1a17e36c

build runner: refactor step evaluation logic

The previous logic was made really messy by the fact that upon entry to the step eval worker, the step may not be ready to run, we may be racing with other workers doing the same check, and we had already acquired our RSS requirement even though we might not run. It also required iterating all dependencies each time we were called to check whether we were even ready to run yet. A much better strategy is for each step to have an atomic counter representing how many of its dependencies are yet to complete. When a step completes (successfully or otherwise), it decrements this value for all of its dependants, and if it drops any to 0, it schedules that step to run. This means each step is scheduled exactly once, and only when all of its dependencies have finished, reducing redundant checks and hence contention. If the step being scheduled needs to claim RSS which isn't available, then it is instead added to `memory_blocked_steps`, which is iterated by the step worker after a step with an RSS claim finishes. This logic is more concise than before, simpler to understand, generally more efficient, and fixes a bug in the RSS tracking. Also, as a nice side effect, it should also play a little bit nicer with `Io.Threaded`'s scheduling strategy, because we no longer spawn extremely short-lived tasks all the time as we previously did. Resolves: https://codeberg.org/ziglang/zig/issues/30742

4 files changed, 162 insertions(+), 176 deletions(-)

lib/compiler/build_runner.zig+139-156
...@@ -503,7 +503,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -503,7 +503,7 @@ pub fn main(init: process.Init.Minimal) !void {
503 var run: Run = .{503 var run: Run = .{
504 .gpa = gpa,504 .gpa = gpa,
505505
506 .max_rss = max_rss,506 .available_rss = max_rss,
507 .max_rss_is_default = false,507 .max_rss_is_default = false,
508 .max_rss_mutex = .init,508 .max_rss_mutex = .init,
509 .skip_oom_steps = skip_oom_steps,509 .skip_oom_steps = skip_oom_steps,
...@@ -514,7 +514,6 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -514,7 +514,6 @@ pub fn main(init: process.Init.Minimal) !void {
514 .memory_blocked_steps = .empty,514 .memory_blocked_steps = .empty,
515 .step_stack = .empty,515 .step_stack = .empty,
516516
517 .claimed_rss = 0,
518 .error_style = error_style,517 .error_style = error_style,
519 .multiline_errors = multiline_errors,518 .multiline_errors = multiline_errors,
520 .summary = summary orelse if (watch or webui_listen != null) .line else .failures,519 .summary = summary orelse if (watch or webui_listen != null) .line else .failures,
...@@ -524,8 +523,8 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -524,8 +523,8 @@ pub fn main(init: process.Init.Minimal) !void {
524 run.step_stack.deinit(gpa);523 run.step_stack.deinit(gpa);
525 }524 }
526525
527 if (run.max_rss == 0) {526 if (run.available_rss == 0) {
528 run.max_rss = process.totalSystemMemory() catch std.math.maxInt(u64);527 run.available_rss = process.totalSystemMemory() catch std.math.maxInt(u64);
529 run.max_rss_is_default = true;528 run.max_rss_is_default = true;
530 }529 }
531530
...@@ -595,6 +594,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -595,6 +594,7 @@ pub fn main(init: process.Init.Minimal) !void {
595 .rebuild => {594 .rebuild => {
596 for (run.step_stack.keys()) |step| {595 for (run.step_stack.keys()) |step| {
597 step.state = .precheck_done;596 step.state = .precheck_done;
597 step.pending_deps = @intCast(step.dependencies.items.len);
598 step.reset(gpa);598 step.reset(gpa);
599 }599 }
600 continue :rebuild;600 continue :rebuild;
...@@ -637,7 +637,7 @@ pub fn main(init: process.Init.Minimal) !void {...@@ -637,7 +637,7 @@ pub fn main(init: process.Init.Minimal) !void {
637637
638fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {638fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void {
639 for (all_steps) |step| switch (step.state) {639 for (all_steps) |step| switch (step.state) {
640 .dependency_failure, .failure, .skipped => step.recursiveReset(gpa),640 .dependency_failure, .failure, .skipped => _ = step.invalidateResult(gpa),
641 else => continue,641 else => continue,
642 };642 };
643 // Now that all dirty steps have been found, the remaining steps that643 // Now that all dirty steps have been found, the remaining steps that
...@@ -658,7 +658,8 @@ fn countSubProcesses(all_steps: []const *Step) usize {...@@ -658,7 +658,8 @@ fn countSubProcesses(all_steps: []const *Step) usize {
658658
659const Run = struct {659const Run = struct {
660 gpa: Allocator,660 gpa: Allocator,
661 max_rss: u64,661
662 available_rss: usize,
662 max_rss_is_default: bool,663 max_rss_is_default: bool,
663 max_rss_mutex: Io.Mutex,664 max_rss_mutex: Io.Mutex,
664 skip_oom_steps: bool,665 skip_oom_steps: bool,
...@@ -670,7 +671,6 @@ const Run = struct {...@@ -670,7 +671,6 @@ const Run = struct {
670 /// Allocated into `gpa`.671 /// Allocated into `gpa`.
671 step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),672 step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
672673
673 claimed_rss: usize,
674 error_style: ErrorStyle,674 error_style: ErrorStyle,
675 multiline_errors: MultilineErrors,675 multiline_errors: MultilineErrors,
676 summary: Summary,676 summary: Summary,
...@@ -715,12 +715,15 @@ fn prepare(...@@ -715,12 +715,15 @@ fn prepare(
715 var any_problems = false;715 var any_problems = false;
716 for (step_stack.keys()) |s| {716 for (step_stack.keys()) |s| {
717 if (s.max_rss == 0) continue;717 if (s.max_rss == 0) continue;
718 if (s.max_rss > run.max_rss) {718 if (s.max_rss > run.available_rss) {
719 if (run.skip_oom_steps) {719 if (run.skip_oom_steps) {
720 s.state = .skipped_oom;720 s.state = .skipped_oom;
721 for (s.dependants.items) |dependant| {
722 dependant.pending_deps -= 1;
723 }
721 } else {724 } else {
722 std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{725 std.debug.print("{s}{s}: this step declares an upper bound of {d} bytes of memory, exceeding the available {d} bytes of memory\n", .{
723 s.owner.dep_prefix, s.name, s.max_rss, run.max_rss,726 s.owner.dep_prefix, s.name, s.max_rss, run.available_rss,
724 });727 });
725 any_problems = true;728 any_problems = true;
726 }729 }
...@@ -747,23 +750,26 @@ fn runStepNames(...@@ -747,23 +750,26 @@ fn runStepNames(
747 const step_stack = &run.step_stack;750 const step_stack = &run.step_stack;
748751
749 {752 {
753 // Collect the initial set of tasks (those with no outstanding dependencies) into a buffer,
754 // then spawn them. The buffer is so that we don't race with `makeStep` and end up thinking
755 // a step is initial when it actually became ready due to an earlier initial step.
756 var initial_set: std.ArrayList(*Step) = .empty;
757 defer initial_set.deinit(gpa);
758 try initial_set.ensureUnusedCapacity(gpa, step_stack.count());
759 for (step_stack.keys()) |s| {
760 if (s.state == .precheck_done and s.pending_deps == 0) {
761 initial_set.appendAssumeCapacity(s);
762 }
763 }
764
750 const step_prog = parent_prog_node.start("steps", step_stack.count());765 const step_prog = parent_prog_node.start("steps", step_stack.count());
751 defer step_prog.end();766 defer step_prog.end();
752767
753 var group: Io.Group = .init;768 var group: Io.Group = .init;
754 defer group.cancel(io);769 defer group.cancel(io);
755770 // Start working on all of the initial steps...
756 // Here we spawn the initial set of tasks with a nice heuristic -771 for (initial_set.items) |s| try stepReady(&group, b, s, step_prog, run);
757 // dependency order. Each worker when it finishes a step will then772 // ...and `makeStep` will trigger every other step when their last dependency finishes.
758 // check whether it should run any dependants.
759 const steps_slice = step_stack.keys();
760 for (0..steps_slice.len) |i| {
761 const step = steps_slice[steps_slice.len - i - 1];
762 if (step.state == .skipped_oom) continue;
763
764 group.async(io, workerMakeOneStep, .{ &group, b, step, step_prog, run });
765 }
766
767 try group.await(io);773 try group.await(io);
768 }774 }
769775
...@@ -798,17 +804,7 @@ fn runStepNames(...@@ -798,17 +804,7 @@ fn runStepNames(
798 switch (s.state) {804 switch (s.state) {
799 .precheck_unstarted => unreachable,805 .precheck_unstarted => unreachable,
800 .precheck_started => unreachable,806 .precheck_started => unreachable,
801 .running => unreachable,807 .precheck_done => unreachable,
802 .precheck_done => {
803 // precheck_done is equivalent to dependency_failure in the case of
804 // transitive dependencies. For example:
805 // A -> B -> C (failure)
806 // B will be marked as dependency_failure, while A may never be queued, and thus
807 // remain in the initial state of precheck_done.
808 s.state = .dependency_failure;
809 if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);
810 pending_count += 1;
811 },
812 .dependency_failure => pending_count += 1,808 .dependency_failure => pending_count += 1,
813 .success => success_count += 1,809 .success => success_count += 1,
814 .skipped, .skipped_oom => skipped_count += 1,810 .skipped, .skipped_oom => skipped_count += 1,
...@@ -1008,7 +1004,6 @@ fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void {...@@ -1008,7 +1004,6 @@ fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void {
1008 .precheck_unstarted => unreachable,1004 .precheck_unstarted => unreachable,
1009 .precheck_started => unreachable,1005 .precheck_started => unreachable,
1010 .precheck_done => unreachable,1006 .precheck_done => unreachable,
1011 .running => unreachable,
10121007
1013 .dependency_failure => {1008 .dependency_failure => {
1014 try stderr.setColor(.dim);1009 try stderr.setColor(.dim);
...@@ -1067,16 +1062,16 @@ fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void {...@@ -1067,16 +1062,16 @@ fn printStepStatus(s: *Step, stderr: Io.Terminal, run: *const Run) !void {
1067 }1062 }
1068 try writer.writeAll("\n");1063 try writer.writeAll("\n");
1069 },1064 },
1070 .skipped, .skipped_oom => |skip| {1065 .skipped => {
1071 try stderr.setColor(.yellow);1066 try stderr.setColor(.yellow);
1072 try writer.writeAll(" skipped");1067 try writer.writeAll(" skipped\n");
1073 if (skip == .skipped_oom) {1068 try stderr.setColor(.reset);
1074 try writer.writeAll(" (not enough memory)");1069 },
1075 try stderr.setColor(.dim);1070 .skipped_oom => {
1076 try writer.print(" upper bound of {d} exceeded runner limit ({d})", .{ s.max_rss, run.max_rss });1071 try stderr.setColor(.yellow);
1077 try stderr.setColor(.yellow);1072 try writer.writeAll(" skipped (not enough memory)");
1078 }1073 try stderr.setColor(.dim);
1079 try writer.writeAll("\n");1074 try writer.print(" upper bound of {d} exceeded runner limit ({d})\n", .{ s.max_rss, run.available_rss });
1080 try stderr.setColor(.reset);1075 try stderr.setColor(.reset);
1081 },1076 },
1082 .failure => {1077 .failure => {
...@@ -1250,10 +1245,10 @@ fn printTreeStep(...@@ -1250,10 +1245,10 @@ fn printTreeStep(
1250/// Each step has its dependencies traversed in random order, this accomplishes1245/// Each step has its dependencies traversed in random order, this accomplishes
1251/// two things:1246/// two things:
1252/// - `step_stack` will be in randomized-depth-first order, so the build runner1247/// - `step_stack` will be in randomized-depth-first order, so the build runner
1253/// spawns steps in a random (but optimized) order1248/// spawns initial steps in a random order
1254/// - each step's `dependants` list is also filled in a random order, so that1249/// - each step's `dependants` list is also filled in a random order, so that
1255/// when it finishes executing in `workerMakeOneStep`, it spawns next steps1250/// when it finishes executing in `makeStep`, it spawns next steps to run in
1256/// to run in random order1251/// random order
1257fn constructGraphAndCheckForDependencyLoop(1252fn constructGraphAndCheckForDependencyLoop(
1258 gpa: Allocator,1253 gpa: Allocator,
1259 b: *std.Build,1254 b: *std.Build,
...@@ -1290,12 +1285,12 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -1290,12 +1285,12 @@ fn constructGraphAndCheckForDependencyLoop(
1290 }1285 }
12911286
1292 s.state = .precheck_done;1287 s.state = .precheck_done;
1288 s.pending_deps = @intCast(s.dependencies.items.len);
1293 },1289 },
1294 .precheck_done => {},1290 .precheck_done => {},
12951291
1296 // These don't happen until we actually run the step graph.1292 // These don't happen until we actually run the step graph.
1297 .dependency_failure => unreachable,1293 .dependency_failure => unreachable,
1298 .running => unreachable,
1299 .success => unreachable,1294 .success => unreachable,
1300 .failure => unreachable,1295 .failure => unreachable,
1301 .skipped => unreachable,1296 .skipped => unreachable,
...@@ -1303,148 +1298,136 @@ fn constructGraphAndCheckForDependencyLoop(...@@ -1303,148 +1298,136 @@ fn constructGraphAndCheckForDependencyLoop(
1303 }1298 }
1304}1299}
13051300
1306fn workerMakeOneStep(1301/// Runs the "make" function of the single step `s`, updates its state, and then spawns newly-ready
1302/// dependant steps in `group`. If `s` makes an RSS claim (i.e. `s.max_rss != 0`), the caller must
1303/// have already subtracted this value from `run.available_rss`. This function will release the RSS
1304/// claim (i.e. add `s.max_rss` back into `run.available_rss`) and queue any viable memory-blocked
1305/// steps after "make" completes for `s`.
1306fn makeStep(
1307 group: *Io.Group,1307 group: *Io.Group,
1308 b: *std.Build,1308 b: *std.Build,
1309 s: *Step,1309 s: *Step,
1310 prog_node: std.Progress.Node,1310 root_prog_node: std.Progress.Node,
1311 run: *Run,1311 run: *Run,
1312) void {1312) Io.Cancelable!void {
1313 const graph = b.graph;1313 const graph = b.graph;
1314 const io = graph.io;1314 const io = graph.io;
1315 const gpa = run.gpa;1315 const gpa = run.gpa;
13161316
1317 // First, check the conditions for running this step. If they are not met,1317 {
1318 // then we return without doing the step, relying on another worker to1318 const step_prog_node = root_prog_node.start(s.name, 0);
1319 // queue this step up again when dependencies are met.1319 defer step_prog_node.end();
1320 for (s.dependencies.items) |dep| {
1321 switch (@atomicLoad(Step.State, &dep.state, .seq_cst)) {
1322 .success, .skipped => continue,
1323 .failure, .dependency_failure, .skipped_oom => {
1324 @atomicStore(Step.State, &s.state, .dependency_failure, .seq_cst);
1325 if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);
1326 return;
1327 },
1328 .precheck_done, .running => {
1329 // dependency is not finished yet.
1330 return;
1331 },
1332 .precheck_unstarted => unreachable,
1333 .precheck_started => unreachable,
1334 }
1335 }
1336
1337 if (s.max_rss != 0) {
1338 run.max_rss_mutex.lockUncancelable(io);
1339 defer run.max_rss_mutex.unlock(io);
1340
1341 // Avoid running steps twice.
1342 if (s.state != .precheck_done) {
1343 // Another worker got the job.
1344 return;
1345 }
13461320
1347 const new_claimed_rss = run.claimed_rss + s.max_rss;1321 if (run.web_server) |*ws| ws.updateStepStatus(s, .wip);
1348 if (new_claimed_rss > run.max_rss) {
1349 // Running this step right now could possibly exceed the allotted RSS.
1350 // Add this step to the queue of memory-blocked steps.
1351 run.memory_blocked_steps.append(gpa, s) catch @panic("OOM");
1352 return;
1353 }
13541322
1355 run.claimed_rss = new_claimed_rss;1323 const new_state: Step.State = for (s.dependencies.items) |dep| {
1356 s.state = .running;1324 switch (@atomicLoad(Step.State, &dep.state, .monotonic)) {
1357 } else {1325 .precheck_unstarted => unreachable,
1358 // Avoid running steps twice.1326 .precheck_started => unreachable,
1359 if (@cmpxchgStrong(Step.State, &s.state, .precheck_done, .running, .seq_cst, .seq_cst) != null) {1327 .precheck_done => unreachable,
1360 // Another worker got the job.
1361 return;
1362 }
1363 }
13641328
1365 const sub_prog_node = prog_node.start(s.name, 0);1329 .failure,
1366 defer sub_prog_node.end();1330 .dependency_failure,
1331 .skipped_oom,
1332 => break .dependency_failure,
13671333
1368 if (run.web_server) |*ws| ws.updateStepStatus(s, .wip);1334 .success, .skipped => {},
1335 }
1336 } else if (s.make(.{
1337 .progress_node = step_prog_node,
1338 .watch = run.watch,
1339 .web_server = if (run.web_server) |*ws| ws else null,
1340 .unit_test_timeout_ns = run.unit_test_timeout_ns,
1341 .gpa = gpa,
1342 })) state: {
1343 break :state .success;
1344 } else |err| switch (err) {
1345 error.MakeFailed => .failure,
1346 error.MakeSkipped => .skipped,
1347 };
13691348
1370 const make_result = s.make(.{1349 @atomicStore(Step.State, &s.state, new_state, .monotonic);
1371 .progress_node = sub_prog_node,
1372 .watch = run.watch,
1373 .web_server = if (run.web_server) |*ws| ws else null,
1374 .unit_test_timeout_ns = run.unit_test_timeout_ns,
1375 .gpa = gpa,
1376 });
13771350
1378 // No matter the result, we want to display error/warning messages.1351 switch (new_state) {
1379 const show_compile_errors = s.result_error_bundle.errorMessageCount() > 0;1352 .precheck_unstarted => unreachable,
1380 const show_error_msgs = s.result_error_msgs.items.len > 0;1353 .precheck_started => unreachable,
1381 const show_stderr = s.result_stderr.len > 0;1354 .precheck_done => unreachable,
1382 if (show_error_msgs or show_compile_errors or show_stderr) {
1383 const stderr = io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode) catch |err| switch (err) {
1384 error.Canceled => return,
1385 };
1386 defer io.unlockStderr();
1387 printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {};
1388 }
13891355
1390 handle_result: {1356 .failure,
1391 if (make_result) |_| {1357 .dependency_failure,
1392 @atomicStore(Step.State, &s.state, .success, .seq_cst);1358 .skipped_oom,
1393 if (run.web_server) |*ws| ws.updateStepStatus(s, .success);1359 => {
1394 } else |err| switch (err) {
1395 error.MakeFailed => {
1396 @atomicStore(Step.State, &s.state, .failure, .seq_cst);
1397 if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);1360 if (run.web_server) |*ws| ws.updateStepStatus(s, .failure);
1398 std.Progress.setStatus(.failure_working);1361 std.Progress.setStatus(.failure_working);
1399 break :handle_result;
1400 },1362 },
1401 error.MakeSkipped => {1363
1402 @atomicStore(Step.State, &s.state, .skipped, .seq_cst);1364 .success,
1365 .skipped,
1366 => {
1403 if (run.web_server) |*ws| ws.updateStepStatus(s, .success);1367 if (run.web_server) |*ws| ws.updateStepStatus(s, .success);
1404 },1368 },
1405 }1369 }
1370 }
14061371
1407 // Successful completion of a step, so we queue up its dependants as well.1372 // No matter the result, we want to display error/warning messages.
1408 for (s.dependants.items) |dep| {1373 if (s.result_error_bundle.errorMessageCount() > 0 or
1409 group.async(io, workerMakeOneStep, .{ group, b, dep, prog_node, run });1374 s.result_error_msgs.items.len > 0 or
1410 }1375 s.result_stderr.len > 0)
1376 {
1377 const stderr = try io.lockStderr(&stdio_buffer_allocation, graph.stderr_mode);
1378 defer io.unlockStderr();
1379 printErrorMessages(gpa, s, .{}, stderr.terminal(), run.error_style, run.multiline_errors) catch {};
1411 }1380 }
14121381
1413 // If this is a step that claims resources, we must now queue up other
1414 // steps that are waiting for resources.
1415 if (s.max_rss != 0) {1382 if (s.max_rss != 0) {
1416 var dispatch_deps: std.ArrayList(*Step) = .empty;1383 var dispatch_set: std.ArrayList(*Step) = .empty;
1417 defer dispatch_deps.deinit(gpa);1384 defer dispatch_set.deinit(gpa);
14181385
1386 // Release our RSS claim and kick off some blocked steps if possible. We use `dispatch_set`
1387 // as a staging buffer to avoid recursing into `makeStep` while `run.max_rss_mutex` is held.
1419 {1388 {
1420 run.max_rss_mutex.lockUncancelable(io);1389 try run.max_rss_mutex.lock(io);
1421 defer run.max_rss_mutex.unlock(io);1390 defer run.max_rss_mutex.unlock(io);
14221391 run.available_rss += s.max_rss;
1423 dispatch_deps.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch @panic("OOM");1392 dispatch_set.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch @panic("OOM");
14241393 while (run.memory_blocked_steps.getLastOrNull()) |candidate| {
1425 // Give the memory back to the scheduler.1394 if (run.available_rss < candidate.max_rss) break;
1426 run.claimed_rss -= s.max_rss;1395 assert(run.memory_blocked_steps.pop() == candidate);
1427 // Avoid kicking off too many tasks that we already know will not have1396 dispatch_set.appendAssumeCapacity(candidate);
1428 // enough resources.
1429 var remaining = run.max_rss - run.claimed_rss;
1430 var i: usize = 0;
1431 for (run.memory_blocked_steps.items) |dep| {
1432 assert(dep.max_rss != 0);
1433 if (dep.max_rss <= remaining) {
1434 remaining -= dep.max_rss;
1435 dispatch_deps.appendAssumeCapacity(dep);
1436 } else {
1437 run.memory_blocked_steps.items[i] = dep;
1438 i += 1;
1439 }
1440 }1397 }
1441 run.memory_blocked_steps.shrinkRetainingCapacity(i);
1442 }1398 }
1443 for (dispatch_deps.items) |dep| {1399 for (dispatch_set.items) |candidate| {
1444 // Must be called without max_rss_mutex held in case it executes recursively.1400 group.async(io, makeStep, .{ group, b, candidate, root_prog_node, run });
1445 group.async(io, workerMakeOneStep, .{ group, b, dep, prog_node, run });1401 }
1402 }
1403
1404 for (s.dependants.items) |dependant| {
1405 // `.acq_rel` synchronizes with itself to ensure all dependencies' final states are visible when this hits 0.
1406 if (@atomicRmw(u32, &dependant.pending_deps, .Sub, 1, .acq_rel) == 1) {
1407 try stepReady(group, b, dependant, root_prog_node, run);
1408 }
1409 }
1410}
1411
1412fn stepReady(
1413 group: *Io.Group,
1414 b: *std.Build,
1415 s: *Step,
1416 root_prog_node: std.Progress.Node,
1417 run: *Run,
1418) !void {
1419 const io = b.graph.io;
1420 if (s.max_rss != 0) {
1421 try run.max_rss_mutex.lock(io);
1422 defer run.max_rss_mutex.unlock(io);
1423 if (run.available_rss < s.max_rss) {
1424 // Running this step right now could possibly exceed the allotted RSS.
1425 run.memory_blocked_steps.append(run.gpa, s) catch @panic("OOM");
1426 return;
1446 }1427 }
1428 run.available_rss -= s.max_rss;
1447 }1429 }
1430 group.async(io, makeStep, .{ group, b, s, root_prog_node, run });
1448}1431}
14491432
1450pub fn printErrorMessages(1433pub fn printErrorMessages(
lib/std/Build/Step.zig+12-7
...@@ -29,7 +29,6 @@ dependants: ArrayList(*Step),...@@ -29,7 +29,6 @@ dependants: ArrayList(*Step),
29/// retain previous value, or update.29/// retain previous value, or update.
30inputs: Inputs,30inputs: Inputs,
3131
32state: State,
33/// Set this field to declare an upper bound on the amount of bytes of memory it will32/// Set this field to declare an upper bound on the amount of bytes of memory it will
34/// take to run the step. Zero means no limit.33/// take to run the step. Zero means no limit.
35///34///
...@@ -51,6 +50,9 @@ state: State,...@@ -51,6 +50,9 @@ state: State,
51/// total system memory available.50/// total system memory available.
52max_rss: usize,51max_rss: usize,
5352
53state: State,
54pending_deps: u32,
55
54result_error_msgs: ArrayList([]const u8),56result_error_msgs: ArrayList([]const u8),
55result_error_bundle: std.zig.ErrorBundle,57result_error_bundle: std.zig.ErrorBundle,
56result_stderr: []const u8,58result_stderr: []const u8,
...@@ -129,7 +131,6 @@ pub const State = enum {...@@ -129,7 +131,6 @@ pub const State = enum {
129 /// file system inputs have been modified, meaning that the step needs to131 /// file system inputs have been modified, meaning that the step needs to
130 /// be re-evaluated.132 /// be re-evaluated.
131 precheck_done,133 precheck_done,
132 running,
133 dependency_failure,134 dependency_failure,
134 success,135 success,
135 failure,136 failure,
...@@ -242,6 +243,7 @@ pub fn init(options: StepOptions) Step {...@@ -242,6 +243,7 @@ pub fn init(options: StepOptions) Step {
242 .dependants = .empty,243 .dependants = .empty,
243 .inputs = Inputs.init,244 .inputs = Inputs.init,
244 .state = .precheck_unstarted,245 .state = .precheck_unstarted,
246 .pending_deps = undefined, // initialized by build runner
245 .max_rss = options.max_rss,247 .max_rss = options.max_rss,
246 .debug_stack_trace = blk: {248 .debug_stack_trace = blk: {
247 const addr_buf = arena.alloc(usize, options.owner.debug_stack_frames_count) catch @panic("OOM");249 const addr_buf = arena.alloc(usize, options.owner.debug_stack_frames_count) catch @panic("OOM");
...@@ -980,14 +982,17 @@ pub fn reset(step: *Step, gpa: Allocator) void {...@@ -980,14 +982,17 @@ pub fn reset(step: *Step, gpa: Allocator) void {
980}982}
981983
982/// Implementation detail of file watching. Prepares the step for being re-evaluated.984/// Implementation detail of file watching. Prepares the step for being re-evaluated.
983pub fn recursiveReset(step: *Step, gpa: Allocator) void {985/// Returns `true` if the step was newly invalidated, `false` if it was already invalidated.
984 assert(step.state != .precheck_done);986pub fn invalidateResult(step: *Step, gpa: Allocator) bool {
987 if (step.state == .precheck_done) return false;
988 assert(step.pending_deps == 0);
985 step.state = .precheck_done;989 step.state = .precheck_done;
986 step.reset(gpa);990 step.reset(gpa);
987 for (step.dependants.items) |dep| {991 for (step.dependants.items) |dependant| {
988 if (dep.state == .precheck_done) continue;992 _ = dependant.invalidateResult(gpa);
989 dep.recursiveReset(gpa);993 dependant.pending_deps += 1;
990 }994 }
995 return true;
991}996}
992997
993test {998test {
lib/std/Build/Watch.zig+2-5
...@@ -901,7 +901,7 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {...@@ -901,7 +901,7 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
901 };901 };
902 for (reaction_set.values()) |step_set| {902 for (reaction_set.values()) |step_set| {
903 for (step_set.keys()) |step| {903 for (step_set.keys()) |step| {
904 step.recursiveReset(gpa);904 _ = step.invalidateResult(gpa);
905 }905 }
906 }906 }
907 }907 }
...@@ -910,10 +910,7 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {...@@ -910,10 +910,7 @@ fn markAllFilesDirty(w: *Watch, gpa: Allocator) void {
910fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {910fn markStepSetDirty(gpa: Allocator, step_set: *StepSet, any_dirty: bool) bool {
911 var this_any_dirty = false;911 var this_any_dirty = false;
912 for (step_set.keys()) |step| {912 for (step_set.keys()) |step| {
913 if (step.state != .precheck_done) {913 if (step.invalidateResult(gpa)) this_any_dirty = true;
914 step.recursiveReset(gpa);
915 this_any_dirty = true;
916 }
917 }914 }
918 return any_dirty or this_any_dirty;915 return any_dirty or this_any_dirty;
919}916}
lib/std/Build/Watch/FsEvents.zig+9-8
...@@ -349,13 +349,17 @@ fn eventCallback(...@@ -349,13 +349,17 @@ fn eventCallback(
349 false => {349 false => {
350 if (fse.watch_paths.get(event_path)) |steps| {350 if (fse.watch_paths.get(event_path)) |steps| {
351 assert(steps.len > 0);351 assert(steps.len > 0);
352 for (steps) |s| dirtyStep(s, gpa, &any_dirty);352 for (steps) |s| {
353 if (s.invalidateResult(gpa)) any_dirty = true;
354 }
353 }355 }
354 if (std.fs.path.dirname(event_path)) |event_dirname| {356 if (std.fs.path.dirname(event_path)) |event_dirname| {
355 // Modifying '/foo/bar' triggers the watch on '/foo'.357 // Modifying '/foo/bar' triggers the watch on '/foo'.
356 if (fse.watch_paths.get(event_dirname)) |steps| {358 if (fse.watch_paths.get(event_dirname)) |steps| {
357 assert(steps.len > 0);359 assert(steps.len > 0);
358 for (steps) |s| dirtyStep(s, gpa, &any_dirty);360 for (steps) |s| {
361 if (s.invalidateResult(gpa)) any_dirty = true;
362 }
359 }363 }
360 }364 }
361 },365 },
...@@ -368,7 +372,9 @@ fn eventCallback(...@@ -368,7 +372,9 @@ fn eventCallback(
368 const changed_path = std.fs.path.dirname(event_path) orelse event_path;372 const changed_path = std.fs.path.dirname(event_path) orelse event_path;
369 for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| {373 for (fse.watch_paths.keys(), fse.watch_paths.values()) |watching_path, steps| {
370 if (dirStartsWith(watching_path, changed_path)) {374 if (dirStartsWith(watching_path, changed_path)) {
371 for (steps) |s| dirtyStep(s, gpa, &any_dirty);375 for (steps) |s| {
376 if (s.invalidateResult(gpa)) any_dirty = true;
377 }
372 }378 }
373 }379 }
374 },380 },
...@@ -379,11 +385,6 @@ fn eventCallback(...@@ -379,11 +385,6 @@ fn eventCallback(
379 _ = dispatch_semaphore_signal(fse.waiting_semaphore);385 _ = dispatch_semaphore_signal(fse.waiting_semaphore);
380 }386 }
381}387}
382fn dirtyStep(s: *std.Build.Step, gpa: Allocator, any_dirty: *bool) void {
383 if (s.state == .precheck_done) return;
384 s.recursiveReset(gpa);
385 any_dirty.* = true;
386}
387fn dirStartsWith(path: []const u8, prefix: []const u8) bool {388fn dirStartsWith(path: []const u8, prefix: []const u8) bool {
388 if (std.mem.eql(u8, path, prefix)) return true;389 if (std.mem.eql(u8, path, prefix)) return true;
389 if (!std.mem.startsWith(u8, path, prefix)) return false;390 if (!std.mem.startsWith(u8, path, prefix)) return false;