authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-08-26 15:34:53+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-10-18 09:28:41+01:00
loge4456d03f3642c4d0e19176aa09dbc12099e7a4c
tree87a005c07784f35ecc18015af6bb304e981af9c4
parent7e7d7875b9af97bd04ca03a98b2e4188d57e3c13
signature Commit is signed but in an unrecognized format.

std.Build.Step.Run: many enhancements

This is a major refactor to `Step.Run` which adds new functionality, primarily to the execution of Zig tests. * All tests are run, even if a test crashes. This happens through the same mechanism as timeouts where the test processes is repeatedly respawned as needed. * The build status output is more precise. For each unit test, it differentiates pass, skip, fail, crash, and timeout. Memory leaks are reported separately, as they do not indicate a test's "status", but are rather an additional property (a test with leaks may still pass!). * The number of memory leaks is tracked and reported, both per-test and for a whole `Run` step. * Reporting is made clearer when a step is failed solely due to error logs (`std.log.err`) where every unit test passed.

6 files changed, 465 insertions(+), 356 deletions(-)

lib/compiler/build_runner.zig+75-35
...@@ -735,11 +735,12 @@ fn runStepNames(...@@ -735,11 +735,12 @@ fn runStepNames(
735735
736 assert(run.memory_blocked_steps.items.len == 0);736 assert(run.memory_blocked_steps.items.len == 0);
737737
738 var test_pass_count: usize = 0;
738 var test_skip_count: usize = 0;739 var test_skip_count: usize = 0;
739 var test_fail_count: usize = 0;740 var test_fail_count: usize = 0;
740 var test_pass_count: usize = 0;741 var test_crash_count: usize = 0;
741 var test_leak_count: usize = 0;
742 var test_timeout_count: usize = 0;742 var test_timeout_count: usize = 0;
743
743 var test_count: usize = 0;744 var test_count: usize = 0;
744745
745 var success_count: usize = 0;746 var success_count: usize = 0;
...@@ -749,11 +750,12 @@ fn runStepNames(...@@ -749,11 +750,12 @@ fn runStepNames(
749 var total_compile_errors: usize = 0;750 var total_compile_errors: usize = 0;
750751
751 for (step_stack.keys()) |s| {752 for (step_stack.keys()) |s| {
752 test_fail_count += s.test_results.fail_count;753 test_pass_count += s.test_results.passCount();
753 test_skip_count += s.test_results.skip_count;754 test_skip_count += s.test_results.skip_count;
754 test_leak_count += s.test_results.leak_count;755 test_fail_count += s.test_results.fail_count;
756 test_crash_count += s.test_results.crash_count;
755 test_timeout_count += s.test_results.timeout_count;757 test_timeout_count += s.test_results.timeout_count;
756 test_pass_count += s.test_results.passCount();758
757 test_count += s.test_results.test_count;759 test_count += s.test_results.test_count;
758760
759 switch (s.state) {761 switch (s.state) {
...@@ -822,6 +824,9 @@ fn runStepNames(...@@ -822,6 +824,9 @@ fn runStepNames(
822 f.waitAndPrintReport();824 f.waitAndPrintReport();
823 }825 }
824826
827 // Every test has a state
828 assert(test_pass_count + test_skip_count + test_fail_count + test_crash_count + test_timeout_count == test_count);
829
825 // A proper command line application defaults to silently succeeding.830 // A proper command line application defaults to silently succeeding.
826 // The user may request verbose mode if they have a different preference.831 // The user may request verbose mode if they have a different preference.
827 const failures_only = switch (run.summary) {832 const failures_only = switch (run.summary) {
...@@ -844,14 +849,16 @@ fn runStepNames(...@@ -844,14 +849,16 @@ fn runStepNames(
844 w.writeAll("\nBuild Summary:") catch {};849 w.writeAll("\nBuild Summary:") catch {};
845 ttyconf.setColor(w, .reset) catch {};850 ttyconf.setColor(w, .reset) catch {};
846 w.print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {};851 w.print(" {d}/{d} steps succeeded", .{ success_count, total_count }) catch {};
847 if (skipped_count > 0) w.print("; {d} skipped", .{skipped_count}) catch {};852 if (skipped_count > 0) w.print(", {d} skipped", .{skipped_count}) catch {};
848 if (failure_count > 0) w.print("; {d} failed", .{failure_count}) catch {};853 if (failure_count > 0) w.print(", {d} failed", .{failure_count}) catch {};
849854
850 if (test_count > 0) w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {};855 if (test_count > 0) {
851 if (test_skip_count > 0) w.print("; {d} skipped", .{test_skip_count}) catch {};856 w.print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {};
852 if (test_fail_count > 0) w.print("; {d} failed", .{test_fail_count}) catch {};857 if (test_skip_count > 0) w.print(", {d} skipped", .{test_skip_count}) catch {};
853 if (test_leak_count > 0) w.print("; {d} leaked", .{test_leak_count}) catch {};858 if (test_fail_count > 0) w.print(", {d} failed", .{test_fail_count}) catch {};
854 if (test_timeout_count > 0) w.print("; {d} timed out", .{test_timeout_count}) catch {};859 if (test_crash_count > 0) w.print(", {d} crashed", .{test_crash_count}) catch {};
860 if (test_timeout_count > 0) w.print(", {d} timed out", .{test_timeout_count}) catch {};
861 }
855862
856 w.writeAll("\n") catch {};863 w.writeAll("\n") catch {};
857864
...@@ -961,11 +968,16 @@ fn printStepStatus(...@@ -961,11 +968,16 @@ fn printStepStatus(
961 try stderr.writeAll(" cached");968 try stderr.writeAll(" cached");
962 } else if (s.test_results.test_count > 0) {969 } else if (s.test_results.test_count > 0) {
963 const pass_count = s.test_results.passCount();970 const pass_count = s.test_results.passCount();
971 assert(s.test_results.test_count == pass_count + s.test_results.skip_count);
964 try stderr.print(" {d} passed", .{pass_count});972 try stderr.print(" {d} passed", .{pass_count});
965 if (s.test_results.skip_count > 0) {973 if (s.test_results.skip_count > 0) {
974 try ttyconf.setColor(stderr, .white);
975 try stderr.writeAll(", ");
966 try ttyconf.setColor(stderr, .yellow);976 try ttyconf.setColor(stderr, .yellow);
967 try stderr.print(" {d} skipped", .{s.test_results.skip_count});977 try stderr.print("{d} skipped", .{s.test_results.skip_count});
968 }978 }
979 try ttyconf.setColor(stderr, .white);
980 try stderr.print(" ({d} total)", .{s.test_results.test_count});
969 } else {981 } else {
970 try stderr.writeAll(" success");982 try stderr.writeAll(" success");
971 }983 }
...@@ -1031,41 +1043,64 @@ fn printStepFailure(...@@ -1031,41 +1043,64 @@ fn printStepFailure(
1031 s.result_error_bundle.errorMessageCount(),1043 s.result_error_bundle.errorMessageCount(),
1032 });1044 });
1033 } else if (!s.test_results.isSuccess()) {1045 } else if (!s.test_results.isSuccess()) {
1034 try stderr.print(" {d}/{d} passed", .{1046 // These first values include all of the test "statuses". Every test is either passsed,
1035 s.test_results.passCount(), s.test_results.test_count,1047 // skipped, failed, crashed, or timed out.
1036 });1048 try ttyconf.setColor(stderr, .green);
1037 if (s.test_results.fail_count > 0) {1049 try stderr.print(" {d} passed", .{s.test_results.passCount()});
1050 try ttyconf.setColor(stderr, .white);
1051 if (s.test_results.skip_count > 0) {
1038 try stderr.writeAll(", ");1052 try stderr.writeAll(", ");
1039 try ttyconf.setColor(stderr, .red);1053 try ttyconf.setColor(stderr, .yellow);
1040 try stderr.print("{d} failed", .{1054 try stderr.print("{d} skipped", .{s.test_results.skip_count});
1041 s.test_results.fail_count,
1042 });
1043 try ttyconf.setColor(stderr, .white);1055 try ttyconf.setColor(stderr, .white);
1044 }1056 }
1045 if (s.test_results.skip_count > 0) {1057 if (s.test_results.fail_count > 0) {
1046 try stderr.writeAll(", ");1058 try stderr.writeAll(", ");
1047 try ttyconf.setColor(stderr, .yellow);1059 try ttyconf.setColor(stderr, .red);
1048 try stderr.print("{d} skipped", .{1060 try stderr.print("{d} failed", .{s.test_results.fail_count});
1049 s.test_results.skip_count,
1050 });
1051 try ttyconf.setColor(stderr, .white);1061 try ttyconf.setColor(stderr, .white);
1052 }1062 }
1053 if (s.test_results.leak_count > 0) {1063 if (s.test_results.crash_count > 0) {
1054 try stderr.writeAll(", ");1064 try stderr.writeAll(", ");
1055 try ttyconf.setColor(stderr, .red);1065 try ttyconf.setColor(stderr, .red);
1056 try stderr.print("{d} leaked", .{1066 try stderr.print("{d} crashed", .{s.test_results.crash_count});
1057 s.test_results.leak_count,
1058 });
1059 try ttyconf.setColor(stderr, .white);1067 try ttyconf.setColor(stderr, .white);
1060 }1068 }
1061 if (s.test_results.timeout_count > 0) {1069 if (s.test_results.timeout_count > 0) {
1062 try stderr.writeAll(", ");1070 try stderr.writeAll(", ");
1063 try ttyconf.setColor(stderr, .red);1071 try ttyconf.setColor(stderr, .red);
1064 try stderr.print("{d} timed out", .{1072 try stderr.print("{d} timed out", .{s.test_results.timeout_count});
1065 s.test_results.timeout_count,1073 try ttyconf.setColor(stderr, .white);
1066 });1074 }
1075 try stderr.print(" ({d} total)", .{s.test_results.test_count});
1076
1077 // Memory leaks are intentionally written after the total, because is isn't a test *status*,
1078 // but just a flag that any tests -- even passed ones -- can have. We also use a different
1079 // separator, so it looks like:
1080 // 2 passed, 1 skipped, 2 failed (5 total); 2 leaks
1081 if (s.test_results.leak_count > 0) {
1082 try stderr.writeAll("; ");
1083 try ttyconf.setColor(stderr, .red);
1084 try stderr.print("{d} leaks", .{s.test_results.leak_count});
1067 try ttyconf.setColor(stderr, .white);1085 try ttyconf.setColor(stderr, .white);
1068 }1086 }
1087
1088 // It's usually not helpful to know how many error logs there were because they tend to
1089 // just come with other errors (e.g. crashes and leaks print stack traces, and clean
1090 // failures print error traces). So only mention them if they're the only thing causing
1091 // the failure.
1092 const show_err_logs: bool = show: {
1093 var alt_results = s.test_results;
1094 alt_results.log_err_count = 0;
1095 break :show alt_results.isSuccess();
1096 };
1097 if (show_err_logs) {
1098 try stderr.writeAll("; ");
1099 try ttyconf.setColor(stderr, .red);
1100 try stderr.print("{d} error logs", .{s.test_results.log_err_count});
1101 try ttyconf.setColor(stderr, .white);
1102 }
1103
1069 try stderr.writeAll("\n");1104 try stderr.writeAll("\n");
1070 } else if (s.result_error_msgs.items.len > 0) {1105 } else if (s.result_error_msgs.items.len > 0) {
1071 try ttyconf.setColor(stderr, .red);1106 try ttyconf.setColor(stderr, .red);
...@@ -1400,7 +1435,12 @@ pub fn printErrorMessages(...@@ -1400,7 +1435,12 @@ pub fn printErrorMessages(
1400 try ttyconf.setColor(stderr, .red);1435 try ttyconf.setColor(stderr, .red);
1401 try stderr.writeAll("error: ");1436 try stderr.writeAll("error: ");
1402 try ttyconf.setColor(stderr, .reset);1437 try ttyconf.setColor(stderr, .reset);
1403 try stderr.writeAll(msg);1438 // If the message has multiple lines, indent the non-initial ones to align them with the 'error:' text.
1439 var it = std.mem.splitScalar(u8, msg, '\n');
1440 try stderr.writeAll(it.first());
1441 while (it.next()) |line| {
1442 try stderr.print("\n {s}", .{line});
1443 }
1404 try stderr.writeAll("\n");1444 try stderr.writeAll("\n");
1405 }1445 }
1406}1446}
lib/compiler/test_runner.zig+16-11
...@@ -132,34 +132,39 @@ fn mainServer() !void {...@@ -132,34 +132,39 @@ fn mainServer() !void {
132 log_err_count = 0;132 log_err_count = 0;
133 const index = try server.receiveBody_u32();133 const index = try server.receiveBody_u32();
134 const test_fn = builtin.test_functions[index];134 const test_fn = builtin.test_functions[index];
135 var fail = false;
136 var skip = false;
137 is_fuzz_test = false;135 is_fuzz_test = false;
138136
139 // let the build server know we're starting the test now137 // let the build server know we're starting the test now
140 try server.serveStringMessage(.test_started, &.{});138 try server.serveStringMessage(.test_started, &.{});
141139
142 test_fn.func() catch |err| switch (err) {140 const TestResults = std.zig.Server.Message.TestResults;
143 error.SkipZigTest => skip = true,141 const status: TestResults.Status = if (test_fn.func()) |v| s: {
144 else => {142 v;
145 fail = true;143 break :s .pass;
144 } else |err| switch (err) {
145 error.SkipZigTest => .skip,
146 else => s: {
146 if (@errorReturnTrace()) |trace| {147 if (@errorReturnTrace()) |trace| {
147 std.debug.dumpStackTrace(trace);148 std.debug.dumpStackTrace(trace);
148 }149 }
150 break :s .fail;
149 },151 },
150 };152 };
151 const leak = testing.allocator_instance.deinit() == .leak;153 const leak_count = testing.allocator_instance.detectLeaks();
154 testing.allocator_instance.deinitWithoutLeakChecks();
152 try server.serveTestResults(.{155 try server.serveTestResults(.{
153 .index = index,156 .index = index,
154 .flags = .{157 .flags = .{
155 .fail = fail,158 .status = status,
156 .skip = skip,
157 .leak = leak,
158 .fuzz = is_fuzz_test,159 .fuzz = is_fuzz_test,
159 .log_err_count = std.math.lossyCast(160 .log_err_count = std.math.lossyCast(
160 @FieldType(std.zig.Server.Message.TestResults.Flags, "log_err_count"),161 @FieldType(TestResults.Flags, "log_err_count"),
161 log_err_count,162 log_err_count,
162 ),163 ),
164 .leak_count = std.math.lossyCast(
165 @FieldType(TestResults.Flags, "leak_count"),
166 leak_count,
167 ),
163 },168 },
164 });169 });
165 },170 },
lib/std/Build/Step.zig+29-5
...@@ -63,19 +63,43 @@ test_results: TestResults,...@@ -63,19 +63,43 @@ test_results: TestResults,
63debug_stack_trace: std.builtin.StackTrace,63debug_stack_trace: std.builtin.StackTrace,
6464
65pub const TestResults = struct {65pub const TestResults = struct {
66 fail_count: u32 = 0,66 /// The total number of tests in the step. Every test has a "status" from the following:
67 /// * passed
68 /// * skipped
69 /// * failed cleanly
70 /// * crashed
71 /// * timed out
72 test_count: u32 = 0,
73
74 /// The number of tests which were skipped (`error.SkipZigTest`).
67 skip_count: u32 = 0,75 skip_count: u32 = 0,
68 leak_count: u32 = 0,76 /// The number of tests which failed cleanly.
77 fail_count: u32 = 0,
78 /// The number of tests which terminated unexpectedly, i.e. crashed.
79 crash_count: u32 = 0,
80 /// The number of tests which timed out.
69 timeout_count: u32 = 0,81 timeout_count: u32 = 0,
82
83 /// The number of detected memory leaks. The associated test may still have passed; indeed, *all*
84 /// individual tests may have passed. However, the step as a whole fails if any test has leaks.
85 leak_count: u32 = 0,
86 /// The number of detected error logs. The associated test may still have passed; indeed, *all*
87 /// individual tests may have passed. However, the step as a whole fails if any test logs errors.
70 log_err_count: u32 = 0,88 log_err_count: u32 = 0,
71 test_count: u32 = 0,
7289
73 pub fn isSuccess(tr: TestResults) bool {90 pub fn isSuccess(tr: TestResults) bool {
74 return tr.fail_count == 0 and tr.leak_count == 0 and tr.log_err_count == 0 and tr.timeout_count == 0;91 // all steps are success or skip
92 return tr.fail_count == 0 and
93 tr.crash_count == 0 and
94 tr.timeout_count == 0 and
95 // no (otherwise successful) step leaked memory or logged errors
96 tr.leak_count == 0 and
97 tr.log_err_count == 0;
75 }98 }
7699
100 /// Computes the number of tests which passed from the other values.
77 pub fn passCount(tr: TestResults) u32 {101 pub fn passCount(tr: TestResults) u32 {
78 return tr.test_count - tr.fail_count - tr.skip_count - tr.timeout_count;102 return tr.test_count - tr.skip_count - tr.fail_count - tr.crash_count - tr.timeout_count;
79 }103 }
80};104};
81105
lib/std/Build/Step/Run.zig+322-289
...@@ -630,6 +630,8 @@ pub fn addCheck(run: *Run, new_check: StdIo.Check) void {...@@ -630,6 +630,8 @@ pub fn addCheck(run: *Run, new_check: StdIo.Check) void {
630630
631pub fn captureStdErr(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPath {631pub fn captureStdErr(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPath {
632 assert(run.stdio != .inherit);632 assert(run.stdio != .inherit);
633 assert(run.stdio != .zig_test);
634
633 const b = run.step.owner;635 const b = run.step.owner;
634636
635 if (run.captured_stderr) |captured| return .{ .generated = .{ .file = &captured.output.generated_file } };637 if (run.captured_stderr) |captured| return .{ .generated = .{ .file = &captured.output.generated_file } };
...@@ -649,6 +651,8 @@ pub fn captureStdErr(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa...@@ -649,6 +651,8 @@ pub fn captureStdErr(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPa
649651
650pub fn captureStdOut(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPath {652pub fn captureStdOut(run: *Run, options: CapturedStdIo.Options) std.Build.LazyPath {
651 assert(run.stdio != .inherit);653 assert(run.stdio != .inherit);
654 assert(run.stdio != .zig_test);
655
652 const b = run.step.owner;656 const b = run.step.owner;
653657
654 if (run.captured_stdout) |captured| return .{ .generated = .{ .file = &captured.output.generated_file } };658 if (run.captured_stdout) |captured| return .{ .generated = .{ .file = &captured.output.generated_file } };
...@@ -1224,7 +1228,7 @@ fn runCommand(...@@ -1224,7 +1228,7 @@ fn runCommand(
12241228
1225 var env_map = run.env_map orelse &b.graph.env_map;1229 var env_map = run.env_map orelse &b.graph.env_map;
12261230
1227 const result = spawnChildAndCollect(run, argv, env_map, has_side_effects, options, fuzz_context) catch |err| term: {1231 const opt_generic_result = spawnChildAndCollect(run, argv, env_map, has_side_effects, options, fuzz_context) catch |err| term: {
1228 // InvalidExe: cpu arch mismatch1232 // InvalidExe: cpu arch mismatch
1229 // FileNotFound: can happen with a wrong dynamic linker path1233 // FileNotFound: can happen with a wrong dynamic linker path
1230 if (err == error.InvalidExe or err == error.FileNotFound) interpret: {1234 if (err == error.InvalidExe or err == error.FileNotFound) interpret: {
...@@ -1365,34 +1369,34 @@ fn runCommand(...@@ -1365,34 +1369,34 @@ fn runCommand(
13651369
1366 break :term spawnChildAndCollect(run, interp_argv.items, env_map, has_side_effects, options, fuzz_context) catch |e| {1370 break :term spawnChildAndCollect(run, interp_argv.items, env_map, has_side_effects, options, fuzz_context) catch |e| {
1367 if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;1371 if (!run.failing_to_execute_foreign_is_an_error) return error.MakeSkipped;
13681372 if (e == error.MakeFailed) return error.MakeFailed; // error already reported
1369 return step.fail("unable to spawn interpreter {s}: {s}", .{1373 return step.fail("unable to spawn interpreter {s}: {s}", .{
1370 interp_argv.items[0], @errorName(e),1374 interp_argv.items[0], @errorName(e),
1371 });1375 });
1372 };1376 };
1373 }1377 }
1378 if (err == error.MakeFailed) return error.MakeFailed; // error already reported
13741379
1375 return step.fail("failed to spawn and capture stdio from {s}: {s}", .{ argv[0], @errorName(err) });1380 return step.fail("failed to spawn and capture stdio from {s}: {s}", .{ argv[0], @errorName(err) });
1376 };1381 };
13771382
1378 step.result_duration_ns = result.elapsed_ns;
1379 step.result_peak_rss = result.peak_rss;
1380 step.test_results = result.stdio.test_results;
1381 if (result.stdio.test_metadata) |tm| {
1382 run.cached_test_metadata = tm.toCachedTestMetadata();
1383 if (options.web_server) |ws| ws.updateTimeReportRunTest(
1384 run,
1385 &run.cached_test_metadata.?,
1386 tm.ns_per_test,
1387 );
1388 }
1389
1390 const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;1383 const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items;
13911384
1392 if (fuzz_context != null) {1385 const generic_result = opt_generic_result orelse {
1393 try step.handleChildProcessTerm(result.term, cwd, final_argv);1386 assert(run.stdio == .zig_test);
1387 // Specific errors have already been reported. All we need to do is detect those and
1388 // report the general "test failed" error, which includes the command argv.
1389 if (!step.test_results.isSuccess() or step.result_error_msgs.items.len > 0) {
1390 return step.fail(
1391 "the following test command failed:\n{s}",
1392 .{try Step.allocPrintCmd(arena, cwd, final_argv)},
1393 );
1394 }
1394 return;1395 return;
1395 }1396 };
1397
1398 assert(fuzz_context == null);
1399 assert(run.stdio != .zig_test);
13961400
1397 // Capture stdout and stderr to GeneratedFile objects.1401 // Capture stdout and stderr to GeneratedFile objects.
1398 const Stream = struct {1402 const Stream = struct {
...@@ -1402,11 +1406,11 @@ fn runCommand(...@@ -1402,11 +1406,11 @@ fn runCommand(
1402 for ([_]Stream{1406 for ([_]Stream{
1403 .{1407 .{
1404 .captured = run.captured_stdout,1408 .captured = run.captured_stdout,
1405 .bytes = result.stdio.stdout,1409 .bytes = generic_result.stdout,
1406 },1410 },
1407 .{1411 .{
1408 .captured = run.captured_stderr,1412 .captured = run.captured_stderr,
1409 .bytes = result.stdio.stderr,1413 .bytes = generic_result.stderr,
1410 },1414 },
1411 }) |stream| {1415 }) |stream| {
1412 if (stream.captured) |captured| {1416 if (stream.captured) |captured| {
...@@ -1436,9 +1440,10 @@ fn runCommand(...@@ -1436,9 +1440,10 @@ fn runCommand(
1436 }1440 }
14371441
1438 switch (run.stdio) {1442 switch (run.stdio) {
1443 .zig_test => unreachable,
1439 .check => |checks| for (checks.items) |check| switch (check) {1444 .check => |checks| for (checks.items) |check| switch (check) {
1440 .expect_stderr_exact => |expected_bytes| {1445 .expect_stderr_exact => |expected_bytes| {
1441 if (!mem.eql(u8, expected_bytes, result.stdio.stderr.?)) {1446 if (!mem.eql(u8, expected_bytes, generic_result.stderr.?)) {
1442 return step.fail(1447 return step.fail(
1443 \\1448 \\
1444 \\========= expected this stderr: =========1449 \\========= expected this stderr: =========
...@@ -1449,13 +1454,13 @@ fn runCommand(...@@ -1449,13 +1454,13 @@ fn runCommand(
1449 \\{s}1454 \\{s}
1450 , .{1455 , .{
1451 expected_bytes,1456 expected_bytes,
1452 result.stdio.stderr.?,1457 generic_result.stderr.?,
1453 try Step.allocPrintCmd(arena, cwd, final_argv),1458 try Step.allocPrintCmd(arena, cwd, final_argv),
1454 });1459 });
1455 }1460 }
1456 },1461 },
1457 .expect_stderr_match => |match| {1462 .expect_stderr_match => |match| {
1458 if (mem.indexOf(u8, result.stdio.stderr.?, match) == null) {1463 if (mem.indexOf(u8, generic_result.stderr.?, match) == null) {
1459 return step.fail(1464 return step.fail(
1460 \\1465 \\
1461 \\========= expected to find in stderr: =========1466 \\========= expected to find in stderr: =========
...@@ -1466,13 +1471,13 @@ fn runCommand(...@@ -1466,13 +1471,13 @@ fn runCommand(
1466 \\{s}1471 \\{s}
1467 , .{1472 , .{
1468 match,1473 match,
1469 result.stdio.stderr.?,1474 generic_result.stderr.?,
1470 try Step.allocPrintCmd(arena, cwd, final_argv),1475 try Step.allocPrintCmd(arena, cwd, final_argv),
1471 });1476 });
1472 }1477 }
1473 },1478 },
1474 .expect_stdout_exact => |expected_bytes| {1479 .expect_stdout_exact => |expected_bytes| {
1475 if (!mem.eql(u8, expected_bytes, result.stdio.stdout.?)) {1480 if (!mem.eql(u8, expected_bytes, generic_result.stdout.?)) {
1476 return step.fail(1481 return step.fail(
1477 \\1482 \\
1478 \\========= expected this stdout: =========1483 \\========= expected this stdout: =========
...@@ -1483,13 +1488,13 @@ fn runCommand(...@@ -1483,13 +1488,13 @@ fn runCommand(
1483 \\{s}1488 \\{s}
1484 , .{1489 , .{
1485 expected_bytes,1490 expected_bytes,
1486 result.stdio.stdout.?,1491 generic_result.stdout.?,
1487 try Step.allocPrintCmd(arena, cwd, final_argv),1492 try Step.allocPrintCmd(arena, cwd, final_argv),
1488 });1493 });
1489 }1494 }
1490 },1495 },
1491 .expect_stdout_match => |match| {1496 .expect_stdout_match => |match| {
1492 if (mem.indexOf(u8, result.stdio.stdout.?, match) == null) {1497 if (mem.indexOf(u8, generic_result.stdout.?, match) == null) {
1493 return step.fail(1498 return step.fail(
1494 \\1499 \\
1495 \\========= expected to find in stdout: =========1500 \\========= expected to find in stdout: =========
...@@ -1500,69 +1505,45 @@ fn runCommand(...@@ -1500,69 +1505,45 @@ fn runCommand(
1500 \\{s}1505 \\{s}
1501 , .{1506 , .{
1502 match,1507 match,
1503 result.stdio.stdout.?,1508 generic_result.stdout.?,
1504 try Step.allocPrintCmd(arena, cwd, final_argv),1509 try Step.allocPrintCmd(arena, cwd, final_argv),
1505 });1510 });
1506 }1511 }
1507 },1512 },
1508 .expect_term => |expected_term| {1513 .expect_term => |expected_term| {
1509 if (!termMatches(expected_term, result.term)) {1514 if (!termMatches(expected_term, generic_result.term)) {
1510 return step.fail("the following command {f} (expected {f}):\n{s}", .{1515 return step.fail("the following command {f} (expected {f}):\n{s}", .{
1511 fmtTerm(result.term),1516 fmtTerm(generic_result.term),
1512 fmtTerm(expected_term),1517 fmtTerm(expected_term),
1513 try Step.allocPrintCmd(arena, cwd, final_argv),1518 try Step.allocPrintCmd(arena, cwd, final_argv),
1514 });1519 });
1515 }1520 }
1516 },1521 },
1517 },1522 },
1518 .zig_test => {
1519 const prefix: []const u8 = p: {
1520 if (result.stdio.test_metadata) |tm| {
1521 if (tm.next_index > 0 and tm.next_index <= tm.names.len) {
1522 const name = tm.testName(tm.next_index - 1);
1523 break :p b.fmt("while executing test '{s}', ", .{name});
1524 }
1525 }
1526 break :p "";
1527 };
1528 const expected_term: std.process.Child.Term = .{ .Exited = 0 };
1529 if (!termMatches(expected_term, result.term)) {
1530 return step.fail("{s}the following command {f} (expected {f}):\n{s}", .{
1531 prefix,
1532 fmtTerm(result.term),
1533 fmtTerm(expected_term),
1534 try Step.allocPrintCmd(arena, cwd, final_argv),
1535 });
1536 }
1537 if (!result.stdio.test_results.isSuccess()) {
1538 return step.fail(
1539 "{s}the following test command failed:\n{s}",
1540 .{ prefix, try Step.allocPrintCmd(arena, cwd, final_argv) },
1541 );
1542 }
1543 },
1544 else => {1523 else => {
1545 // On failure, print stderr if captured.1524 // On failure, print stderr if captured.
1546 const bad_exit = switch (result.term) {1525 const bad_exit = switch (generic_result.term) {
1547 .Exited => |code| code != 0,1526 .Exited => |code| code != 0,
1548 .Signal, .Stopped, .Unknown => true,1527 .Signal, .Stopped, .Unknown => true,
1549 };1528 };
15501529
1551 if (bad_exit) if (result.stdio.stderr) |err| {1530 if (bad_exit) if (generic_result.stderr) |err| {
1552 try step.addError("stderr:\n{s}", .{err});1531 try step.addError("stderr:\n{s}", .{err});
1553 };1532 };
15541533
1555 try step.handleChildProcessTerm(result.term, cwd, final_argv);1534 try step.handleChildProcessTerm(generic_result.term, cwd, final_argv);
1556 },1535 },
1557 }1536 }
1558}1537}
15591538
1560const ChildProcResult = struct {1539const EvalZigTestResult = struct {
1540 test_results: Step.TestResults,
1541 test_metadata: ?TestMetadata,
1542};
1543const EvalGenericResult = struct {
1561 term: std.process.Child.Term,1544 term: std.process.Child.Term,
1562 elapsed_ns: u64,1545 stdout: ?[]const u8,
1563 peak_rss: usize,1546 stderr: ?[]const u8,
1564
1565 stdio: StdIoResult,
1566};1547};
15671548
1568fn spawnChildAndCollect(1549fn spawnChildAndCollect(
...@@ -1572,7 +1553,7 @@ fn spawnChildAndCollect(...@@ -1572,7 +1553,7 @@ fn spawnChildAndCollect(
1572 has_side_effects: bool,1553 has_side_effects: bool,
1573 options: Step.MakeOptions,1554 options: Step.MakeOptions,
1574 fuzz_context: ?FuzzContext,1555 fuzz_context: ?FuzzContext,
1575) !ChildProcResult {1556) !?EvalGenericResult {
1576 const b = run.step.owner;1557 const b = run.step.owner;
1577 const arena = b.allocator;1558 const arena = b.allocator;
15781559
...@@ -1613,121 +1594,237 @@ fn spawnChildAndCollect(...@@ -1613,121 +1594,237 @@ fn spawnChildAndCollect(
1613 child.stdin_behavior = .Pipe;1594 child.stdin_behavior = .Pipe;
1614 }1595 }
16151596
1616 const inherit = child.stdout_behavior == .Inherit or child.stderr_behavior == .Inherit;1597 if (run.stdio == .zig_test) {
16171598 var timer = try std.time.Timer.start();
1618 if (run.stdio != .zig_test and !run.disable_zig_progress and !inherit) {1599 const res = try evalZigTest(run, &child, options, fuzz_context);
1619 child.progress_node = options.progress_node;1600 run.step.result_duration_ns = timer.read();
1620 }1601 run.step.test_results = res.test_results;
16211602 if (res.test_metadata) |tm| {
1622 const term, const result, const elapsed_ns = t: {1603 run.cached_test_metadata = tm.toCachedTestMetadata();
1604 if (options.web_server) |ws| ws.updateTimeReportRunTest(
1605 run,
1606 &run.cached_test_metadata.?,
1607 tm.ns_per_test,
1608 );
1609 }
1610 return null;
1611 } else {
1612 const inherit = child.stdout_behavior == .Inherit or child.stderr_behavior == .Inherit;
1613 if (!run.disable_zig_progress and !inherit) {
1614 child.progress_node = options.progress_node;
1615 }
1623 if (inherit) std.debug.lockStdErr();1616 if (inherit) std.debug.lockStdErr();
1624 defer if (inherit) std.debug.unlockStdErr();1617 defer if (inherit) std.debug.unlockStdErr();
1618 var timer = try std.time.Timer.start();
1619 const res = try evalGeneric(run, &child);
1620 run.step.result_duration_ns = timer.read();
1621 return .{ .term = res.term, .stdout = res.stdout, .stderr = res.stderr };
1622 }
1623}
1624
1625const StdioPollEnum = enum { stdout, stderr };
1626
1627fn evalZigTest(
1628 run: *Run,
1629 child: *std.process.Child,
1630 options: Step.MakeOptions,
1631 fuzz_context: ?FuzzContext,
1632) !EvalZigTestResult {
1633 const gpa = run.step.owner.allocator;
1634 const arena = run.step.owner.allocator;
1635
1636 // We will update this every time a child runs.
1637 run.step.result_peak_rss = 0;
1638
1639 var result: EvalZigTestResult = .{
1640 .test_results = .{
1641 .test_count = 0,
1642 .skip_count = 0,
1643 .fail_count = 0,
1644 .crash_count = 0,
1645 .timeout_count = 0,
1646 .leak_count = 0,
1647 .log_err_count = 0,
1648 },
1649 .test_metadata = null,
1650 };
16251651
1652 while (true) {
1626 try child.spawn();1653 try child.spawn();
1627 errdefer {1654 var poller = std.Io.poll(gpa, StdioPollEnum, .{
1655 .stdout = child.stdout.?,
1656 .stderr = child.stderr.?,
1657 });
1658 var child_killed = false;
1659 defer if (!child_killed) {
1628 _ = child.kill() catch {};1660 _ = child.kill() catch {};
1629 }1661 poller.deinit();
1662 run.step.result_peak_rss = @max(
1663 run.step.result_peak_rss,
1664 child.resource_usage_statistics.getMaxRss() orelse 0,
1665 );
1666 };
16301667
1631 // We need to report `error.InvalidExe` *now* if applicable.
1632 try child.waitForSpawn();1668 try child.waitForSpawn();
16331669
1634 var timer = try std.time.Timer.start();1670 switch (try pollZigTest(
1671 run,
1672 child,
1673 options,
1674 fuzz_context,
1675 &poller,
1676 &result.test_metadata,
1677 &result.test_results,
1678 )) {
1679 .write_failed => |err| {
1680 // The runner unexpectedly closed a stdio pipe, which means a crash. Make sure we've captured
1681 // all available stderr to make our error output as useful as possible.
1682 while (try poller.poll()) {}
1683 run.step.result_stderr = try arena.dupe(u8, poller.reader(.stderr).buffered());
1684
1685 // Clean up everything and wait for the child to exit.
1686 child.stdin.?.close();
1687 child.stdin = null;
1688 poller.deinit();
1689 child_killed = true;
1690 const term = try child.wait();
1691 run.step.result_peak_rss = @max(
1692 run.step.result_peak_rss,
1693 child.resource_usage_statistics.getMaxRss() orelse 0,
1694 );
16351695
1636 const result = if (run.stdio == .zig_test)1696 try run.step.addError("unable to write stdin ({t}); test process unexpectedly {f}", .{ err, fmtTerm(term) });
1637 try evalZigTest(run, &child, options, fuzz_context)1697 return result;
1638 else1698 },
1639 try evalGeneric(run, &child);1699 .no_poll => |no_poll| {
1700 // This might be a success (we requested exit and the child dutifully closed stdout) or
1701 // a crash of some kind. Either way, the child will terminate by itself -- wait for it.
1702 const stderr_owned = try arena.dupe(u8, poller.reader(.stderr).buffered());
1703 poller.reader(.stderr).tossBuffered();
1704
1705 // Clean up everything and wait for the child to exit.
1706 child.stdin.?.close();
1707 child.stdin = null;
1708 poller.deinit();
1709 child_killed = true;
1710 const term = try child.wait();
1711 run.step.result_peak_rss = @max(
1712 run.step.result_peak_rss,
1713 child.resource_usage_statistics.getMaxRss() orelse 0,
1714 );
16401715
1641 break :t .{ try child.wait(), result, timer.read() };1716 if (no_poll.active_test_index) |test_index| {
1642 };1717 // A test was running, so this is definitely a crash. Report it against that
1718 // test, and continue to the next test.
1719 result.test_metadata.?.ns_per_test[test_index] = no_poll.ns_elapsed;
1720 result.test_results.crash_count += 1;
1721 try run.step.addError("'{s}' {f}{s}{s}", .{
1722 result.test_metadata.?.testName(test_index),
1723 fmtTerm(term),
1724 if (stderr_owned.len != 0) " with stderr:\n" else "",
1725 std.mem.trim(u8, stderr_owned, "\n"),
1726 });
1727 continue;
1728 }
16431729
1644 return .{1730 // Report an error if the child terminated uncleanly or if we were still trying to run more tests.
1645 .stdio = result,1731 run.step.result_stderr = stderr_owned;
1646 .term = term,1732 const tests_done = result.test_metadata != null and result.test_metadata.?.next_index == std.math.maxInt(u32);
1647 .elapsed_ns = elapsed_ns,1733 if (!tests_done or !termMatches(.{ .Exited = 0 }, term)) {
1648 .peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0,1734 try run.step.addError("test process unexpectedly {f}", .{fmtTerm(term)});
1649 };1735 }
1736 return result;
1737 },
1738 .timeout => |timeout| {
1739 const stderr = poller.reader(.stderr).buffered();
1740 poller.reader(.stderr).tossBuffered();
1741 if (timeout.active_test_index) |test_index| {
1742 // A test was running. Report the timeout against that test, and continue on to
1743 // the next test.
1744 result.test_metadata.?.ns_per_test[test_index] = timeout.ns_elapsed;
1745 result.test_results.timeout_count += 1;
1746 try run.step.addError("'{s}' timed out after {D}{s}{s}", .{
1747 result.test_metadata.?.testName(test_index),
1748 timeout.ns_elapsed,
1749 if (stderr.len != 0) " with stderr:\n" else "",
1750 std.mem.trim(u8, stderr, "\n"),
1751 });
1752 continue;
1753 }
1754 // Just log an error and let the child be killed.
1755 run.step.result_stderr = try arena.dupe(u8, stderr);
1756 return run.step.fail("test runner failed to respond for {D}", .{timeout.ns_elapsed});
1757 },
1758 }
1759 comptime unreachable;
1760 }
1650}1761}
16511762
1652const StdIoResult = struct {1763/// Polls stdout of a Zig test process until a termination condition is reached:
1653 stdout: ?[]const u8,1764/// * A write fails, indicating the child unexpectedly closed stdin
1654 stderr: ?[]const u8,1765/// * A test (or a response from the test runner) times out
1655 test_results: Step.TestResults,1766/// * `poll` fails, indicating the child closed stdout and stderr
1656 test_metadata: ?TestMetadata,1767fn pollZigTest(
1657};
1658
1659fn evalZigTest(
1660 run: *Run,1768 run: *Run,
1661 child: *std.process.Child,1769 child: *std.process.Child,
1662 options: Step.MakeOptions,1770 options: Step.MakeOptions,
1663 fuzz_context: ?FuzzContext,1771 fuzz_context: ?FuzzContext,
1664) !StdIoResult {1772 poller: *std.Io.Poller(StdioPollEnum),
1773 opt_metadata: *?TestMetadata,
1774 results: *Step.TestResults,
1775) !union(enum) {
1776 write_failed: anyerror,
1777 no_poll: struct {
1778 active_test_index: ?u32,
1779 ns_elapsed: u64,
1780 },
1781 timeout: struct {
1782 active_test_index: ?u32,
1783 ns_elapsed: u64,
1784 },
1785} {
1665 const gpa = run.step.owner.allocator;1786 const gpa = run.step.owner.allocator;
1666 const arena = run.step.owner.allocator;1787 const arena = run.step.owner.allocator;
16671788
1668 const PollEnum = enum { stdout, stderr };1789 var sub_prog_node: ?std.Progress.Node = null;
16691790 defer if (sub_prog_node) |n| n.end();
1670 var poller = std.Io.poll(gpa, PollEnum, .{
1671 .stdout = child.stdout.?,
1672 .stderr = child.stderr.?,
1673 });
1674 defer poller.deinit();
16751791
1676 // If this is `true`, we avoid ever entering the polling loop below, because the stdin pipe has1792 if (fuzz_context) |ctx| {
1677 // somehow already closed; instead, we go straight to capturing stderr in case it has anything1793 assert(opt_metadata.* == null); // fuzz processes are never restarted
1678 // useful.1794 switch (ctx.fuzz.mode) {
1679 const first_write_failed = if (fuzz_context) |fctx| failed: {
1680 switch (fctx.fuzz.mode) {
1681 .forever => {1795 .forever => {
1682 const instance_id = 0; // will be used by mutiprocess forever fuzzing1796 sendRunFuzzTestMessage(
1683 sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .forever, instance_id) catch |err| {1797 child.stdin.?,
1684 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});1798 ctx.unit_test_index,
1685 break :failed true;1799 .forever,
1686 };1800 0, // instance ID; will be used by multiprocess forever fuzzing in the future
1687 break :failed false;1801 ) catch |err| return .{ .write_failed = err };
1688 },1802 },
1689 .limit => |limit| {1803 .limit => |limit| {
1690 sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .iterations, limit.amount) catch |err| {1804 sendRunFuzzTestMessage(
1691 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});1805 child.stdin.?,
1692 break :failed true;1806 ctx.unit_test_index,
1693 };1807 .iterations,
1694 break :failed false;1808 limit.amount,
1809 ) catch |err| return .{ .write_failed = err };
1695 },1810 },
1696 }1811 }
1697 } else failed: {1812 } else if (opt_metadata.*) |*md| {
1813 // Previous unit test process died or was killed; we're continuing where it left off
1814 requestNextTest(child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err };
1815 } else {
1816 // Running unit tests normally
1698 run.fuzz_tests.clearRetainingCapacity();1817 run.fuzz_tests.clearRetainingCapacity();
1699 sendMessage(child.stdin.?, .query_test_metadata) catch |err| {1818 sendMessage(child.stdin.?, .query_test_metadata) catch |err| return .{ .write_failed = err };
1700 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});1819 }
1701 break :failed true;
1702 };
1703 break :failed false;
1704 };
1705
1706 var fail_count: u32 = 0;
1707 var skip_count: u32 = 0;
1708 var leak_count: u32 = 0;
1709 var timeout_count: u32 = 0;
1710 var test_count: u32 = 0;
1711 var log_err_count: u32 = 0;
1712
1713 var metadata: ?TestMetadata = null;
1714 var coverage_id: ?u64 = null;
1715
1716 var test_is_running = false;
17171820
1718 // String allocated into `gpa`. Owned by this function while it runs, then moved to the `Step`.1821 var active_test_index: ?u32 = null;
1719 var result_stderr: []u8 = &.{};
1720 defer run.step.result_stderr = result_stderr;
17211822
1722 // `null` means this host does not support `std.time.Timer`. This timer is `reset()` whenever we1823 // `null` means this host does not support `std.time.Timer`. This timer is `reset()` whenever we
1723 // toggle `test_is_running`, i.e. whenever a test starts or finishes.1824 // change `active_test_index`, i.e. whenever a test starts or finishes.
1724 var timer: ?std.time.Timer = std.time.Timer.start() catch t: {1825 var timer: ?std.time.Timer = std.time.Timer.start() catch null;
1725 std.log.warn("std.time.Timer not supported on host; test timeouts will be ignored", .{});
1726 break :t null;
1727 };
17281826
1729 var sub_prog_node: ?std.Progress.Node = null;1827 var coverage_id: ?u64 = null;
1730 defer if (sub_prog_node) |n| n.end();
17311828
1732 // This timeout is used when we're waiting on the test runner itself rather than a user-specified1829 // This timeout is used when we're waiting on the test runner itself rather than a user-specified
1733 // test. For instance, if the test runner leaves this much time between us requesting a test to1830 // test. For instance, if the test runner leaves this much time between us requesting a test to
...@@ -1735,12 +1832,10 @@ fn evalZigTest(...@@ -1735,12 +1832,10 @@ fn evalZigTest(
1735 // *should* never happen, but could in theory be caused by some very unlucky IB in a test.1832 // *should* never happen, but could in theory be caused by some very unlucky IB in a test.
1736 const response_timeout_ns = 30 * std.time.ns_per_s;1833 const response_timeout_ns = 30 * std.time.ns_per_s;
17371834
1738 const any_write_failed = first_write_failed or poll: while (true) {1835 const stdout = poller.reader(.stdout);
1739 // These are scoped inside the loop because we sometimes respawn the child and recreate1836 const stderr = poller.reader(.stderr);
1740 // `poller` which invaldiates these readers.
1741 const stdout = poller.reader(.stdout);
1742 const stderr = poller.reader(.stderr);
17431837
1838 while (true) {
1744 const Header = std.zig.Server.Message.Header;1839 const Header = std.zig.Server.Message.Header;
17451840
1746 // This block is exited when `stdout` contains enough bytes for a `Header`.1841 // This block is exited when `stdout` contains enough bytes for a `Header`.
...@@ -1753,15 +1848,21 @@ fn evalZigTest(...@@ -1753,15 +1848,21 @@ fn evalZigTest(
1753 // Always `null` if `timer` is `null`.1848 // Always `null` if `timer` is `null`.
1754 const opt_timeout_ns: ?u64 = ns: {1849 const opt_timeout_ns: ?u64 = ns: {
1755 if (timer == null) break :ns null;1850 if (timer == null) break :ns null;
1756 if (!test_is_running) break :ns response_timeout_ns;1851 if (active_test_index == null) break :ns response_timeout_ns;
1757 break :ns options.unit_test_timeout_ns;1852 break :ns options.unit_test_timeout_ns;
1758 };1853 };
17591854
1760 if (opt_timeout_ns) |timeout_ns| {1855 if (opt_timeout_ns) |timeout_ns| {
1761 const remaining_ns = timeout_ns -| timer.?.read();1856 const remaining_ns = timeout_ns -| timer.?.read();
1762 if (!try poller.pollTimeout(remaining_ns)) break :poll false;1857 if (!try poller.pollTimeout(remaining_ns)) return .{ .no_poll = .{
1858 .active_test_index = active_test_index,
1859 .ns_elapsed = if (timer) |*t| t.read() else 0,
1860 } };
1763 } else {1861 } else {
1764 if (!try poller.poll()) break :poll false;1862 if (!try poller.poll()) return .{ .no_poll = .{
1863 .active_test_index = active_test_index,
1864 .ns_elapsed = if (timer) |*t| t.read() else 0,
1865 } };
1765 }1866 }
17661867
1767 if (stdout.buffered().len >= @sizeOf(Header)) {1868 if (stdout.buffered().len >= @sizeOf(Header)) {
...@@ -1769,73 +1870,29 @@ fn evalZigTest(...@@ -1769,73 +1870,29 @@ fn evalZigTest(
1769 break :header_ready;1870 break :header_ready;
1770 }1871 }
17711872
1772 const timeout_ns = opt_timeout_ns orelse continue;1873 if (opt_timeout_ns) |timeout_ns| {
1773 const cur_ns = timer.?.read();1874 const cur_ns = timer.?.read();
1774 if (cur_ns < timeout_ns) continue;1875 if (cur_ns >= timeout_ns) return .{ .timeout = .{
17751876 .active_test_index = active_test_index,
1776 // There was a timeout.1877 .ns_elapsed = cur_ns,
17771878 } };
1778 if (!test_is_running) {
1779 // The child stopped responding while *not* running a test. To avoid getting into
1780 // a loop if something's broken, don't retry; just report an error and stop.
1781 try run.step.addError("test runner failed to respond for {D}", .{cur_ns});
1782 break :poll false;
1783 }
1784
1785 // A test has probably just gotten stuck. We'll report an error, then just kill the
1786 // child and continue with the next test in the list.
1787
1788 const md = &metadata.?;
1789 const test_index = md.next_index - 1;
1790
1791 timeout_count += 1;
1792 try run.step.addError(
1793 "'{s}' timed out after {D}",
1794 .{ md.testName(test_index), cur_ns },
1795 );
1796 if (stderr.buffered().len > 0) {
1797 const new_bytes = stderr.buffered();
1798 const old_len = result_stderr.len;
1799 result_stderr = try gpa.realloc(result_stderr, old_len + new_bytes.len);
1800 @memcpy(result_stderr[old_len..], new_bytes);
1801 }1879 }
18021880 continue;
1803 _ = try child.kill();
1804 // Respawn the test runner. There's a double-cleanup if this fails, but that's
1805 // fine because our caller's `kill` will just return `error.AlreadyTerminated`.
1806 try child.spawn();
1807 try child.waitForSpawn();
1808
1809 // After respawning the child, we must update the poller's streams.
1810 poller.deinit();
1811 poller = std.Io.poll(gpa, PollEnum, .{
1812 .stdout = child.stdout.?,
1813 .stderr = child.stderr.?,
1814 });
1815
1816 test_is_running = false;
1817 md.ns_per_test[test_index] = timer.?.lap();
1818
1819 requestNextTest(child.stdin.?, md, &sub_prog_node) catch |err| {
1820 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1821 break :poll true;
1822 };
1823
1824 continue :poll; // continue work with the new (respawned) child
1825 }1881 }
1826 // There is definitely a header available now -- read it.1882 // There is definitely a header available now -- read it.
1827 const header = stdout.takeStruct(Header, .little) catch unreachable;1883 const header = stdout.takeStruct(Header, .little) catch unreachable;
18281884
1829 while (stdout.buffered().len < header.bytes_len) if (!try poller.poll()) break :poll false;1885 while (stdout.buffered().len < header.bytes_len) if (!try poller.poll()) return .{ .no_poll = .{
1886 .active_test_index = active_test_index,
1887 .ns_elapsed = if (timer) |*t| t.read() else 0,
1888 } };
1830 const body = stdout.take(header.bytes_len) catch unreachable;1889 const body = stdout.take(header.bytes_len) catch unreachable;
1831 switch (header.tag) {1890 switch (header.tag) {
1832 .zig_version => {1891 .zig_version => {
1833 if (!std.mem.eql(u8, builtin.zig_version_string, body)) {1892 if (!std.mem.eql(u8, builtin.zig_version_string, body)) return run.step.fail(
1834 return run.step.fail(1893 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",
1835 "zig version mismatch build runner vs compiler: '{s}' vs '{s}'",1894 .{ builtin.zig_version_string, body },
1836 .{ builtin.zig_version_string, body },1895 );
1837 );
1838 }
1839 },1896 },
1840 .test_metadata => {1897 .test_metadata => {
1841 assert(fuzz_context == null);1898 assert(fuzz_context == null);
...@@ -1843,14 +1900,14 @@ fn evalZigTest(...@@ -1843,14 +1900,14 @@ fn evalZigTest(
1843 // `metadata` would only be populated if we'd already seen a `test_metadata`, but we1900 // `metadata` would only be populated if we'd already seen a `test_metadata`, but we
1844 // only request it once (and importantly, we don't re-request it if we kill and1901 // only request it once (and importantly, we don't re-request it if we kill and
1845 // restart the test runner).1902 // restart the test runner).
1846 assert(metadata == null);1903 assert(opt_metadata.* == null);
18471904
1848 const TmHdr = std.zig.Server.Message.TestMetadata;1905 const TmHdr = std.zig.Server.Message.TestMetadata;
1849 const tm_hdr = @as(*align(1) const TmHdr, @ptrCast(body));1906 const tm_hdr: *align(1) const TmHdr = @ptrCast(body);
1850 test_count = tm_hdr.tests_len;1907 results.test_count = tm_hdr.tests_len;
18511908
1852 const names_bytes = body[@sizeOf(TmHdr)..][0 .. test_count * @sizeOf(u32)];1909 const names_bytes = body[@sizeOf(TmHdr)..][0 .. results.test_count * @sizeOf(u32)];
1853 const expected_panic_msgs_bytes = body[@sizeOf(TmHdr) + names_bytes.len ..][0 .. test_count * @sizeOf(u32)];1910 const expected_panic_msgs_bytes = body[@sizeOf(TmHdr) + names_bytes.len ..][0 .. results.test_count * @sizeOf(u32)];
1854 const string_bytes = body[@sizeOf(TmHdr) + names_bytes.len + expected_panic_msgs_bytes.len ..][0..tm_hdr.string_bytes_len];1911 const string_bytes = body[@sizeOf(TmHdr) + names_bytes.len + expected_panic_msgs_bytes.len ..][0..tm_hdr.string_bytes_len];
18551912
1856 const names = std.mem.bytesAsSlice(u32, names_bytes);1913 const names = std.mem.bytesAsSlice(u32, names_bytes);
...@@ -1863,68 +1920,70 @@ fn evalZigTest(...@@ -1863,68 +1920,70 @@ fn evalZigTest(
1863 for (expected_panic_msgs_aligned, expected_panic_msgs) |*dest, src| dest.* = src;1920 for (expected_panic_msgs_aligned, expected_panic_msgs) |*dest, src| dest.* = src;
18641921
1865 options.progress_node.setEstimatedTotalItems(names.len);1922 options.progress_node.setEstimatedTotalItems(names.len);
1866 metadata = .{1923 opt_metadata.* = .{
1867 .string_bytes = try arena.dupe(u8, string_bytes),1924 .string_bytes = try arena.dupe(u8, string_bytes),
1868 .ns_per_test = try arena.alloc(u64, test_count),1925 .ns_per_test = try arena.alloc(u64, results.test_count),
1869 .names = names_aligned,1926 .names = names_aligned,
1870 .expected_panic_msgs = expected_panic_msgs_aligned,1927 .expected_panic_msgs = expected_panic_msgs_aligned,
1871 .next_index = 0,1928 .next_index = 0,
1872 .prog_node = options.progress_node,1929 .prog_node = options.progress_node,
1873 };1930 };
1874 @memset(metadata.?.ns_per_test, std.math.maxInt(u64));1931 @memset(opt_metadata.*.?.ns_per_test, std.math.maxInt(u64));
18751932
1876 test_is_running = false;1933 active_test_index = null;
1877 if (timer) |*t| t.reset();1934 if (timer) |*t| t.reset();
18781935
1879 requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node) catch |err| {1936 requestNextTest(child.stdin.?, &opt_metadata.*.?, &sub_prog_node) catch |err| return .{ .write_failed = err };
1880 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1881 break :poll true;
1882 };
1883 },1937 },
1884 .test_started => {1938 .test_started => {
1885 test_is_running = true;1939 active_test_index = opt_metadata.*.?.next_index - 1;
1886 if (timer) |*t| t.reset();1940 if (timer) |*t| t.reset();
1887 },1941 },
1888 .test_results => {1942 .test_results => {
1889 assert(fuzz_context == null);1943 assert(fuzz_context == null);
1890 const md = &metadata.?;1944 const md = &opt_metadata.*.?;
18911945
1892 const TrHdr = std.zig.Server.Message.TestResults;1946 const TrHdr = std.zig.Server.Message.TestResults;
1893 const tr_hdr: *align(1) const TrHdr = @ptrCast(body);1947 const tr_hdr: *align(1) const TrHdr = @ptrCast(body);
1894 fail_count +|= @intFromBool(tr_hdr.flags.fail);1948 assert(tr_hdr.index == active_test_index);
1895 skip_count +|= @intFromBool(tr_hdr.flags.skip);1949
1896 leak_count +|= @intFromBool(tr_hdr.flags.leak);1950 switch (tr_hdr.flags.status) {
1897 log_err_count +|= tr_hdr.flags.log_err_count;1951 .pass => {},
1952 .skip => results.skip_count +|= 1,
1953 .fail => results.fail_count +|= 1,
1954 }
1955 const leak_count = tr_hdr.flags.leak_count;
1956 const log_err_count = tr_hdr.flags.log_err_count;
1957 results.leak_count +|= leak_count;
1958 results.log_err_count +|= log_err_count;
18981959
1899 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);1960 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
19001961
1901 if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) {1962 if (tr_hdr.flags.status == .fail) {
1902 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);1963 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
1903 const stderr_contents = stderr.buffered();1964 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
1904 stderr.toss(stderr_contents.len);1965 stderr.tossBuffered();
1905 const msg = std.mem.trim(u8, stderr_contents, "\n");1966 if (stderr_bytes.len == 0) {
1906 const label = if (tr_hdr.flags.fail)1967 try run.step.addError("'{s}' failed without output", .{name});
1907 "failed"
1908 else if (tr_hdr.flags.leak)
1909 "leaked"
1910 else if (tr_hdr.flags.log_err_count > 0)
1911 "logged errors"
1912 else
1913 unreachable;
1914 if (msg.len > 0) {
1915 try run.step.addError("'{s}' {s}: {s}", .{ name, label, msg });
1916 } else {1968 } else {
1917 try run.step.addError("'{s}' {s}", .{ name, label });1969 try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });
1918 }1970 }
1971 } else if (leak_count > 0) {
1972 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
1973 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
1974 stderr.tossBuffered();
1975 try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });
1976 } else if (log_err_count > 0) {
1977 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
1978 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
1979 stderr.tossBuffered();
1980 try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });
1919 }1981 }
19201982
1921 test_is_running = false;1983 active_test_index = null;
1922 if (timer) |*t| md.ns_per_test[tr_hdr.index] = t.lap();1984 if (timer) |*t| md.ns_per_test[tr_hdr.index] = t.lap();
19231985
1924 requestNextTest(child.stdin.?, md, &sub_prog_node) catch |err| {1986 requestNextTest(child.stdin.?, md, &sub_prog_node) catch |err| return .{ .write_failed = err };
1925 try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
1926 break :poll true;
1927 };
1928 },1987 },
1929 .coverage_id => {1988 .coverage_id => {
1930 const fuzz = fuzz_context.?.fuzz;1989 const fuzz = fuzz_context.?.fuzz;
...@@ -1961,39 +2020,7 @@ fn evalZigTest(...@@ -1961,39 +2020,7 @@ fn evalZigTest(
1961 },2020 },
1962 else => {}, // ignore other messages2021 else => {}, // ignore other messages
1963 }2022 }
1964 };
1965
1966 if (any_write_failed) {
1967 // The compiler unexpectedly closed stdin; something is very wrong and has probably crashed.
1968 // We want to make sure we've captured all of stderr so that it's logged below.
1969 while (try poller.poll()) {}
1970 }2023 }
1971
1972 const stderr = poller.reader(.stderr);
1973 if (stderr.buffered().len > 0) {
1974 const new_bytes = stderr.buffered();
1975 const old_len = result_stderr.len;
1976 result_stderr = try gpa.realloc(result_stderr, old_len + new_bytes.len);
1977 @memcpy(result_stderr[old_len..], new_bytes);
1978 }
1979
1980 // Send EOF to stdin.
1981 child.stdin.?.close();
1982 child.stdin = null;
1983
1984 return .{
1985 .stdout = null,
1986 .stderr = null,
1987 .test_results = .{
1988 .test_count = test_count,
1989 .fail_count = fail_count,
1990 .skip_count = skip_count,
1991 .leak_count = leak_count,
1992 .timeout_count = timeout_count,
1993 .log_err_count = log_err_count,
1994 },
1995 .test_metadata = metadata,
1996 };
1997}2024}
19982025
1999const TestMetadata = struct {2026const TestMetadata = struct {
...@@ -2077,10 +2104,15 @@ fn sendRunFuzzTestMessage(...@@ -2077,10 +2104,15 @@ fn sendRunFuzzTestMessage(
2077 try file.writeAll(full_msg);2104 try file.writeAll(full_msg);
2078}2105}
20792106
2080fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {2107fn evalGeneric(run: *Run, child: *std.process.Child) !EvalGenericResult {
2081 const b = run.step.owner;2108 const b = run.step.owner;
2082 const arena = b.allocator;2109 const arena = b.allocator;
20832110
2111 try child.spawn();
2112 errdefer _ = child.kill() catch {};
2113
2114 try child.waitForSpawn();
2115
2084 switch (run.stdin) {2116 switch (run.stdin) {
2085 .bytes => |bytes| {2117 .bytes => |bytes| {
2086 child.stdin.?.writeAll(bytes) catch |err| {2118 child.stdin.?.writeAll(bytes) catch |err| {
...@@ -2170,11 +2202,12 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {...@@ -2170,11 +2202,12 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
2170 }2202 }
2171 };2203 };
21722204
2205 run.step.result_peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0;
2206
2173 return .{2207 return .{
2208 .term = try child.wait(),
2174 .stdout = stdout_bytes,2209 .stdout = stdout_bytes,
2175 .stderr = stderr_bytes,2210 .stderr = stderr_bytes,
2176 .test_results = .{},
2177 .test_metadata = null,
2178 };2211 };
2179}2212}
21802213
lib/std/heap/debug_allocator.zig+16-10
...@@ -421,10 +421,10 @@ pub fn DebugAllocator(comptime config: Config) type {...@@ -421,10 +421,10 @@ pub fn DebugAllocator(comptime config: Config) type {
421 return usedBitsCount(slot_count) * @sizeOf(usize);421 return usedBitsCount(slot_count) * @sizeOf(usize);
422 }422 }
423423
424 fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) bool {424 fn detectLeaksInBucket(bucket: *BucketHeader, size_class_index: usize, used_bits_count: usize) usize {
425 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));425 const size_class = @as(usize, 1) << @as(Log2USize, @intCast(size_class_index));
426 const slot_count = slot_counts[size_class_index];426 const slot_count = slot_counts[size_class_index];
427 var leaks = false;427 var leaks: usize = 0;
428 for (0..used_bits_count) |used_bits_byte| {428 for (0..used_bits_count) |used_bits_byte| {
429 const used_int = bucket.usedBits(used_bits_byte).*;429 const used_int = bucket.usedBits(used_bits_byte).*;
430 if (used_int != 0) {430 if (used_int != 0) {
...@@ -437,7 +437,7 @@ pub fn DebugAllocator(comptime config: Config) type {...@@ -437,7 +437,7 @@ pub fn DebugAllocator(comptime config: Config) type {
437 const page_addr = @intFromPtr(bucket) & ~(page_size - 1);437 const page_addr = @intFromPtr(bucket) & ~(page_size - 1);
438 const addr = page_addr + slot_index * size_class;438 const addr = page_addr + slot_index * size_class;
439 log.err("memory address 0x{x} leaked: {f}", .{ addr, stack_trace });439 log.err("memory address 0x{x} leaked: {f}", .{ addr, stack_trace });
440 leaks = true;440 leaks += 1;
441 }441 }
442 }442 }
443 }443 }
...@@ -445,16 +445,16 @@ pub fn DebugAllocator(comptime config: Config) type {...@@ -445,16 +445,16 @@ pub fn DebugAllocator(comptime config: Config) type {
445 return leaks;445 return leaks;
446 }446 }
447447
448 /// Emits log messages for leaks and then returns whether there were any leaks.448 /// Emits log messages for leaks and then returns the number of detected leaks (0 if no leaks were detected).
449 pub fn detectLeaks(self: *Self) bool {449 pub fn detectLeaks(self: *Self) usize {
450 var leaks = false;450 var leaks: usize = 0;
451451
452 for (self.buckets, 0..) |init_optional_bucket, size_class_index| {452 for (self.buckets, 0..) |init_optional_bucket, size_class_index| {
453 var optional_bucket = init_optional_bucket;453 var optional_bucket = init_optional_bucket;
454 const slot_count = slot_counts[size_class_index];454 const slot_count = slot_counts[size_class_index];
455 const used_bits_count = usedBitsCount(slot_count);455 const used_bits_count = usedBitsCount(slot_count);
456 while (optional_bucket) |bucket| {456 while (optional_bucket) |bucket| {
457 leaks = detectLeaksInBucket(bucket, size_class_index, used_bits_count) or leaks;457 leaks += detectLeaksInBucket(bucket, size_class_index, used_bits_count);
458 optional_bucket = bucket.prev;458 optional_bucket = bucket.prev;
459 }459 }
460 }460 }
...@@ -466,7 +466,7 @@ pub fn DebugAllocator(comptime config: Config) type {...@@ -466,7 +466,7 @@ pub fn DebugAllocator(comptime config: Config) type {
466 log.err("memory address 0x{x} leaked: {f}", .{466 log.err("memory address 0x{x} leaked: {f}", .{
467 @intFromPtr(large_alloc.bytes.ptr), stack_trace,467 @intFromPtr(large_alloc.bytes.ptr), stack_trace,
468 });468 });
469 leaks = true;469 leaks += 1;
470 }470 }
471 return leaks;471 return leaks;
472 }472 }
...@@ -498,11 +498,17 @@ pub fn DebugAllocator(comptime config: Config) type {...@@ -498,11 +498,17 @@ pub fn DebugAllocator(comptime config: Config) type {
498498
499 /// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise.499 /// Returns `std.heap.Check.leak` if there were leaks; `std.heap.Check.ok` otherwise.
500 pub fn deinit(self: *Self) std.heap.Check {500 pub fn deinit(self: *Self) std.heap.Check {
501 const leaks = if (config.safety) self.detectLeaks() else false;501 const leaks: usize = if (config.safety) self.detectLeaks() else 0;
502 self.deinitWithoutLeakChecks();
503 return if (leaks == 0) .ok else .leak;
504 }
505
506 /// Like `deinit`, but does not check for memory leaks. This is useful if leaks have already
507 /// been detected manually with `detectLeaks` to avoid reporting them for a second time.
508 pub fn deinitWithoutLeakChecks(self: *Self) void {
502 if (config.retain_metadata) self.freeRetainedMetadata();509 if (config.retain_metadata) self.freeRetainedMetadata();
503 self.large_allocations.deinit(self.backing_allocator);510 self.large_allocations.deinit(self.backing_allocator);
504 self.* = undefined;511 self.* = undefined;
505 return if (leaks) .leak else .ok;
506 }512 }
507513
508 fn collectStackTrace(first_trace_addr: usize, addr_buf: *[stack_n]usize) void {514 fn collectStackTrace(first_trace_addr: usize, addr_buf: *[stack_n]usize) void {
lib/std/zig/Server.zig+7-6
...@@ -96,15 +96,16 @@ pub const Message = struct {...@@ -96,15 +96,16 @@ pub const Message = struct {
9696
97 pub const TestResults = extern struct {97 pub const TestResults = extern struct {
98 index: u32,98 index: u32,
99 flags: Flags,99 flags: Flags align(4),
100100
101 pub const Flags = packed struct(u32) {101 pub const Flags = packed struct(u64) {
102 fail: bool,102 status: Status,
103 skip: bool,
104 leak: bool,
105 fuzz: bool,103 fuzz: bool,
106 log_err_count: u28 = 0,104 log_err_count: u30,
105 leak_count: u31,
107 };106 };
107
108 pub const Status = enum(u2) { pass, fail, skip };
108 };109 };
109110
110 /// Trailing is the same as in `std.Build.abi.time_report.CompileResult`, excluding `step_name`.111 /// Trailing is the same as in `std.Build.abi.time_report.CompileResult`, excluding `step_name`.