authorgravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2026-03-11 21:19:22-04:00
committergravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2026-03-11 21:19:22-04:00
log5ecef2934af9349507061b2ff0498a089e7f5de0
tree7c0e592ec2aa1071da8c180be636f6b2757b4b31
parentbca057afd28bdcc38b123c05915ca7d775a2a17d

rerun fuzz tests from name instead of index

The indexes can change between recompilation due to conditional compilation and compiler quirks. While unit test names are still not a perfect solution, they are better than indexes.

4 files changed, 44 insertions(+), 24 deletions(-)

lib/compiler/test_runner.zig+17-1
......@@ -180,7 +180,23 @@ fn mainServer(init: std.process.Init.Minimal) !void {
180180 // since they are not present.
181181 if (!builtin.fuzz) unreachable;
182182
183 const index = try server.receiveBody_u32();
183 const index: u32 = @intCast(index: {
184 testing.allocator_instance = .{};
185 defer if (testing.allocator_instance.deinit() == .leak) {
186 @panic("internal test runner memory leak");
187 };
188
189 const name_len = try server.receiveBody_u32();
190 const name = try server.in.readAlloc(testing.allocator, @intCast(name_len));
191 defer testing.allocator.free(name);
192 for (0.., builtin.test_functions) |i, test_fn| {
193 if (std.mem.eql(u8, name, test_fn.name)) {
194 break :index i;
195 }
196 } else {
197 std.debug.panic("fuzz test {s} no longer exists", .{name});
198 }
199 });
184200 const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
185201 const amount_or_instance = try server.receiveBody_u64();
186202
lib/std/Build/Fuzz.zig+7-8
......@@ -145,9 +145,9 @@ pub fn start(fuzz: *Fuzz) void {
145145 }
146146
147147 for (fuzz.run_steps) |run| {
148 for (run.fuzz_tests.items) |unit_test_index| {
148 for (run.fuzz_tests.items) |unit_test_name| {
149149 assert(run.rebuilt_executable != null);
150 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });
150 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_name });
151151 }
152152 }
153153}
......@@ -193,17 +193,16 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod
193193 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
194194}
195195
196fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
196fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_name: []const u8) void {
197197 const owner = run.step.owner;
198198 const gpa = owner.allocator;
199199 const graph = owner.graph;
200200 const io = graph.io;
201 const test_name = run.cached_test_metadata.?.testName(unit_test_index);
202201
203 const prog_node = fuzz.prog_node.start(test_name, 0);
202 const prog_node = fuzz.prog_node.start(unit_test_name, 0);
204203 defer prog_node.end();
205204
206 run.rerunInFuzzMode(fuzz, unit_test_index, prog_node) catch |err| switch (err) {
205 run.rerunInFuzzMode(fuzz, unit_test_name, prog_node) catch |err| switch (err) {
207206 error.MakeFailed => {
208207 var buf: [256]u8 = undefined;
209208 const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) {
......@@ -214,7 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
214213 return;
215214 },
216215 else => {
217 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, test_name, err });
216 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, unit_test_name, err });
218217 return;
219218 },
220219 };
......@@ -588,7 +587,7 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
588587 \\
589588 , .{
590589 cov.run.step.name,
591 cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
590 cov.run.fuzz_tests.items[0],
592591 cov.id,
593592 cov.cumulative.runs,
594593 header.n_runs,
lib/std/Build/Step/Run.zig+18-14
......@@ -88,9 +88,10 @@ dep_output_file: ?*Output,
8888
8989has_side_effects: bool,
9090
91/// If this is a Zig unit test binary, this tracks the indexes of the unit
92/// tests that are also fuzz tests.
93fuzz_tests: std.ArrayList(u32),
91/// If this is a Zig unit test binary, this tracks the names of the unit
92/// tests that are also fuzz tests. Indexes cannot be used as they may
93/// change between reruns.
94fuzz_tests: std.ArrayList([]const u8),
9495cached_test_metadata: ?CachedTestMetadata = null,
9596
9697/// Populated during the fuzz phase if this run step corresponds to a unit test
......@@ -1067,7 +1068,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
10671068pub fn rerunInFuzzMode(
10681069 run: *Run,
10691070 fuzz: *std.Build.Fuzz,
1070 unit_test_index: u32,
1071 unit_test_name: []const u8,
10711072 prog_node: std.Progress.Node,
10721073) !void {
10731074 const step = &run.step;
......@@ -1138,7 +1139,7 @@ pub fn rerunInFuzzMode(
11381139 .unit_test_timeout_ns = null, // don't time out fuzz tests for now
11391140 .gpa = fuzz.gpa,
11401141 }, .{
1141 .unit_test_index = unit_test_index,
1142 .unit_test_name = unit_test_name,
11421143 .fuzz = fuzz,
11431144 });
11441145}
......@@ -1210,7 +1211,7 @@ fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool {
12101211
12111212const FuzzContext = struct {
12121213 fuzz: *std.Build.Fuzz,
1213 unit_test_index: u32,
1214 unit_test_name: []const u8,
12141215};
12151216
12161217fn runCommand(
......@@ -1843,7 +1844,7 @@ fn waitZigTest(
18431844 sendRunFuzzTestMessage(
18441845 io,
18451846 child.stdin.?,
1846 ctx.unit_test_index,
1847 ctx.unit_test_name,
18471848 .forever,
18481849 0, // instance ID; will be used by multiprocess forever fuzzing in the future
18491850 ) catch |err| return .{ .write_failed = err };
......@@ -1852,7 +1853,7 @@ fn waitZigTest(
18521853 sendRunFuzzTestMessage(
18531854 io,
18541855 child.stdin.?,
1855 ctx.unit_test_index,
1856 ctx.unit_test_name,
18561857 .iterations,
18571858 limit.amount,
18581859 ) catch |err| return .{ .write_failed = err };
......@@ -2001,10 +2002,10 @@ fn waitZigTest(
20012002 results.leak_count +|= leak_count;
20022003 results.log_err_count +|= log_err_count;
20032004
2004 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);
2005 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index));
20052006
20062007 if (tr_hdr.flags.status == .fail) {
2007 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
2008 const name = md.testName(tr_hdr.index);
20082009 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
20092010 stderr.tossBuffered();
20102011 if (stderr_bytes.len == 0) {
......@@ -2013,12 +2014,12 @@ fn waitZigTest(
20132014 try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });
20142015 }
20152016 } else if (leak_count > 0) {
2016 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
2017 const name = md.testName(tr_hdr.index);
20172018 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
20182019 stderr.tossBuffered();
20192020 try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });
20202021 } else if (log_err_count > 0) {
2021 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);
2022 const name = md.testName(tr_hdr.index);
20222023 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
20232024 stderr.tossBuffered();
20242025 try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });
......@@ -2148,7 +2149,7 @@ fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, in
21482149fn sendRunFuzzTestMessage(
21492150 io: Io,
21502151 file: Io.File,
2151 index: u32,
2152 test_name: []const u8,
21522153 kind: std.Build.abi.fuzz.LimitKind,
21532154 amount_or_instance: u64,
21542155) !void {
......@@ -2160,7 +2161,10 @@ fn sendRunFuzzTestMessage(
21602161 w.interface.writeStruct(header, .little) catch |err| switch (err) {
21612162 error.WriteFailed => return w.err.?,
21622163 };
2163 w.interface.writeInt(u32, index, .little) catch |err| switch (err) {
2164 w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) {
2165 error.WriteFailed => return w.err.?,
2166 };
2167 w.interface.writeAll(test_name) catch |err| switch (err) {
21642168 error.WriteFailed => return w.err.?,
21652169 };
21662170 w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) {
lib/std/zig/Client.zig+2-1
......@@ -35,7 +35,8 @@ pub const Message = struct {
3535 run_test,
3636 /// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
3737 /// The message body is:
38 /// - a u32 test index.
38 /// - a u32 test name len.
39 /// - a test name with the above length
3940 /// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
4041 /// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
4142 start_fuzzing,