authorgravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2026-03-11 21:19:22-04:00
committergravatar for goon.pri.low@gmail.comKendall Condon <goon.pri.low@gmail.com> 2026-03-11 21:19:22-04:00
log5ecef2934af9349507061b2ff0498a089e7f5de0
tree7c0e592ec2aa1071da8c180be636f6b2757b4b31
parentbca057afd28bdcc38b123c05915ca7d775a2a17d

rerun fuzz tests from name instead of index

The indexes can change between recompilation due to conditional compilation and compiler quirks. While unit test names are still not a perfect solution, they are better than indexes.

4 files changed, 44 insertions(+), 24 deletions(-)

lib/compiler/test_runner.zig+17-1
...@@ -180,7 +180,23 @@ fn mainServer(init: std.process.Init.Minimal) !void {...@@ -180,7 +180,23 @@ fn mainServer(init: std.process.Init.Minimal) !void {
180 // since they are not present.180 // since they are not present.
181 if (!builtin.fuzz) unreachable;181 if (!builtin.fuzz) unreachable;
182182
183 const index = try server.receiveBody_u32();183 const index: u32 = @intCast(index: {
184 testing.allocator_instance = .{};
185 defer if (testing.allocator_instance.deinit() == .leak) {
186 @panic("internal test runner memory leak");
187 };
188
189 const name_len = try server.receiveBody_u32();
190 const name = try server.in.readAlloc(testing.allocator, @intCast(name_len));
191 defer testing.allocator.free(name);
192 for (0.., builtin.test_functions) |i, test_fn| {
193 if (std.mem.eql(u8, name, test_fn.name)) {
194 break :index i;
195 }
196 } else {
197 std.debug.panic("fuzz test {s} no longer exists", .{name});
198 }
199 });
184 const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());200 const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
185 const amount_or_instance = try server.receiveBody_u64();201 const amount_or_instance = try server.receiveBody_u64();
186202
lib/std/Build/Fuzz.zig+7-8
...@@ -145,9 +145,9 @@ pub fn start(fuzz: *Fuzz) void {...@@ -145,9 +145,9 @@ pub fn start(fuzz: *Fuzz) void {
145 }145 }
146146
147 for (fuzz.run_steps) |run| {147 for (fuzz.run_steps) |run| {
148 for (run.fuzz_tests.items) |unit_test_index| {148 for (run.fuzz_tests.items) |unit_test_name| {
149 assert(run.rebuilt_executable != null);149 assert(run.rebuilt_executable != null);
150 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });150 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_name });
151 }151 }
152 }152 }
153}153}
...@@ -193,17 +193,16 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod...@@ -193,17 +193,16 @@ fn rebuildTestsWorkerRunFallible(run: *Step.Run, gpa: Allocator, parent_prog_nod
193 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);193 run.rebuilt_executable = try rebuilt_bin_path.join(gpa, compile.out_filename);
194}194}
195195
196fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {196fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_name: []const u8) void {
197 const owner = run.step.owner;197 const owner = run.step.owner;
198 const gpa = owner.allocator;198 const gpa = owner.allocator;
199 const graph = owner.graph;199 const graph = owner.graph;
200 const io = graph.io;200 const io = graph.io;
201 const test_name = run.cached_test_metadata.?.testName(unit_test_index);
202201
203 const prog_node = fuzz.prog_node.start(test_name, 0);202 const prog_node = fuzz.prog_node.start(unit_test_name, 0);
204 defer prog_node.end();203 defer prog_node.end();
205204
206 run.rerunInFuzzMode(fuzz, unit_test_index, prog_node) catch |err| switch (err) {205 run.rerunInFuzzMode(fuzz, unit_test_name, prog_node) catch |err| switch (err) {
207 error.MakeFailed => {206 error.MakeFailed => {
208 var buf: [256]u8 = undefined;207 var buf: [256]u8 = undefined;
209 const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) {208 const stderr = io.lockStderr(&buf, graph.stderr_mode) catch |e| switch (e) {
...@@ -214,7 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {...@@ -214,7 +213,7 @@ fn fuzzWorkerRun(fuzz: *Fuzz, run: *Step.Run, unit_test_index: u32) void {
214 return;213 return;
215 },214 },
216 else => {215 else => {
217 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, test_name, err });216 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, unit_test_name, err });
218 return;217 return;
219 },218 },
220 };219 };
...@@ -588,7 +587,7 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {...@@ -588,7 +587,7 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) Io.Cancelable!void {
588 \\587 \\
589 , .{588 , .{
590 cov.run.step.name,589 cov.run.step.name,
591 cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),590 cov.run.fuzz_tests.items[0],
592 cov.id,591 cov.id,
593 cov.cumulative.runs,592 cov.cumulative.runs,
594 header.n_runs,593 header.n_runs,
lib/std/Build/Step/Run.zig+18-14
...@@ -88,9 +88,10 @@ dep_output_file: ?*Output,...@@ -88,9 +88,10 @@ dep_output_file: ?*Output,
8888
89has_side_effects: bool,89has_side_effects: bool,
9090
91/// If this is a Zig unit test binary, this tracks the indexes of the unit91/// If this is a Zig unit test binary, this tracks the names of the unit
92/// tests that are also fuzz tests.92/// tests that are also fuzz tests. Indexes cannot be used as they may
93fuzz_tests: std.ArrayList(u32),93/// change between reruns.
94fuzz_tests: std.ArrayList([]const u8),
94cached_test_metadata: ?CachedTestMetadata = null,95cached_test_metadata: ?CachedTestMetadata = null,
9596
96/// Populated during the fuzz phase if this run step corresponds to a unit test97/// Populated during the fuzz phase if this run step corresponds to a unit test
...@@ -1067,7 +1068,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {...@@ -1067,7 +1068,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void {
1067pub fn rerunInFuzzMode(1068pub fn rerunInFuzzMode(
1068 run: *Run,1069 run: *Run,
1069 fuzz: *std.Build.Fuzz,1070 fuzz: *std.Build.Fuzz,
1070 unit_test_index: u32,1071 unit_test_name: []const u8,
1071 prog_node: std.Progress.Node,1072 prog_node: std.Progress.Node,
1072) !void {1073) !void {
1073 const step = &run.step;1074 const step = &run.step;
...@@ -1138,7 +1139,7 @@ pub fn rerunInFuzzMode(...@@ -1138,7 +1139,7 @@ pub fn rerunInFuzzMode(
1138 .unit_test_timeout_ns = null, // don't time out fuzz tests for now1139 .unit_test_timeout_ns = null, // don't time out fuzz tests for now
1139 .gpa = fuzz.gpa,1140 .gpa = fuzz.gpa,
1140 }, .{1141 }, .{
1141 .unit_test_index = unit_test_index,1142 .unit_test_name = unit_test_name,
1142 .fuzz = fuzz,1143 .fuzz = fuzz,
1143 });1144 });
1144}1145}
...@@ -1210,7 +1211,7 @@ fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool {...@@ -1210,7 +1211,7 @@ fn termMatches(expected: ?process.Child.Term, actual: process.Child.Term) bool {
12101211
1211const FuzzContext = struct {1212const FuzzContext = struct {
1212 fuzz: *std.Build.Fuzz,1213 fuzz: *std.Build.Fuzz,
1213 unit_test_index: u32,1214 unit_test_name: []const u8,
1214};1215};
12151216
1216fn runCommand(1217fn runCommand(
...@@ -1843,7 +1844,7 @@ fn waitZigTest(...@@ -1843,7 +1844,7 @@ fn waitZigTest(
1843 sendRunFuzzTestMessage(1844 sendRunFuzzTestMessage(
1844 io,1845 io,
1845 child.stdin.?,1846 child.stdin.?,
1846 ctx.unit_test_index,1847 ctx.unit_test_name,
1847 .forever,1848 .forever,
1848 0, // instance ID; will be used by multiprocess forever fuzzing in the future1849 0, // instance ID; will be used by multiprocess forever fuzzing in the future
1849 ) catch |err| return .{ .write_failed = err };1850 ) catch |err| return .{ .write_failed = err };
...@@ -1852,7 +1853,7 @@ fn waitZigTest(...@@ -1852,7 +1853,7 @@ fn waitZigTest(
1852 sendRunFuzzTestMessage(1853 sendRunFuzzTestMessage(
1853 io,1854 io,
1854 child.stdin.?,1855 child.stdin.?,
1855 ctx.unit_test_index,1856 ctx.unit_test_name,
1856 .iterations,1857 .iterations,
1857 limit.amount,1858 limit.amount,
1858 ) catch |err| return .{ .write_failed = err };1859 ) catch |err| return .{ .write_failed = err };
...@@ -2001,10 +2002,10 @@ fn waitZigTest(...@@ -2001,10 +2002,10 @@ fn waitZigTest(
2001 results.leak_count +|= leak_count;2002 results.leak_count +|= leak_count;
2002 results.log_err_count +|= log_err_count;2003 results.log_err_count +|= log_err_count;
20032004
2004 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index);2005 if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, md.testName(tr_hdr.index));
20052006
2006 if (tr_hdr.flags.status == .fail) {2007 if (tr_hdr.flags.status == .fail) {
2007 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);2008 const name = md.testName(tr_hdr.index);
2008 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");2009 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
2009 stderr.tossBuffered();2010 stderr.tossBuffered();
2010 if (stderr_bytes.len == 0) {2011 if (stderr_bytes.len == 0) {
...@@ -2013,12 +2014,12 @@ fn waitZigTest(...@@ -2013,12 +2014,12 @@ fn waitZigTest(
2013 try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });2014 try run.step.addError("'{s}' failed:\n{s}", .{ name, stderr_bytes });
2014 }2015 }
2015 } else if (leak_count > 0) {2016 } else if (leak_count > 0) {
2016 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);2017 const name = md.testName(tr_hdr.index);
2017 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");2018 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
2018 stderr.tossBuffered();2019 stderr.tossBuffered();
2019 try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });2020 try run.step.addError("'{s}' leaked {d} allocations:\n{s}", .{ name, leak_count, stderr_bytes });
2020 } else if (log_err_count > 0) {2021 } else if (log_err_count > 0) {
2021 const name = std.mem.sliceTo(md.testName(tr_hdr.index), 0);2022 const name = md.testName(tr_hdr.index);
2022 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");2023 const stderr_bytes = std.mem.trim(u8, stderr.buffered(), "\n");
2023 stderr.tossBuffered();2024 stderr.tossBuffered();
2024 try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });2025 try run.step.addError("'{s}' logged {d} errors:\n{s}", .{ name, log_err_count, stderr_bytes });
...@@ -2148,7 +2149,7 @@ fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, in...@@ -2148,7 +2149,7 @@ fn sendRunTestMessage(io: Io, file: Io.File, tag: std.zig.Client.Message.Tag, in
2148fn sendRunFuzzTestMessage(2149fn sendRunFuzzTestMessage(
2149 io: Io,2150 io: Io,
2150 file: Io.File,2151 file: Io.File,
2151 index: u32,2152 test_name: []const u8,
2152 kind: std.Build.abi.fuzz.LimitKind,2153 kind: std.Build.abi.fuzz.LimitKind,
2153 amount_or_instance: u64,2154 amount_or_instance: u64,
2154) !void {2155) !void {
...@@ -2160,7 +2161,10 @@ fn sendRunFuzzTestMessage(...@@ -2160,7 +2161,10 @@ fn sendRunFuzzTestMessage(
2160 w.interface.writeStruct(header, .little) catch |err| switch (err) {2161 w.interface.writeStruct(header, .little) catch |err| switch (err) {
2161 error.WriteFailed => return w.err.?,2162 error.WriteFailed => return w.err.?,
2162 };2163 };
2163 w.interface.writeInt(u32, index, .little) catch |err| switch (err) {2164 w.interface.writeInt(u32, @intCast(test_name.len), .little) catch |err| switch (err) {
2165 error.WriteFailed => return w.err.?,
2166 };
2167 w.interface.writeAll(test_name) catch |err| switch (err) {
2164 error.WriteFailed => return w.err.?,2168 error.WriteFailed => return w.err.?,
2165 };2169 };
2166 w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) {2170 w.interface.writeByte(@intFromEnum(kind)) catch |err| switch (err) {
lib/std/zig/Client.zig+2-1
...@@ -35,7 +35,8 @@ pub const Message = struct {...@@ -35,7 +35,8 @@ pub const Message = struct {
35 run_test,35 run_test,
36 /// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.36 /// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
37 /// The message body is:37 /// The message body is:
38 /// - a u32 test index.38 /// - a u32 test name len.
39 /// - a test name with the above length
39 /// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)40 /// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
40 /// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)41 /// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
41 start_fuzzing,42 start_fuzzing,