1//! Default test runner for unit tests.
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Io = std.Io;
6const fatal = std.process.fatal;
7const testing = std.testing;
8const assert = std.debug.assert;
9const panic = std.debug.panic;
10const fuzz_abi = std.Build.abi.fuzz;
11
12pub const std_options: std.Options = .{
13 .logFn = log,
14};
15
16var log_err_count: usize = 0;
17var fba: std.heap.FixedBufferAllocator = .init(&fba_buffer);
18var fba_buffer: [8192]u8 = undefined;
19var stdin_buffer: [4096]u8 = undefined;
20var stdout_buffer: [4096]u8 = undefined;
21var stdin_reader: Io.File.Reader = undefined;
22var stdout_writer: Io.File.Writer = undefined;
23const runner_threaded_io: Io = Io.Threaded.global_single_threaded.io();
24
25/// Keep in sync with logic in `std.Build.addRunArtifact` which decides whether
26/// the test runner will communicate with the build runner via `std.zig.Server`.
27const need_simple = switch (builtin.zig_backend) {
28 .stage2_aarch64,
29 .stage2_loongarch,
30 .stage2_powerpc,
31 .stage2_riscv64,
32 => true,
33 else => false,
34};
35
36pub fn main(init: std.process.Init.Minimal) void {
37 @disableInstrumentation();
38
39 if (builtin.cpu.arch.isSpirV()) {
40 // SPIR-V needs an special test-runner
41 return;
42 }
43
44 if (need_simple) {
45 return mainSimple() catch |err| panic("test failure: {t}", .{err});
46 }
47
48 const args = init.args.toSlice(fba.allocator()) catch |err| panic("unable to parse command line args: {t}", .{err});
49
50 var listen = false;
51 var opt_cache_dir: ?[]const u8 = null;
52
53 for (args[1..]) |arg| {
54 if (std.mem.eql(u8, arg, "--listen=-")) {
55 listen = true;
56 } else if (std.mem.startsWith(u8, arg, "--seed=")) {
57 testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch
58 @panic("unable to parse --seed command line argument");
59 } else if (std.mem.startsWith(u8, arg, "--cache-dir")) {
60 opt_cache_dir = arg["--cache-dir=".len..];
61 } else {
62 panic("unrecognized command line argument: {s}", .{arg});
63 }
64 }
65
66 if (builtin.fuzz) {
67 const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
68 fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
69 }
70
71 if (listen) {
72 return mainServer(init) catch |err| panic("internal test runner failure: {t}", .{err});
73 } else {
74 return mainTerminal(init);
75 }
76}
77
78fn mainServer(init: std.process.Init.Minimal) !void {
79 @disableInstrumentation();
80 stdin_reader = .initStreaming(.stdin(), runner_threaded_io, &stdin_buffer);
81 stdout_writer = .initStreaming(.stdout(), runner_threaded_io, &stdout_buffer);
82 var server: std.zig.Server = .{
83 .in = &stdin_reader.interface,
84 .out = &stdout_writer.interface,
85 };
86 try server.serveStringMessage(.zig_version, builtin.zig_version_string);
87
88 while (true) {
89 const hdr = try server.receiveMessage();
90 switch (hdr.tag) {
91 .exit => {
92 return std.process.exit(0);
93 },
94 .query_test_metadata => {
95 var sa: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{});
96 defer if (sa.deinit() != 0) @panic("internal test runner memory leak");
97 const gpa = sa.allocator();
98
99 var string_bytes: std.ArrayList(u8) = .empty;
100 defer string_bytes.deinit(gpa);
101 try string_bytes.append(gpa, 0); // Reserve 0 for null.
102
103 const test_fns = builtin.test_functions;
104 const names = try gpa.alloc(u32, test_fns.len);
105 defer gpa.free(names);
106 const expected_panic_msgs = try gpa.alloc(u32, test_fns.len);
107 defer gpa.free(expected_panic_msgs);
108
109 for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| {
110 name.* = @intCast(string_bytes.items.len);
111 try string_bytes.ensureUnusedCapacity(gpa, test_fn.name.len + 1);
112 string_bytes.appendSliceAssumeCapacity(test_fn.name);
113 string_bytes.appendAssumeCapacity(0);
114 expected_panic_msg.* = 0;
115 }
116
117 try server.serveTestMetadata(.{
118 .names = names,
119 .expected_panic_msgs = expected_panic_msgs,
120 .string_bytes = string_bytes.items,
121 });
122 },
123
124 .run_test => {
125 testing.environ = init.environ;
126 testing.allocator_instance = .init(std.heap.page_allocator, .{
127 .canary = 0xc3a701ba,
128 .check_write_after_free = true,
129 });
130 testing.io_instance = .init(testing.allocator, .{
131 .argv0 = .init(init.args),
132 .environ = init.environ,
133 });
134 log_err_count = 0;
135 const index = try server.receiveBody_u32();
136 const test_fn = builtin.test_functions[index];
137 is_fuzz_test = false;
138
139 // let the build server know we're starting the test now
140 try server.serveStringMessage(.test_started, &.{});
141
142 const TestResults = std.zig.Server.Message.TestResults;
143 const status: TestResults.Status = if (test_fn.func()) |v| s: {
144 v;
145 break :s .pass;
146 } else |err| switch (err) {
147 error.SkipZigTest => .skip,
148 else => s: {
149 if (@errorReturnTrace()) |trace| {
150 std.debug.dumpErrorReturnTrace(trace);
151 }
152 break :s .fail;
153 },
154 };
155 testing.io_instance.deinit();
156 const leak_count = testing.allocator_instance.deinit();
157 try server.serveTestResults(.{
158 .index = index,
159 .flags = .{
160 .status = status,
161 .fuzz = is_fuzz_test,
162 .log_err_count = std.math.lossyCast(
163 @FieldType(TestResults.Flags, "log_err_count"),
164 log_err_count,
165 ),
166 .leak_count = std.math.lossyCast(
167 @FieldType(TestResults.Flags, "leak_count"),
168 leak_count,
169 ),
170 },
171 });
172 },
173 .start_fuzzing => {
174 // This ensures that this code won't be analyzed and hence reference fuzzer symbols
175 // since they are not present.
176 if (!builtin.fuzz) unreachable;
177
178 var gpa_instance: std.heap.SafeAllocator = .init(std.heap.page_allocator, .{});
179 defer if (gpa_instance.deinit() != 0) {
180 @panic("internal test runner memory leak");
181 };
182 const gpa = gpa_instance.allocator();
183 var io_instance: Io.Threaded = .init(gpa, .{
184 .argv0 = .init(init.args),
185 .environ = init.environ,
186 });
187 defer io_instance.deinit();
188
189 const mode: fuzz_abi.LimitKind = @fromBackingInt(@intCast(try server.receiveBody_u8()));
190 const amount_or_instance = try server.receiveBody_u64();
191 const main_instance = mode == .iterations or amount_or_instance == 0;
192
193 if (main_instance) {
194 const coverage = fuzz_abi.fuzzer_coverage();
195 try server.serveCoverageIdMessage(
196 coverage.id,
197 coverage.runs,
198 coverage.unique,
199 coverage.seen,
200 );
201 }
202
203 const n_tests: u32 = try server.receiveBody_u32();
204 const test_indexes = try gpa.alloc(u32, n_tests);
205 defer gpa.free(test_indexes);
206 fuzz_runner = .{
207 .indexes = test_indexes,
208 .server = &server,
209 .gpa = gpa,
210 .threaded_io = &io_instance,
211 .input_poller = undefined,
212 };
213
214 {
215 var large_name_buf: std.ArrayList(u8) = .empty;
216 defer large_name_buf.deinit(gpa);
217 for (test_indexes) |*i| {
218 const name_len = try server.receiveBody_u32();
219 const name = if (name_len <= server.in.buffer.len)
220 try server.in.take(name_len)
221 else large_name: {
222 try large_name_buf.resize(gpa, name_len);
223 try server.in.readSliceAll(large_name_buf.items);
224 break :large_name large_name_buf.items;
225 };
226
227 for (0.., builtin.test_functions) |test_i, test_fn| {
228 if (std.mem.eql(u8, name, test_fn.name)) {
229 i.* = @intCast(test_i);
230 break;
231 }
232 } else {
233 panic("fuzz test {s} no longer exists", .{name});
234 }
235
236 if (main_instance) {
237 const relocated_entry_addr = @intFromPtr(builtin.test_functions[i.*].func);
238 const entry_addr = fuzz_abi.fuzzer_unslide_address(relocated_entry_addr);
239 try server.serveU64Message(.fuzz_start_addr, entry_addr);
240 }
241 }
242 }
243
244 fuzz_abi.fuzzer_main(n_tests, testing.random_seed, mode, amount_or_instance);
245
246 assert(mode != .forever);
247 std.process.exit(0);
248 },
249
250 else => {
251 std.debug.print("unsupported message: {x}\n", .{@backingInt(hdr.tag)});
252 std.process.exit(1);
253 },
254 }
255 }
256}
257
258fn mainTerminal(init: std.process.Init.Minimal) void {
259 @disableInstrumentation();
260 if (builtin.fuzz) @panic("fuzz test requires server");
261
262 const test_fn_list = builtin.test_functions;
263 var ok_count: usize = 0;
264 var skip_count: usize = 0;
265 var fail_count: usize = 0;
266 var fuzz_count: usize = 0;
267 const root_node = if (builtin.fuzz) std.Progress.Node.none else std.Progress.start(runner_threaded_io, .{
268 .root_name = "Test",
269 .estimated_total_items = test_fn_list.len,
270 });
271 const have_tty = Io.File.stderr().isTty(runner_threaded_io) catch unreachable;
272
273 var leaks: usize = 0;
274 for (test_fn_list, 0..) |test_fn, i| {
275 testing.allocator_instance = .init(std.heap.page_allocator, .{
276 .canary = 0xc3a701ba,
277 .check_write_after_free = true,
278 });
279 testing.io_instance = .init(testing.allocator, .{
280 .argv0 = .init(init.args),
281 .environ = init.environ,
282 });
283 defer {
284 testing.io_instance.deinit();
285 if (testing.allocator_instance.deinit() != 0) leaks += 1;
286 }
287 testing.log_level = .warn;
288 testing.environ = init.environ;
289
290 const test_node = root_node.start(test_fn.name, 0);
291 if (!have_tty) {
292 std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name });
293 }
294 is_fuzz_test = false;
295 if (test_fn.func()) |_| {
296 ok_count += 1;
297 test_node.end();
298 if (!have_tty) std.debug.print("OK\n", .{});
299 } else |err| switch (err) {
300 error.SkipZigTest => {
301 skip_count += 1;
302 if (have_tty) {
303 std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name });
304 } else {
305 std.debug.print("SKIP\n", .{});
306 }
307 test_node.end();
308 },
309 else => {
310 fail_count += 1;
311 if (have_tty) {
312 std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
313 i + 1, test_fn_list.len, test_fn.name, err,
314 });
315 } else {
316 std.debug.print("FAIL ({t})\n", .{err});
317 }
318 if (@errorReturnTrace()) |trace| {
319 std.debug.dumpErrorReturnTrace(trace);
320 }
321 test_node.end();
322 },
323 }
324 fuzz_count += @intFromBool(is_fuzz_test);
325 }
326 root_node.end();
327 if (ok_count == test_fn_list.len) {
328 std.debug.print("All {d} tests passed.\n", .{ok_count});
329 } else {
330 std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count });
331 }
332 if (log_err_count != 0) {
333 std.debug.print("{d} errors were logged.\n", .{log_err_count});
334 }
335 if (leaks != 0) {
336 std.debug.print("{d} tests leaked memory.\n", .{leaks});
337 }
338 if (fuzz_count != 0) {
339 std.debug.print("{d} fuzz tests found.\n", .{fuzz_count});
340 }
341 if (leaks != 0 or log_err_count != 0 or fail_count != 0) {
342 std.process.exit(1);
343 }
344}
345
346pub fn log(
347 comptime message_level: std.log.Level,
348 comptime scope: @EnumLiteral(),
349 comptime format: []const u8,
350 args: anytype,
351) void {
352 @disableInstrumentation();
353 if (@backingInt(message_level) <= @backingInt(std.log.Level.err)) {
354 log_err_count +|= 1;
355 }
356 if (@backingInt(message_level) <= @backingInt(testing.log_level)) {
357 std.debug.print(
358 "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n",
359 args,
360 );
361 }
362}
363
364/// Simpler main(), exercising fewer language features, so that
365/// work-in-progress backends can handle it.
366pub fn mainSimple() anyerror!void {
367 @disableInstrumentation();
368 // is the backend capable of calling `Io.File.writeAll`?
369 const enable_write = switch (builtin.zig_backend) {
370 .stage2_aarch64, .stage2_riscv64 => true,
371 else => false,
372 };
373 // is the backend capable of calling `Io.Writer.print`?
374 const enable_print = switch (builtin.zig_backend) {
375 .stage2_aarch64, .stage2_riscv64 => true,
376 else => false,
377 };
378
379 testing.allocator_instance = .init(std.heap.page_allocator, .{});
380 testing.io_instance = .init(testing.allocator, .{});
381
382 var passed: u64 = 0;
383 var skipped: u64 = 0;
384 var failed: u64 = 0;
385
386 // we don't want to bring in File and Writer if the backend doesn't support it
387 const stdout = if (enable_write) Io.File.stdout() else {};
388
389 for (builtin.test_functions) |test_fn| {
390 if (enable_write) {
391 stdout.writeStreamingAll(runner_threaded_io, test_fn.name) catch {};
392 stdout.writeStreamingAll(runner_threaded_io, "... ") catch {};
393 }
394 if (test_fn.func()) |_| {
395 if (enable_write) stdout.writeStreamingAll(runner_threaded_io, "PASS\n") catch {};
396 } else |err| {
397 if (err != error.SkipZigTest) {
398 if (enable_write) stdout.writeStreamingAll(runner_threaded_io, "FAIL\n") catch {};
399 failed += 1;
400 if (!enable_write) return err;
401 continue;
402 }
403 if (enable_write) stdout.writeStreamingAll(runner_threaded_io, "SKIP\n") catch {};
404 skipped += 1;
405 continue;
406 }
407 passed += 1;
408 }
409 if (enable_print) {
410 var unbuffered_stdout_writer = stdout.writer(runner_threaded_io, &.{});
411 unbuffered_stdout_writer.interface.print(
412 "{} passed, {} skipped, {} failed\n",
413 .{ passed, skipped, failed },
414 ) catch {};
415 }
416 if (failed != 0) std.process.exit(1);
417}
418
419var is_fuzz_test: bool = undefined;
420var fuzz_runner: if (builtin.fuzz) struct {
421 indexes: []u32,
422 server: *std.zig.Server,
423 gpa: std.mem.Allocator,
424 threaded_io: *Io.Threaded,
425 input_poller: Io.Future(Io.Cancelable!void),
426
427 comptime {
428 assert(builtin.fuzz); // `fuzz_runner` was analyzed in non-fuzzing compilation
429 }
430
431 export fn runner_test_run(i: u32) void {
432 @disableInstrumentation();
433
434 fuzz_runner.server.serveU32Message(.fuzz_test_change, i) catch |e| switch (e) {
435 error.WriteFailed => panic("failed to write to stdout: {t}", .{stdout_writer.err.?}),
436 };
437
438 testing.allocator_instance = .init(std.heap.page_allocator, .{
439 .canary = 0xc3a701ba,
440 .check_write_after_free = true,
441 });
442 defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1);
443 is_fuzz_test = false;
444
445 testing.io_instance = .init(testing.allocator, .{
446 .argv0 = fuzz_runner.threaded_io.argv0,
447 .environ = fuzz_runner.threaded_io.environ.process_environ,
448 });
449 defer testing.io_instance.deinit();
450
451 builtin.test_functions[fuzz_runner.indexes[i]].func() catch |err| switch (err) {
452 error.SkipZigTest => return,
453 else => {
454 if (@errorReturnTrace()) |trace| {
455 std.debug.dumpErrorReturnTrace(trace);
456 }
457 std.debug.print("failed with error.{t}\n", .{err});
458 std.process.exit(1);
459 },
460 };
461
462 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
463 if (log_err_count != 0) @panic("error logs detected");
464 }
465
466 export fn runner_test_name(i: u32) fuzz_abi.Slice {
467 @disableInstrumentation();
468 return .fromSlice(builtin.test_functions[fuzz_runner.indexes[i]].name);
469 }
470
471 export fn runner_broadcast_input(test_i: u32, bytes_slice: fuzz_abi.Slice) void {
472 @disableInstrumentation();
473 const bytes = bytes_slice.toSlice();
474 fuzz_runner.server.serveBroadcastFuzzInputMessage(test_i, bytes) catch |e| switch (e) {
475 error.WriteFailed => panic("failed to write to stdout: {t}", .{stdout_writer.err.?}),
476 };
477 }
478
479 export fn runner_start_input_poller() void {
480 @disableInstrumentation();
481 const io = fuzz_runner.threaded_io.io();
482 const future = io.concurrent(inputPoller, .{}) catch |e| switch (e) {
483 error.ConcurrencyUnavailable => @panic("failed to spawn concurrent fuzz input poller"),
484 };
485 fuzz_runner.input_poller = future;
486 }
487
488 export fn runner_stop_input_poller() void {
489 @disableInstrumentation();
490 const io = fuzz_runner.threaded_io.io();
491 assert(fuzz_runner.input_poller.cancel(io) == error.Canceled);
492 }
493
494 export fn runner_futex_wait(ptr: *const u32, expected: u32) bool {
495 @disableInstrumentation();
496 const io = fuzz_runner.threaded_io.io();
497 return io.futexWait(u32, ptr, expected) == error.Canceled;
498 }
499
500 export fn runner_futex_wake(ptr: *const u32, waiters: u32) void {
501 @disableInstrumentation();
502 const io = fuzz_runner.threaded_io.io();
503 io.futexWake(u32, ptr, waiters);
504 }
505
506 fn inputPoller() Io.Cancelable!void {
507 @disableInstrumentation();
508 switch (inputPollerInner()) {
509 error.Canceled => |e| return e,
510 error.ReadFailed => {
511 if (stdin_reader.err.? == error.Canceled) return error.Canceled;
512 panic("failed to read from stdin: {t}", .{stdin_reader.err.?});
513 },
514 error.EndOfStream => @panic("unexpected end of stdin"),
515 }
516 }
517
518 fn inputPollerInner() (Io.Cancelable || Io.Reader.Error) {
519 @disableInstrumentation();
520 const server = fuzz_runner.server;
521 var large_bytes_list: std.ArrayList(u8) = .empty;
522 defer large_bytes_list.deinit(fuzz_runner.gpa);
523 while (true) {
524 const hdr = try server.receiveMessage();
525 if (hdr.tag != .new_fuzz_input) {
526 panic("unexpected message: {x}\n", .{@backingInt(hdr.tag)});
527 }
528 const test_i = try server.receiveBody_u32();
529 const input_len = hdr.bytes_len - 4;
530 const bytes = if (input_len <= server.in.buffer.len)
531 try server.in.take(input_len)
532 else bytes: {
533 large_bytes_list.resize(fuzz_runner.gpa, @intCast(input_len)) catch @panic("OOM");
534 try server.in.readSliceAll(large_bytes_list.items);
535 break :bytes large_bytes_list.items;
536 };
537 if (fuzz_abi.fuzzer_receive_input(test_i, .fromSlice(bytes))) {
538 return error.Canceled;
539 }
540 }
541 }
542} else void = undefined;
543
544pub fn fuzz(
545 context: anytype,
546 comptime testOne: fn (context: @TypeOf(context), *std.testing.Smith) anyerror!void,
547 options: testing.FuzzInputOptions,
548) anyerror!void {
549 // Prevent this function from confusing the fuzzer by omitting its own code
550 // coverage from being considered.
551 @disableInstrumentation();
552
553 // Some compiler backends are not capable of handling fuzz testing yet but
554 // we still want CI test coverage enabled.
555 if (need_simple) return;
556
557 // Smoke test to ensure the test did not use conditional compilation to
558 // contradict itself by making it not actually be a fuzz test when the test
559 // is built in fuzz mode.
560 is_fuzz_test = true;
561
562 // Ensure no test failure occurred before starting fuzzing.
563 if (log_err_count != 0) @panic("error logs detected");
564
565 // libfuzzer is in a separate compilation unit so that its own code can be
566 // excluded from code coverage instrumentation. It needs a function pointer
567 // it can call for checking exactly one input. Inside this function we do
568 // our standard unit test checks such as memory leaks, and interaction with
569 // error logs.
570 const global = struct {
571 var ctx: @TypeOf(context) = undefined;
572
573 fn test_one() callconv(.c) bool {
574 @disableInstrumentation();
575 testing.allocator_instance = .init(std.heap.page_allocator, .{
576 .canary = 0xcacce5e0,
577 .check_write_after_free = true,
578 });
579 defer if (testing.allocator_instance.deinit() != 0) std.process.exit(1);
580 log_err_count = 0;
581 testOne(ctx, @constCast(&testing.Smith{ .in = null })) catch |err| switch (err) {
582 error.SkipZigTest => return true,
583 else => {
584 const stderr = std.debug.lockStderr(&.{}).terminal();
585 p: {
586 if (@errorReturnTrace()) |trace| {
587 std.debug.writeErrorReturnTrace(trace, stderr) catch break :p;
588 }
589 stderr.writer.print("failed with error.{t}\n", .{err}) catch break :p;
590 }
591 std.process.exit(1);
592 },
593 };
594 if (log_err_count != 0) {
595 const stderr = std.debug.lockStderr(&.{}).terminal();
596 stderr.writer.print("error logs detected\n", .{}) catch {};
597 std.process.exit(1);
598 }
599 return false;
600 }
601 };
602
603 if (builtin.fuzz) {
604 // Preserve the calling test's allocator state
605 const prev_allocator_state = testing.allocator_instance;
606 defer testing.allocator_instance = prev_allocator_state;
607
608 global.ctx = context;
609 fuzz_abi.fuzzer_set_test(&global.test_one);
610 for (options.corpus) |elem|
611 fuzz_abi.fuzzer_new_input(.fromSlice(elem));
612 fuzz_abi.fuzzer_start_test();
613 return;
614 }
615
616 // When the unit test executable is not built in fuzz mode, only run the
617 // provided corpus.
618 for (options.corpus) |input| {
619 var smith: testing.Smith = .{ .in = input };
620 try testOne(context, &smith);
621 }
622
623 // In case there is no provided corpus, also use an empty
624 // string as a smoke test.
625 var smith: testing.Smith = .{ .in = "" };
626 try testOne(context, &smith);
627}