authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-05-04 19:36:59+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-05-08 15:15:23+03:00
log59f9253d94331cedd4d0518250c8094a064f6cd2
treee87664a007e91a1af83240eb46b7890f35c418ab
parent530e67cb868fbd24900d48eee891efa1aa135096

allow tests to fail

2 files changed, 72 insertions(+), 51 deletions(-)

lib/std/special/test_runner.zig+9-6
......@@ -23,6 +23,7 @@ pub fn main() anyerror!void {
2323 const test_fn_list = builtin.test_functions;
2424 var ok_count: usize = 0;
2525 var skip_count: usize = 0;
26 var fail_count: usize = 0;
2627 var progress = std.Progress{};
2728 const root_node = progress.start("Test", test_fn_list.len) catch |err| switch (err) {
2829 // TODO still run tests in this case
......@@ -62,7 +63,7 @@ pub fn main() anyerror!void {
6263 .blocking => {
6364 skip_count += 1;
6465 test_node.end();
65 progress.log("{s}...SKIP (async test)\n", .{test_fn.name});
66 progress.log("{s}... SKIP (async test)\n", .{test_fn.name});
6667 if (progress.terminal == null) std.debug.print("SKIP (async test)\n", .{});
6768 continue;
6869 },
......@@ -75,12 +76,14 @@ pub fn main() anyerror!void {
7576 error.SkipZigTest => {
7677 skip_count += 1;
7778 test_node.end();
78 progress.log("{s}...SKIP\n", .{test_fn.name});
79 progress.log("{s}... SKIP\n", .{test_fn.name});
7980 if (progress.terminal == null) std.debug.print("SKIP\n", .{});
8081 },
8182 else => {
82 progress.log("", .{});
83 return err;
83 fail_count += 1;
84 test_node.end();
85 progress.log("{s}... FAIL ({s})\n", .{ test_fn.name, @errorName(err) });
86 if (progress.terminal == null) std.debug.print("FAIL ({s})\n", .{@errorName(err)});
8487 },
8588 }
8689 }
......@@ -88,7 +91,7 @@ pub fn main() anyerror!void {
8891 if (ok_count == test_fn_list.len) {
8992 std.debug.print("All {d} tests passed.\n", .{ok_count});
9093 } else {
91 std.debug.print("{d} passed; {d} skipped.\n", .{ ok_count, skip_count });
94 std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count });
9295 }
9396 if (log_err_count != 0) {
9497 std.debug.print("{d} errors were logged.\n", .{log_err_count});
......@@ -96,7 +99,7 @@ pub fn main() anyerror!void {
9699 if (leaks != 0) {
97100 std.debug.print("{d} tests leaked memory.\n", .{leaks});
98101 }
99 if (leaks != 0 or log_err_count != 0) {
102 if (leaks != 0 or log_err_count != 0 or fail_count != 0) {
100103 std.process.exit(1);
101104 }
102105}
lib/std/testing.zig+63-45
......@@ -27,15 +27,17 @@ pub var zig_exe_path: []const u8 = undefined;
2727
2828/// This function is intended to be used only in tests. It prints diagnostics to stderr
2929/// and then aborts when actual_error_union is not expected_error.
30pub fn expectError(expected_error: anyerror, actual_error_union: anytype) void {
30pub fn expectError(expected_error: anyerror, actual_error_union: anytype) !void {
3131 if (actual_error_union) |actual_payload| {
32 std.debug.panic("expected error.{s}, found {any}", .{ @errorName(expected_error), actual_payload });
32 std.debug.print("expected error.{s}, found {any}", .{ @errorName(expected_error), actual_payload });
33 return error.TestUnexpectedError;
3334 } else |actual_error| {
3435 if (expected_error != actual_error) {
35 std.debug.panic("expected error.{s}, found error.{s}", .{
36 std.debug.print("expected error.{s}, found error.{s}", .{
3637 @errorName(expected_error),
3738 @errorName(actual_error),
3839 });
40 return error.TestExpectedError;
3941 }
4042 }
4143}
......@@ -44,7 +46,7 @@ pub fn expectError(expected_error: anyerror, actual_error_union: anytype) void {
4446/// equal, prints diagnostics to stderr to show exactly how they are not equal,
4547/// then aborts.
4648/// `actual` is casted to the type of `expected`.
47pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
49pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) !void {
4850 switch (@typeInfo(@TypeOf(actual))) {
4951 .NoReturn,
5052 .BoundFn,
......@@ -60,7 +62,8 @@ pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
6062
6163 .Type => {
6264 if (actual != expected) {
63 std.debug.panic("expected type {s}, found type {s}", .{ @typeName(expected), @typeName(actual) });
65 std.debug.print("expected type {s}, found type {s}", .{ @typeName(expected), @typeName(actual) });
66 return error.TestExpectedEqual;
6467 }
6568 },
6669
......@@ -75,7 +78,8 @@ pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
7578 .ErrorSet,
7679 => {
7780 if (actual != expected) {
78 std.debug.panic("expected {}, found {}", .{ expected, actual });
81 std.debug.print("expected {}, found {}", .{ expected, actual });
82 return error.TestExpectedEqual;
7983 }
8084 },
8185
......@@ -83,34 +87,38 @@ pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
8387 switch (pointer.size) {
8488 .One, .Many, .C => {
8589 if (actual != expected) {
86 std.debug.panic("expected {*}, found {*}", .{ expected, actual });
90 std.debug.print("expected {*}, found {*}", .{ expected, actual });
91 return error.TestExpectedEqual;
8792 }
8893 },
8994 .Slice => {
9095 if (actual.ptr != expected.ptr) {
91 std.debug.panic("expected slice ptr {*}, found {*}", .{ expected.ptr, actual.ptr });
96 std.debug.print("expected slice ptr {*}, found {*}", .{ expected.ptr, actual.ptr });
97 return error.TestExpectedEqual;
9298 }
9399 if (actual.len != expected.len) {
94 std.debug.panic("expected slice len {}, found {}", .{ expected.len, actual.len });
100 std.debug.print("expected slice len {}, found {}", .{ expected.len, actual.len });
101 return error.TestExpectedEqual;
95102 }
96103 },
97104 }
98105 },
99106
100 .Array => |array| expectEqualSlices(array.child, &expected, &actual),
107 .Array => |array| try expectEqualSlices(array.child, &expected, &actual),
101108
102109 .Vector => |vectorType| {
103110 var i: usize = 0;
104111 while (i < vectorType.len) : (i += 1) {
105112 if (!std.meta.eql(expected[i], actual[i])) {
106 std.debug.panic("index {} incorrect. expected {}, found {}", .{ i, expected[i], actual[i] });
113 std.debug.print("index {} incorrect. expected {}, found {}", .{ i, expected[i], actual[i] });
114 return error.TestExpectedEqual;
107115 }
108116 }
109117 },
110118
111119 .Struct => |structType| {
112120 inline for (structType.fields) |field| {
113 expectEqual(@field(expected, field.name), @field(actual, field.name));
121 try expectEqual(@field(expected, field.name), @field(actual, field.name));
114122 }
115123 },
116124
......@@ -124,12 +132,12 @@ pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
124132 const expectedTag = @as(Tag, expected);
125133 const actualTag = @as(Tag, actual);
126134
127 expectEqual(expectedTag, actualTag);
135 try expectEqual(expectedTag, actualTag);
128136
129137 // we only reach this loop if the tags are equal
130138 inline for (std.meta.fields(@TypeOf(actual))) |fld| {
131139 if (std.mem.eql(u8, fld.name, @tagName(actualTag))) {
132 expectEqual(@field(expected, fld.name), @field(actual, fld.name));
140 try expectEqual(@field(expected, fld.name), @field(actual, fld.name));
133141 return;
134142 }
135143 }
......@@ -143,13 +151,15 @@ pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
143151 .Optional => {
144152 if (expected) |expected_payload| {
145153 if (actual) |actual_payload| {
146 expectEqual(expected_payload, actual_payload);
154 try expectEqual(expected_payload, actual_payload);
147155 } else {
148 std.debug.panic("expected {any}, found null", .{expected_payload});
156 std.debug.print("expected {any}, found null", .{expected_payload});
157 return error.TestExpectedEqual;
149158 }
150159 } else {
151160 if (actual) |actual_payload| {
152 std.debug.panic("expected null, found {any}", .{actual_payload});
161 std.debug.print("expected null, found {any}", .{actual_payload});
162 return error.TestExpectedEqual;
153163 }
154164 }
155165 },
......@@ -157,15 +167,17 @@ pub fn expectEqual(expected: anytype, actual: @TypeOf(expected)) void {
157167 .ErrorUnion => {
158168 if (expected) |expected_payload| {
159169 if (actual) |actual_payload| {
160 expectEqual(expected_payload, actual_payload);
170 try expectEqual(expected_payload, actual_payload);
161171 } else |actual_err| {
162 std.debug.panic("expected {any}, found {}", .{ expected_payload, actual_err });
172 std.debug.print("expected {any}, found {}", .{ expected_payload, actual_err });
173 return error.TestExpectedEqual;
163174 }
164175 } else |expected_err| {
165176 if (actual) |actual_payload| {
166 std.debug.panic("expected {}, found {any}", .{ expected_err, actual_payload });
177 std.debug.print("expected {}, found {any}", .{ expected_err, actual_payload });
178 return error.TestExpectedEqual;
167179 } else |actual_err| {
168 expectEqual(expected_err, actual_err);
180 try expectEqual(expected_err, actual_err);
169181 }
170182 }
171183 },
......@@ -181,7 +193,7 @@ test "expectEqual.union(enum)" {
181193 const a10 = T{ .a = 10 };
182194 const a20 = T{ .a = 20 };
183195
184 expectEqual(a10, a10);
196 try expectEqual(a10, a10);
185197}
186198
187199/// This function is intended to be used only in tests. When the formatted result of the template
......@@ -197,7 +209,7 @@ pub fn expectFmt(expected: []const u8, comptime template: []const u8, args: anyt
197209 print("\n======== instead found this: =========\n", .{});
198210 print("{s}", .{result});
199211 print("\n======================================\n", .{});
200 return error.TestFailed;
212 return error.TestExpectedFmt;
201213}
202214
203215pub const expectWithinMargin = @compileError("expectWithinMargin is deprecated, use expectApproxEqAbs or expectApproxEqRel");
......@@ -208,12 +220,14 @@ pub const expectWithinEpsilon = @compileError("expectWithinEpsilon is deprecated
208220/// to show exactly how they are not equal, then aborts.
209221/// See `math.approxEqAbs` for more informations on the tolerance parameter.
210222/// The types must be floating point
211pub fn expectApproxEqAbs(expected: anytype, actual: @TypeOf(expected), tolerance: @TypeOf(expected)) void {
223pub fn expectApproxEqAbs(expected: anytype, actual: @TypeOf(expected), tolerance: @TypeOf(expected)) !void {
212224 const T = @TypeOf(expected);
213225
214226 switch (@typeInfo(T)) {
215 .Float => if (!math.approxEqAbs(T, expected, actual, tolerance))
216 std.debug.panic("actual {}, not within absolute tolerance {} of expected {}", .{ actual, tolerance, expected }),
227 .Float => if (!math.approxEqAbs(T, expected, actual, tolerance)) {
228 std.debug.print("actual {}, not within absolute tolerance {} of expected {}", .{ actual, tolerance, expected });
229 return error.TestExpectedApproxEqAbs;
230 },
217231
218232 .ComptimeFloat => @compileError("Cannot approximately compare two comptime_float values"),
219233
......@@ -228,8 +242,8 @@ test "expectApproxEqAbs" {
228242 const neg_x: T = -12.0;
229243 const neg_y: T = -12.06;
230244
231 expectApproxEqAbs(pos_x, pos_y, 0.1);
232 expectApproxEqAbs(neg_x, neg_y, 0.1);
245 try expectApproxEqAbs(pos_x, pos_y, 0.1);
246 try expectApproxEqAbs(neg_x, neg_y, 0.1);
233247 }
234248}
235249
......@@ -238,12 +252,14 @@ test "expectApproxEqAbs" {
238252/// to show exactly how they are not equal, then aborts.
239253/// See `math.approxEqRel` for more informations on the tolerance parameter.
240254/// The types must be floating point
241pub fn expectApproxEqRel(expected: anytype, actual: @TypeOf(expected), tolerance: @TypeOf(expected)) void {
255pub fn expectApproxEqRel(expected: anytype, actual: @TypeOf(expected), tolerance: @TypeOf(expected)) !void {
242256 const T = @TypeOf(expected);
243257
244258 switch (@typeInfo(T)) {
245 .Float => if (!math.approxEqRel(T, expected, actual, tolerance))
246 std.debug.panic("actual {}, not within relative tolerance {} of expected {}", .{ actual, tolerance, expected }),
259 .Float => if (!math.approxEqRel(T, expected, actual, tolerance)) {
260 std.debug.print("actual {}, not within relative tolerance {} of expected {}", .{ actual, tolerance, expected });
261 return error.TestExpectedApproxEqRel;
262 },
247263
248264 .ComptimeFloat => @compileError("Cannot approximately compare two comptime_float values"),
249265
......@@ -261,8 +277,8 @@ test "expectApproxEqRel" {
261277 const neg_x: T = -12.0;
262278 const neg_y: T = neg_x - 2 * eps_value;
263279
264 expectApproxEqRel(pos_x, pos_y, sqrt_eps_value);
265 expectApproxEqRel(neg_x, neg_y, sqrt_eps_value);
280 try expectApproxEqRel(pos_x, pos_y, sqrt_eps_value);
281 try expectApproxEqRel(neg_x, neg_y, sqrt_eps_value);
266282 }
267283}
268284
......@@ -270,26 +286,28 @@ test "expectApproxEqRel" {
270286/// equal, prints diagnostics to stderr to show exactly how they are not equal,
271287/// then aborts.
272288/// If your inputs are UTF-8 encoded strings, consider calling `expectEqualStrings` instead.
273pub fn expectEqualSlices(comptime T: type, expected: []const T, actual: []const T) void {
289pub fn expectEqualSlices(comptime T: type, expected: []const T, actual: []const T) !void {
274290 // TODO better printing of the difference
275291 // If the arrays are small enough we could print the whole thing
276292 // If the child type is u8 and no weird bytes, we could print it as strings
277293 // Even for the length difference, it would be useful to see the values of the slices probably.
278294 if (expected.len != actual.len) {
279 std.debug.panic("slice lengths differ. expected {d}, found {d}", .{ expected.len, actual.len });
295 std.debug.print("slice lengths differ. expected {d}, found {d}", .{ expected.len, actual.len });
296 return error.TestExpectedEqual;
280297 }
281298 var i: usize = 0;
282299 while (i < expected.len) : (i += 1) {
283300 if (!std.meta.eql(expected[i], actual[i])) {
284 std.debug.panic("index {} incorrect. expected {any}, found {any}", .{ i, expected[i], actual[i] });
301 std.debug.print("index {} incorrect. expected {any}, found {any}", .{ i, expected[i], actual[i] });
302 return error.TestExpectedEqual;
285303 }
286304 }
287305}
288306
289307/// This function is intended to be used only in tests. When `ok` is false, the test fails.
290308/// A message is printed to stderr and then abort is called.
291pub fn expect(ok: bool) void {
292 if (!ok) @panic("test failure");
309pub fn expect(ok: bool) !void {
310 if (!ok) return error.TestUnexpectedResult;
293311}
294312
295313pub const TmpDir = struct {
......@@ -356,17 +374,17 @@ test "expectEqual nested array" {
356374 [_]f32{ 0.0, 1.0 },
357375 };
358376
359 expectEqual(a, b);
377 try expectEqual(a, b);
360378}
361379
362380test "expectEqual vector" {
363381 var a = @splat(4, @as(u32, 4));
364382 var b = @splat(4, @as(u32, 4));
365383
366 expectEqual(a, b);
384 try expectEqual(a, b);
367385}
368386
369pub fn expectEqualStrings(expected: []const u8, actual: []const u8) void {
387pub fn expectEqualStrings(expected: []const u8, actual: []const u8) !void {
370388 if (std.mem.indexOfDiff(u8, actual, expected)) |diff_index| {
371389 print("\n====== expected this output: =========\n", .{});
372390 printWithVisibleNewlines(expected);
......@@ -386,11 +404,11 @@ pub fn expectEqualStrings(expected: []const u8, actual: []const u8) void {
386404 print("found:\n", .{});
387405 printIndicatorLine(actual, diff_index);
388406
389 @panic("test failure");
407 return error.TestExpectedEqual;
390408 }
391409}
392410
393pub fn expectStringEndsWith(actual: []const u8, expected_ends_with: []const u8) void {
411pub fn expectStringEndsWith(actual: []const u8, expected_ends_with: []const u8) !void {
394412 if (std.mem.endsWith(u8, actual, expected_ends_with))
395413 return;
396414
......@@ -407,7 +425,7 @@ pub fn expectStringEndsWith(actual: []const u8, expected_ends_with: []const u8)
407425 printWithVisibleNewlines(actual);
408426 print("\n======================================\n", .{});
409427
410 @panic("test failure");
428 return error.TestExpectedEndsWith;
411429}
412430
413431fn printIndicatorLine(source: []const u8, indicator_index: usize) void {
......@@ -446,7 +464,7 @@ fn printLine(line: []const u8) void {
446464}
447465
448466test {
449 expectEqualStrings("foo", "foo");
467 try expectEqualStrings("foo", "foo");
450468}
451469
452470/// Given a type, reference all the declarations inside, so that the semantic analyzer sees them.