authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-06-16 03:50:56-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-06-16 03:50:56-04:00
logf595545c10a35b85879edfa3c002ce308ffeb6c2
tree16c24db60fcf42a02902a8e86b07678909b62272
parent2bb3e1aff4976b2d04fb08a46d9221c77da0b204
parenta99e61ebaa71aa74dfa95869ea8d02131ef9f696
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5422 from pixelherodev/error_tests

[Stage2/Testing] ZIR tests for expected errors

3 files changed, 304 insertions(+), 174 deletions(-)

src-self-hosted/test.zig+257-172
......@@ -6,8 +6,7 @@ const zir = @import("zir.zig");
66const Package = @import("Package.zig");
77
88test "self-hosted" {
9 var ctx: TestContext = undefined;
10 try ctx.init();
9 var ctx = TestContext.init();
1110 defer ctx.deinit();
1211
1312 try @import("stage2_tests").addCases(&ctx);
......@@ -15,46 +14,152 @@ test "self-hosted" {
1514 try ctx.run();
1615}
1716
17const ErrorMsg = struct {
18 msg: []const u8,
19 line: u32,
20 column: u32,
21};
22
1823pub const TestContext = struct {
24 // TODO: remove these. They are deprecated.
1925 zir_cmp_output_cases: std.ArrayList(ZIRCompareOutputCase),
20 zir_transform_cases: std.ArrayList(ZIRTransformCase),
2126
27 /// TODO: find a way to treat cases as individual tests (shouldn't show "1 test passed" if there are 200 cases)
28 zir_cases: std.ArrayList(ZIRCase),
29
30 // TODO: remove
2231 pub const ZIRCompareOutputCase = struct {
2332 name: []const u8,
2433 src_list: []const []const u8,
2534 expected_stdout_list: []const []const u8,
2635 };
2736
28 pub const ZIRTransformCase = struct {
29 name: []const u8,
30 cross_target: std.zig.CrossTarget,
31 updates: std.ArrayList(Update),
32
33 pub const Update = struct {
34 expected: Expected,
35 src: [:0]const u8,
36 };
37 pub const ZIRUpdateType = enum {
38 /// A transformation update transforms the input ZIR and tests against
39 /// the expected output
40 Transformation,
41 /// An error update attempts to compile bad code, and ensures that it
42 /// fails to compile, and for the expected reasons
43 Error,
44 /// An execution update compiles and runs the input ZIR, feeding in
45 /// provided input and ensuring that the outputs match what is expected
46 Execution,
47 /// A compilation update checks that the ZIR compiles without any issues
48 Compiles,
49 };
3750
38 pub const Expected = union(enum) {
39 zir: []const u8,
40 errors: []const []const u8,
41 };
51 pub const ZIRUpdate = struct {
52 /// The input to the current update. We simulate an incremental update
53 /// with the file's contents changed to this value each update.
54 ///
55 /// This value can change entirely between updates, which would be akin
56 /// to deleting the source file and creating a new one from scratch; or
57 /// you can keep it mostly consistent, with small changes, testing the
58 /// effects of the incremental compilation.
59 src: [:0]const u8,
60 case: union(ZIRUpdateType) {
61 /// The expected output ZIR
62 Transformation: [:0]const u8,
63 /// A slice containing the expected errors *in sequential order*.
64 Error: []const ErrorMsg,
65
66 /// Input to feed to the program, and expected outputs.
67 ///
68 /// If stdout, stderr, and exit_code are all null, addZIRCase will
69 /// discard the test. To test for successful compilation, use a
70 /// dedicated Compile update instead.
71 Execution: struct {
72 stdin: ?[]const u8,
73 stdout: ?[]const u8,
74 stderr: ?[]const u8,
75 exit_code: ?u8,
76 },
77 /// A Compiles test checks only that compilation of the given ZIR
78 /// succeeds. To test outputs, use an Execution test. It is good to
79 /// use a Compiles test before an Execution, as the overhead should
80 /// be low (due to incremental compilation) and TODO: provide a way
81 /// to check changed / new / etc decls in testing mode
82 /// (usingnamespace a debug info struct with a comptime flag?)
83 Compiles: void,
84 },
85 };
4286
43 pub fn addZIR(case: *ZIRTransformCase, src: [:0]const u8, zir_text: []const u8) void {
44 case.updates.append(.{
87 /// A ZIRCase consists of a set of *updates*. A update can transform ZIR,
88 /// compile it, ensure that compilation fails, and more. The same Module is
89 /// used for each update, so each update's source is treated as a single file
90 /// being updated by the test harness and incrementally compiled.
91 pub const ZIRCase = struct {
92 name: []const u8,
93 /// The platform the ZIR targets. For non-native platforms, an emulator
94 /// such as QEMU is required for tests to complete.
95 target: std.zig.CrossTarget,
96 updates: std.ArrayList(ZIRUpdate),
97
98 /// Adds a subcase in which the module is updated with new ZIR, and the
99 /// resulting ZIR is validated.
100 pub fn addTransform(self: *ZIRCase, src: [:0]const u8, result: [:0]const u8) void {
101 self.updates.append(.{
45102 .src = src,
46 .expected = .{ .zir = zir_text },
103 .case = .{ .Transformation = result },
47104 }) catch unreachable;
48105 }
49106
50 pub fn addError(case: *ZIRTransformCase, src: [:0]const u8, errors: []const []const u8) void {
51 case.updates.append(.{
52 .src = src,
53 .expected = .{ .errors = errors },
54 }) catch unreachable;
107 /// Adds a subcase in which the module is updated with invalid ZIR, and
108 /// ensures that compilation fails for the expected reasons.
109 ///
110 /// Errors must be specified in sequential order.
111 pub fn addError(self: *ZIRCase, src: [:0]const u8, errors: []const []const u8) void {
112 var array = self.updates.allocator.alloc(ErrorMsg, errors.len) catch unreachable;
113 for (errors) |e, i| {
114 if (e[0] != ':') {
115 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
116 }
117 var cur = e[1..];
118 var line_index = std.mem.indexOf(u8, cur, ":");
119 if (line_index == null) {
120 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
121 }
122 const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number");
123 cur = cur[line_index.? + 1 ..];
124 const column_index = std.mem.indexOf(u8, cur, ":");
125 if (column_index == null) {
126 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
127 }
128 const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number");
129 cur = cur[column_index.? + 2 ..];
130 if (!std.mem.eql(u8, cur[0..7], "error: ")) {
131 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
132 }
133 const msg = cur[7..];
134
135 if (line == 0 or column == 0) {
136 @panic("Invalid test: error line and column must be specified starting at one!");
137 }
138
139 array[i] = .{
140 .msg = msg,
141 .line = line - 1,
142 .column = column - 1,
143 };
144 }
145 self.updates.append(.{ .src = src, .case = .{ .Error = array } }) catch unreachable;
55146 }
56147 };
57148
149 pub fn addZIRMulti(
150 ctx: *TestContext,
151 name: []const u8,
152 target: std.zig.CrossTarget,
153 ) *ZIRCase {
154 const case = ZIRCase{
155 .name = name,
156 .target = target,
157 .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator),
158 };
159 ctx.zir_cases.append(case) catch unreachable;
160 return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1];
161 }
162
58163 pub fn addZIRCompareOutput(
59164 ctx: *TestContext,
60165 name: []const u8,
......@@ -71,70 +176,164 @@ pub const TestContext = struct {
71176 pub fn addZIRTransform(
72177 ctx: *TestContext,
73178 name: []const u8,
74 cross_target: std.zig.CrossTarget,
179 target: std.zig.CrossTarget,
75180 src: [:0]const u8,
76 expected_zir: []const u8,
181 result: [:0]const u8,
77182 ) void {
78 const case = ctx.zir_transform_cases.addOne() catch unreachable;
79 case.* = .{
80 .name = name,
81 .cross_target = cross_target,
82 .updates = std.ArrayList(ZIRTransformCase.Update).init(std.heap.page_allocator),
83 };
84 case.updates.append(.{
85 .src = src,
86 .expected = .{ .zir = expected_zir },
87 }) catch unreachable;
183 var c = ctx.addZIRMulti(name, target);
184 c.addTransform(src, result);
88185 }
89186
90 pub fn addZIRMulti(
187 pub fn addZIRError(
91188 ctx: *TestContext,
92189 name: []const u8,
93 cross_target: std.zig.CrossTarget,
94 ) *ZIRTransformCase {
95 const case = ctx.zir_transform_cases.addOne() catch unreachable;
96 case.* = .{
97 .name = name,
98 .cross_target = cross_target,
99 .updates = std.ArrayList(ZIRTransformCase.Update).init(std.heap.page_allocator),
100 };
101 return case;
190 target: std.zig.CrossTarget,
191 src: [:0]const u8,
192 expected_errors: []const []const u8,
193 ) void {
194 var c = ctx.addZIRMulti(name, target);
195 c.addError(src, expected_errors);
102196 }
103197
104 fn init(self: *TestContext) !void {
105 self.* = .{
106 .zir_cmp_output_cases = std.ArrayList(ZIRCompareOutputCase).init(std.heap.page_allocator),
107 .zir_transform_cases = std.ArrayList(ZIRTransformCase).init(std.heap.page_allocator),
198 fn init() TestContext {
199 const allocator = std.heap.page_allocator;
200 return .{
201 .zir_cmp_output_cases = std.ArrayList(ZIRCompareOutputCase).init(allocator),
202 .zir_cases = std.ArrayList(ZIRCase).init(allocator),
108203 };
109204 }
110205
111206 fn deinit(self: *TestContext) void {
112207 self.zir_cmp_output_cases.deinit();
113 self.zir_transform_cases.deinit();
208 for (self.zir_cases.items) |c| {
209 for (c.updates.items) |u| {
210 if (u.case == .Error) {
211 c.updates.allocator.free(u.case.Error);
212 }
213 }
214 c.updates.deinit();
215 }
216 self.zir_cases.deinit();
114217 self.* = undefined;
115218 }
116219
117220 fn run(self: *TestContext) !void {
118221 var progress = std.Progress{};
119 const root_node = try progress.start("zir", self.zir_cmp_output_cases.items.len +
120 self.zir_transform_cases.items.len);
222 const root_node = try progress.start("zir", self.zir_cases.items.len);
121223 defer root_node.end();
122224
123225 const native_info = try std.zig.system.NativeTargetInfo.detect(std.heap.page_allocator, .{});
124226
125 for (self.zir_cmp_output_cases.items) |case| {
227 for (self.zir_cases.items) |case| {
126228 std.testing.base_allocator_instance.reset();
127 try self.runOneZIRCmpOutputCase(std.testing.allocator, root_node, case, native_info.target);
229 const info = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, case.target);
230 try self.runOneZIRCase(std.testing.allocator, root_node, case, info.target);
128231 try std.testing.allocator_instance.validate();
129232 }
130 for (self.zir_transform_cases.items) |case| {
233
234 // TODO: wipe the rest of this function
235 for (self.zir_cmp_output_cases.items) |case| {
131236 std.testing.base_allocator_instance.reset();
132 const info = try std.zig.system.NativeTargetInfo.detect(std.testing.allocator, case.cross_target);
133 try self.runOneZIRTransformCase(std.testing.allocator, root_node, case, info.target);
237 try self.runOneZIRCmpOutputCase(std.testing.allocator, root_node, case, native_info.target);
134238 try std.testing.allocator_instance.validate();
135239 }
136240 }
137241
242 fn runOneZIRCase(self: *TestContext, allocator: *Allocator, root_node: *std.Progress.Node, case: ZIRCase, target: std.Target) !void {
243 var tmp = std.testing.tmpDir(.{});
244 defer tmp.cleanup();
245
246 const tmp_src_path = "test_case.zir";
247 const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path);
248 defer root_pkg.destroy();
249
250 var prg_node = root_node.start(case.name, case.updates.items.len);
251 prg_node.activate();
252 defer prg_node.end();
253
254 var module = try Module.init(allocator, .{
255 .target = target,
256 // This is an Executable, as opposed to e.g. a *library*. This does
257 // not mean no ZIR is generated.
258 //
259 // TODO: support tests for object file building, and library builds
260 // and linking. This will require a rework to support multi-file
261 // tests.
262 .output_mode = .Obj,
263 // TODO: support testing optimizations
264 .optimize_mode = .Debug,
265 .bin_file_dir = tmp.dir,
266 .bin_file_path = "test_case.o",
267 .root_pkg = root_pkg,
268 });
269 defer module.deinit();
270
271 for (case.updates.items) |update| {
272 var update_node = prg_node.start("update", 4);
273 update_node.activate();
274 defer update_node.end();
275
276 var sync_node = update_node.start("write", null);
277 sync_node.activate();
278 try tmp.dir.writeFile(tmp_src_path, update.src);
279 sync_node.end();
280
281 var module_node = update_node.start("parse/analysis/codegen", null);
282 module_node.activate();
283 try module.update();
284 module_node.end();
285
286 switch (update.case) {
287 .Transformation => |expected_output| {
288 var emit_node = update_node.start("emit", null);
289 emit_node.activate();
290 var new_zir_module = try zir.emit(allocator, module);
291 defer new_zir_module.deinit(allocator);
292 emit_node.end();
293
294 var write_node = update_node.start("write", null);
295 write_node.activate();
296 var out_zir = std.ArrayList(u8).init(allocator);
297 defer out_zir.deinit();
298 try new_zir_module.writeToStream(allocator, out_zir.outStream());
299 write_node.end();
300
301 std.testing.expectEqualSlices(u8, expected_output, out_zir.items);
302 },
303 .Error => |e| {
304 var handled_errors = try allocator.alloc(bool, e.len);
305 defer allocator.free(handled_errors);
306 for (handled_errors) |*h| {
307 h.* = false;
308 }
309 var all_errors = try module.getAllErrorsAlloc();
310 defer all_errors.deinit(allocator);
311 for (all_errors.list) |a| {
312 for (e) |ex, i| {
313 if (a.line == ex.line and a.column == ex.column and std.mem.eql(u8, ex.msg, a.msg)) {
314 handled_errors[i] = true;
315 break;
316 }
317 } else {
318 std.debug.warn("{}\nUnexpected error:\n================\n:{}:{}: error: {}\n================\nTest failed.\n", .{ case.name, a.line + 1, a.column + 1, a.msg });
319 std.process.exit(1);
320 }
321 }
322
323 for (handled_errors) |h, i| {
324 if (!h) {
325 const er = e[i];
326 std.debug.warn("{}\nDid not receive error:\n================\n{}:{}: {}\n================\nTest failed.\n", .{ case.name, er.line, er.column, er.msg });
327 std.process.exit(1);
328 }
329 }
330 },
331
332 else => return error.unimplemented,
333 }
334 }
335 }
336
138337 fn runOneZIRCmpOutputCase(
139338 self: *TestContext,
140339 allocator: *Allocator,
......@@ -208,118 +407,4 @@ pub const TestContext = struct {
208407 }
209408 }
210409 }
211
212 fn runOneZIRTransformCase(
213 self: *TestContext,
214 allocator: *Allocator,
215 root_node: *std.Progress.Node,
216 case: ZIRTransformCase,
217 target: std.Target,
218 ) !void {
219 var tmp = std.testing.tmpDir(.{});
220 defer tmp.cleanup();
221
222 var update_node = root_node.start(case.name, case.updates.items.len);
223 update_node.activate();
224 defer update_node.end();
225
226 const tmp_src_path = "test-case.zir";
227 const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path);
228 defer root_pkg.destroy();
229
230 var module = try Module.init(allocator, .{
231 .target = target,
232 .output_mode = .Obj,
233 .optimize_mode = .Debug,
234 .bin_file_dir = tmp.dir,
235 .bin_file_path = "test-case.o",
236 .root_pkg = root_pkg,
237 });
238 defer module.deinit();
239
240 for (case.updates.items) |update| {
241 var prg_node = update_node.start("", 3);
242 prg_node.activate();
243 defer prg_node.end();
244
245 try tmp.dir.writeFile(tmp_src_path, update.src);
246
247 var module_node = prg_node.start("parse/analysis/codegen", null);
248 module_node.activate();
249 try module.update();
250 module_node.end();
251
252 switch (update.expected) {
253 .zir => |expected_zir| {
254 var emit_node = prg_node.start("emit", null);
255 emit_node.activate();
256 var new_zir_module = try zir.emit(allocator, module);
257 defer new_zir_module.deinit(allocator);
258 emit_node.end();
259
260 var write_node = prg_node.start("write", null);
261 write_node.activate();
262 var out_zir = std.ArrayList(u8).init(allocator);
263 defer out_zir.deinit();
264 try new_zir_module.writeToStream(allocator, out_zir.outStream());
265 write_node.end();
266
267 std.testing.expectEqualSlices(u8, expected_zir, out_zir.items);
268 },
269 .errors => |expected_errors| {
270 var all_errors = try module.getAllErrorsAlloc();
271 defer all_errors.deinit(module.allocator);
272 for (expected_errors) |expected_error| {
273 for (all_errors.list) |full_err_msg| {
274 const text = try std.fmt.allocPrint(allocator, ":{}:{}: error: {}", .{
275 full_err_msg.line + 1,
276 full_err_msg.column + 1,
277 full_err_msg.msg,
278 });
279 defer allocator.free(text);
280 if (std.mem.eql(u8, text, expected_error)) {
281 break;
282 }
283 } else {
284 std.debug.warn(
285 "{}\nExpected this error:\n================\n{}\n================\nBut found these errors:\n================\n",
286 .{ case.name, expected_error },
287 );
288 for (all_errors.list) |full_err_msg| {
289 std.debug.warn(":{}:{}: error: {}\n", .{
290 full_err_msg.line + 1,
291 full_err_msg.column + 1,
292 full_err_msg.msg,
293 });
294 }
295 std.debug.warn("================\nTest failed\n", .{});
296 std.process.exit(1);
297 }
298 }
299 },
300 }
301 }
302 }
303410};
304
305fn debugPrintErrors(src: []const u8, errors: var) void {
306 std.debug.warn("\n", .{});
307 var nl = true;
308 var line: usize = 1;
309 for (src) |byte| {
310 if (nl) {
311 std.debug.warn("{: >3}| ", .{line});
312 nl = false;
313 }
314 if (byte == '\n') {
315 nl = true;
316 line += 1;
317 }
318 std.debug.warn("{c}", .{byte});
319 }
320 std.debug.warn("\n", .{});
321 for (errors) |err_msg| {
322 const loc = std.zig.findLineColumn(src, err_msg.byte_offset);
323 std.debug.warn("{}:{}: error: {}\n", .{ loc.line + 1, loc.column + 1, err_msg.msg });
324 }
325}
test/stage2/compile_errors.zig+45
......@@ -1,8 +1,53 @@
11const TestContext = @import("../../src-self-hosted/test.zig").TestContext;
2const std = @import("std");
3
4const ErrorMsg = @import("../../src-self-hosted/Module.zig").ErrorMsg;
5
6const linux_x64 = std.zig.CrossTarget{
7 .cpu_arch = .x86_64,
8 .os_tag = .linux,
9};
210
311pub fn addCases(ctx: *TestContext) !void {
12 ctx.addZIRError("call undefined local", linux_x64,
13 \\@noreturn = primitive(noreturn)
14 \\
15 \\@start_fnty = fntype([], @noreturn, cc=Naked)
16 \\@start = fn(@start_fnty, {
17 \\ %0 = call(%test, [])
18 \\})
19 // TODO: address inconsistency in this message and the one in the next test
20 , &[_][]const u8{":5:13: error: unrecognized identifier: %test"});
21
22 ctx.addZIRError("call with non-existent target", linux_x64,
23 \\@noreturn = primitive(noreturn)
24 \\
25 \\@start_fnty = fntype([], @noreturn, cc=Naked)
26 \\@start = fn(@start_fnty, {
27 \\ %0 = call(@notafunc, [])
28 \\})
29 \\@0 = str("_start")
30 \\@1 = ref(@0)
31 \\@2 = export(@1, @start)
32 , &[_][]const u8{":5:13: error: use of undeclared identifier 'notafunc'"});
33
34 // TODO: this error should occur at the call site, not the fntype decl
35 ctx.addZIRError("call naked function", linux_x64,
36 \\@noreturn = primitive(noreturn)
37 \\
38 \\@start_fnty = fntype([], @noreturn, cc=Naked)
39 \\@s = fn(@start_fnty, {})
40 \\@start = fn(@start_fnty, {
41 \\ %0 = call(@s, [])
42 \\})
43 \\@0 = str("_start")
44 \\@1 = ref(@0)
45 \\@2 = export(@1, @start)
46 , &[_][]const u8{":4:9: error: unable to call function with naked calling convention"});
47
448 // TODO: re-enable these tests.
549 // https://github.com/ziglang/zig/issues/1364
50 // TODO: add Zig AST -> ZIR testing pipeline
651
752 //try ctx.testCompileError(
853 // \\export fn entry() void {}
test/stage2/zir.zig+2-2
......@@ -92,7 +92,7 @@ pub fn addCases(ctx: *TestContext) void {
9292
9393 {
9494 var case = ctx.addZIRMulti("reference cycle with compile error in the cycle", linux_x64);
95 case.addZIR(
95 case.addTransform(
9696 \\@void = primitive(void)
9797 \\@fnty = fntype([], @void, cc=C)
9898 \\
......@@ -171,7 +171,7 @@ pub fn addCases(ctx: *TestContext) void {
171171 // Now we remove the call to `a`. `a` and `b` form a cycle, but no entry points are
172172 // referencing either of them. This tests that the cycle is detected, and the error
173173 // goes away.
174 case.addZIR(
174 case.addTransform(
175175 \\@void = primitive(void)
176176 \\@fnty = fntype([], @void, cc=C)
177177 \\