authorgravatar for noam@pixelhero.devNoam Preil <noam@pixelhero.dev> 2020-06-24 22:31:54-04:00
committergravatar for noam@pixelhero.devNoam Preil <noam@pixelhero.dev> 2020-06-24 22:43:18-04:00
log5d7e981f95423b3b009e0d7eebccae6c856f68ca
treef004a1cd292013b2f053bd44602de1c9f65eea6a
parentd337469e4484ffd160b4508e2366fefd435f6c8a
signature Commit is signed but in an unrecognized format.

Clean up test harness

5 files changed, 92 insertions(+), 126 deletions(-)

src-self-hosted/test.zig+68-100
......@@ -21,9 +21,10 @@ const ErrorMsg = struct {
2121};
2222
2323pub const TestContext = struct {
24 zir_cases: std.ArrayList(Case),
24 /// TODO: find a way to treat cases as individual tests (shouldn't show "1 test passed" if there are 200 cases)
25 cases: std.ArrayList(Case),
2526
26 pub const ZIRUpdate = struct {
27 pub const Update = struct {
2728 /// The input to the current update. We simulate an incremental update
2829 /// with the file's contents changed to this value each update.
2930 ///
......@@ -40,67 +41,70 @@ pub const TestContext = struct {
4041 /// fails to compile, and for the expected reasons.
4142 /// A slice containing the expected errors *in sequential order*.
4243 Error: []const ErrorMsg,
43 /// An execution update compiles and runs the input ZIR, feeding in
44 /// provided input and ensuring that the stdout match what is expected.
44 /// An execution update compiles and runs the input, testing the
45 /// stdout against the expected results
4546 Execution: []const u8,
4647 },
4748 };
4849
49 /// A Case consists of a set of *updates*. A update can transform ZIR,
50 /// compile it, ensure that compilation fails, and more. The same Module is
51 /// used for each update, so each update's source is treated as a single file
52 /// being updated by the test harness and incrementally compiled.
50 pub const TestType = enum {
51 Zig,
52 ZIR,
53 };
54
55 /// A Case consists of a set of *updates*. The same Module is used for each
56 /// update, so each update's source is treated as a single file being
57 /// updated by the test harness and incrementally compiled.
5358 pub const Case = struct {
5459 name: []const u8,
55 /// The platform the ZIR targets. For non-native platforms, an emulator
60 /// The platform the test targets. For non-native platforms, an emulator
5661 /// such as QEMU is required for tests to complete.
5762 target: std.zig.CrossTarget,
58 updates: std.ArrayList(ZIRUpdate),
5963 output_mode: std.builtin.OutputMode,
60 /// Either ".zir" or ".zig"
61 extension: [4]u8,
64 updates: std.ArrayList(Update),
65 @"type": TestType,
6266
6367 /// Adds a subcase in which the module is updated with new ZIR, and the
6468 /// resulting ZIR is validated.
65 pub fn addTransform(self: *Case, src: [:0]const u8, result: [:0]const u8) void {
66 self.updates.append(.{
69 pub fn addTransform(self: *Case, src: [:0]const u8, result: [:0]const u8) !void {
70 try self.updates.append(.{
6771 .src = src,
6872 .case = .{ .Transformation = result },
69 }) catch unreachable;
73 });
7074 }
7175
72 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void {
73 self.updates.append(.{
76 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) !void {
77 try self.updates.append(.{
7478 .src = src,
7579 .case = .{ .Execution = result },
76 }) catch unreachable;
80 });
7781 }
7882
7983 /// Adds a subcase in which the module is updated with invalid ZIR, and
8084 /// ensures that compilation fails for the expected reasons.
8185 ///
8286 /// Errors must be specified in sequential order.
83 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
84 var array = self.updates.allocator.alloc(ErrorMsg, errors.len) catch unreachable;
87 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) !void {
88 var array = try self.updates.allocator.alloc(ErrorMsg, errors.len);
8589 for (errors) |e, i| {
8690 if (e[0] != ':') {
87 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
91 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
8892 }
8993 var cur = e[1..];
9094 var line_index = std.mem.indexOf(u8, cur, ":");
9195 if (line_index == null) {
92 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
96 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
9397 }
9498 const line = std.fmt.parseInt(u32, cur[0..line_index.?], 10) catch @panic("Unable to parse line number");
9599 cur = cur[line_index.? + 1 ..];
96100 const column_index = std.mem.indexOf(u8, cur, ":");
97101 if (column_index == null) {
98 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
102 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
99103 }
100104 const column = std.fmt.parseInt(u32, cur[0..column_index.?], 10) catch @panic("Unable to parse column number");
101105 cur = cur[column_index.? + 2 ..];
102106 if (!std.mem.eql(u8, cur[0..7], "error: ")) {
103 std.debug.panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n", .{});
107 @panic("Invalid test: error must be specified as follows:\n:line:column: error: message\n=========\n");
104108 }
105109 const msg = cur[7..];
106110
......@@ -114,125 +118,87 @@ pub const TestContext = struct {
114118 .column = column - 1,
115119 };
116120 }
117 self.updates.append(.{ .src = src, .case = .{ .Error = array } }) catch unreachable;
121 try self.updates.append(.{ .src = src, .case = .{ .Error = array } });
118122 }
119123 };
120124
121 pub fn addExeZIR(
122 ctx: *TestContext,
123 name: []const u8,
124 target: std.zig.CrossTarget,
125 ) *Case {
126 const case = Case{
127 .name = name,
128 .target = target,
129 .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator),
130 .output_mode = .Exe,
131 .extension = ".zir".*,
132 };
133 ctx.zir_cases.append(case) catch unreachable;
134 return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1];
135 }
136
137 pub fn addObjZIR(
138 ctx: *TestContext,
139 name: []const u8,
140 target: std.zig.CrossTarget,
141 ) *Case {
142 const case = Case{
143 .name = name,
144 .target = target,
145 .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator),
146 .output_mode = .Obj,
147 .extension = ".zir".*,
148 };
149 ctx.zir_cases.append(case) catch unreachable;
150 return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1];
151 }
152
153125 pub fn addExe(
154126 ctx: *TestContext,
155127 name: []const u8,
156128 target: std.zig.CrossTarget,
157 ) *Case {
129 T: TestType,
130 ) !*Case {
158131 const case = Case{
159132 .name = name,
160133 .target = target,
161 .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator),
134 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
162135 .output_mode = .Exe,
163 .extension = ".zig".*,
136 .@"type" = T,
164137 };
165 ctx.zir_cases.append(case) catch unreachable;
166 return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1];
138 try ctx.cases.append(case);
139 return &ctx.cases.items[ctx.cases.items.len - 1];
167140 }
168141
169142 pub fn addObj(
170143 ctx: *TestContext,
171144 name: []const u8,
172145 target: std.zig.CrossTarget,
173 ) *Case {
174 const case = Case{
146 T: TestType,
147 ) !*Case {
148 try ctx.cases.append(Case{
175149 .name = name,
176150 .target = target,
177 .updates = std.ArrayList(ZIRUpdate).init(ctx.zir_cases.allocator),
151 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
178152 .output_mode = .Obj,
179 .extension = ".zig".*,
180 };
181 ctx.zir_cases.append(case) catch unreachable;
182 return &ctx.zir_cases.items[ctx.zir_cases.items.len - 1];
183 }
184
185 pub fn addZIRCompareOutput(
186 ctx: *TestContext,
187 name: []const u8,
188 src: [:0]const u8,
189 expected_stdout: []const u8,
190 ) void {
191 var c = ctx.addExeZIR(name, .{});
192 c.addCompareOutput(src, expected_stdout);
153 .@"type" = T,
154 });
155 return &ctx.cases.items[ctx.cases.items.len - 1];
193156 }
194157
195158 pub fn addCompareOutput(
196159 ctx: *TestContext,
197160 name: []const u8,
161 T: TestType,
198162 src: [:0]const u8,
199163 expected_stdout: []const u8,
200 ) void {
201 var c = ctx.addExe(name, .{});
202 c.addCompareOutput(src, expected_stdout);
164 ) !void {
165 var c = try ctx.addExe(name, .{}, T);
166 try c.addCompareOutput(src, expected_stdout);
203167 }
204168
205 pub fn addZIRTransform(
169 pub fn addTransform(
206170 ctx: *TestContext,
207171 name: []const u8,
208172 target: std.zig.CrossTarget,
173 T: TestType,
209174 src: [:0]const u8,
210175 result: [:0]const u8,
211 ) void {
212 var c = ctx.addObjZIR(name, target);
213 c.addTransform(src, result);
176 ) !void {
177 var c = try ctx.addObj(name, target, T);
178 try c.addTransform(src, result);
214179 }
215180
216 pub fn addZIRError(
181 pub fn addError(
217182 ctx: *TestContext,
218183 name: []const u8,
219184 target: std.zig.CrossTarget,
185 T: TestType,
220186 src: [:0]const u8,
221187 expected_errors: []const []const u8,
222 ) void {
223 var c = ctx.addObjZIR(name, target);
224 c.addError(src, expected_errors);
188 ) !void {
189 var c = try ctx.addObj(name, target, T);
190 try c.addError(src, expected_errors);
225191 }
226192
227193 fn init() TestContext {
228194 const allocator = std.heap.page_allocator;
229195 return .{
230 .zir_cases = std.ArrayList(Case).init(allocator),
196 .cases = std.ArrayList(Case).init(allocator),
231197 };
232198 }
233199
234200 fn deinit(self: *TestContext) void {
235 for (self.zir_cases.items) |c| {
201 for (self.cases.items) |c| {
236202 for (c.updates.items) |u| {
237203 if (u.case == .Error) {
238204 c.updates.allocator.free(u.case.Error);
......@@ -240,18 +206,18 @@ pub const TestContext = struct {
240206 }
241207 c.updates.deinit();
242208 }
243 self.zir_cases.deinit();
209 self.cases.deinit();
244210 self.* = undefined;
245211 }
246212
247213 fn run(self: *TestContext) !void {
248214 var progress = std.Progress{};
249 const root_node = try progress.start("zir", self.zir_cases.items.len);
215 const root_node = try progress.start("tests", self.cases.items.len);
250216 defer root_node.end();
251217
252218 const native_info = try std.zig.system.NativeTargetInfo.detect(std.heap.page_allocator, .{});
253219
254 for (self.zir_cases.items) |case| {
220 for (self.cases.items) |case| {
255221 std.testing.base_allocator_instance.reset();
256222
257223 var prg_node = root_node.start(case.name, case.updates.items.len);
......@@ -267,17 +233,19 @@ pub const TestContext = struct {
267233 }
268234 }
269235
270 fn runOneCase(self: *TestContext, allocator: *Allocator, prg_node: *std.Progress.Node, case: Case, target: std.Target) !void {
236 fn runOneCase(self: *TestContext, allocator: *Allocator, root_node: *std.Progress.Node, case: Case, target: std.Target) !void {
271237 var tmp = std.testing.tmpDir(.{});
272238 defer tmp.cleanup();
273239
274 const root_name = "test_case";
275 const tmp_src_path = try std.fmt.allocPrint(allocator, "{}{}", .{ root_name, case.extension });
276 defer allocator.free(tmp_src_path);
240 const tmp_src_path = if (case.type == .Zig) "test_case.zig" else if (case.type == .ZIR) "test_case.zir" else unreachable;
277241 const root_pkg = try Package.create(allocator, tmp.dir, ".", tmp_src_path);
278242 defer root_pkg.destroy();
279243
280 const bin_name = try std.zig.binNameAlloc(allocator, root_name, target, case.output_mode, null);
244 var prg_node = root_node.start(case.name, case.updates.items.len);
245 prg_node.activate();
246 defer prg_node.end();
247
248 const bin_name = try std.zig.binNameAlloc(allocator, "test_case", target, case.output_mode, null);
281249 defer allocator.free(bin_name);
282250
283251 var module = try Module.init(allocator, .{
test/stage2/compare_output.zig+4-4
......@@ -17,9 +17,9 @@ pub fn addCases(ctx: *TestContext) !void {
1717 }
1818
1919 {
20 var case = ctx.addExe("hello world with updates", linux_x64);
20 var case = try ctx.addExe("hello world with updates", linux_x64, .Zig);
2121 // Regular old hello world
22 case.addCompareOutput(
22 try case.addCompareOutput(
2323 \\export fn _start() noreturn {
2424 \\ print();
2525 \\
......@@ -51,7 +51,7 @@ pub fn addCases(ctx: *TestContext) !void {
5151 "Hello, World!\n",
5252 );
5353 // Now change the message only
54 case.addCompareOutput(
54 try case.addCompareOutput(
5555 \\export fn _start() noreturn {
5656 \\ print();
5757 \\
......@@ -83,7 +83,7 @@ pub fn addCases(ctx: *TestContext) !void {
8383 "What is up? This is a longer message that will force the data to be relocated in virtual address space.\n",
8484 );
8585 // Now we print it twice.
86 case.addCompareOutput(
86 try case.addCompareOutput(
8787 \\export fn _start() noreturn {
8888 \\ print();
8989 \\ print();
test/stage2/compile_errors.zig+10-12
......@@ -9,7 +9,7 @@ const linux_x64 = std.zig.CrossTarget{
99};
1010
1111pub fn addCases(ctx: *TestContext) !void {
12 ctx.addZIRError("call undefined local", linux_x64,
12 try ctx.addError("call undefined local", linux_x64, .ZIR,
1313 \\@noreturn = primitive(noreturn)
1414 \\
1515 \\@start_fnty = fntype([], @noreturn, cc=Naked)
......@@ -19,7 +19,7 @@ pub fn addCases(ctx: *TestContext) !void {
1919 // TODO: address inconsistency in this message and the one in the next test
2020 , &[_][]const u8{":5:13: error: unrecognized identifier: %test"});
2121
22 ctx.addZIRError("call with non-existent target", linux_x64,
22 try ctx.addError("call with non-existent target", linux_x64, .ZIR,
2323 \\@noreturn = primitive(noreturn)
2424 \\
2525 \\@start_fnty = fntype([], @noreturn, cc=Naked)
......@@ -31,7 +31,7 @@ pub fn addCases(ctx: *TestContext) !void {
3131 , &[_][]const u8{":5:13: error: decl 'notafunc' not found"});
3232
3333 // TODO: this error should occur at the call site, not the fntype decl
34 ctx.addZIRError("call naked function", linux_x64,
34 try ctx.addError("call naked function", linux_x64, .ZIR,
3535 \\@noreturn = primitive(noreturn)
3636 \\
3737 \\@start_fnty = fntype([], @noreturn, cc=Naked)
......@@ -45,17 +45,15 @@ pub fn addCases(ctx: *TestContext) !void {
4545
4646 // TODO: re-enable these tests.
4747 // https://github.com/ziglang/zig/issues/1364
48 // TODO: add Zig AST -> ZIR testing pipeline
4948
50 //try ctx.testCompileError(
51 // \\export fn entry() void {}
52 // \\export fn entry() void {}
53 //, "1.zig", 2, 8, "exported symbol collision: 'entry'");
54
55 //try ctx.testCompileError(
56 // \\fn() void {}
57 //, "1.zig", 1, 1, "missing function name");
49 // try ctx.addError("Export same symbol twice", linux_x64, .Zig,
50 // \\export fn entry() void {}
51 // \\export fn entry() void {}
52 // , &[_][]const u8{":2:1: error: exported symbol collision"});
5853
54 // try ctx.addError("Missing function name", linux_x64, .Zig,
55 // \\fn() void {}
56 // , &[_][]const u8{":1:3: error: missing function name"});
5957 //try ctx.testCompileError(
6058 // \\comptime {
6159 // \\ return;
test/stage2/test.zig+1-1
......@@ -3,5 +3,5 @@ const TestContext = @import("../../src-self-hosted/test.zig").TestContext;
33pub fn addCases(ctx: *TestContext) !void {
44 try @import("compile_errors.zig").addCases(ctx);
55 try @import("compare_output.zig").addCases(ctx);
6 @import("zir.zig").addCases(ctx);
6 try @import("zir.zig").addCases(ctx);
77}
test/stage2/zir.zig+9-9
......@@ -8,8 +8,8 @@ const linux_x64 = std.zig.CrossTarget{
88 .os_tag = .linux,
99};
1010
11pub fn addCases(ctx: *TestContext) void {
12 ctx.addZIRTransform("referencing decls which appear later in the file", linux_x64,
11pub fn addCases(ctx: *TestContext) !void {
12 try ctx.addTransform("referencing decls which appear later in the file", linux_x64, .ZIR,
1313 \\@void = primitive(void)
1414 \\@fnty = fntype([], @void, cc=C)
1515 \\
......@@ -32,7 +32,7 @@ pub fn addCases(ctx: *TestContext) void {
3232 \\})
3333 \\
3434 );
35 ctx.addZIRTransform("elemptr, add, cmp, condbr, return, breakpoint", linux_x64,
35 try ctx.addTransform("elemptr, add, cmp, condbr, return, breakpoint", linux_x64, .ZIR,
3636 \\@void = primitive(void)
3737 \\@usize = primitive(usize)
3838 \\@fnty = fntype([], @void, cc=C)
......@@ -86,8 +86,8 @@ pub fn addCases(ctx: *TestContext) void {
8686 );
8787
8888 {
89 var case = ctx.addObjZIR("reference cycle with compile error in the cycle", linux_x64);
90 case.addTransform(
89 var case = try ctx.addObj("reference cycle with compile error in the cycle", linux_x64, .ZIR);
90 try case.addTransform(
9191 \\@void = primitive(void)
9292 \\@fnty = fntype([], @void, cc=C)
9393 \\
......@@ -133,7 +133,7 @@ pub fn addCases(ctx: *TestContext) void {
133133 \\
134134 );
135135 // Now we introduce a compile error
136 case.addError(
136 try case.addError(
137137 \\@void = primitive(void)
138138 \\@fnty = fntype([], @void, cc=C)
139139 \\
......@@ -163,7 +163,7 @@ pub fn addCases(ctx: *TestContext) void {
163163 // Now we remove the call to `a`. `a` and `b` form a cycle, but no entry points are
164164 // referencing either of them. This tests that the cycle is detected, and the error
165165 // goes away.
166 case.addTransform(
166 try case.addTransform(
167167 \\@void = primitive(void)
168168 \\@fnty = fntype([], @void, cc=C)
169169 \\
......@@ -207,7 +207,7 @@ pub fn addCases(ctx: *TestContext) void {
207207 return;
208208 }
209209
210 ctx.addZIRCompareOutput("hello world ZIR",
210 try ctx.addCompareOutput("hello world ZIR", .ZIR,
211211 \\@noreturn = primitive(noreturn)
212212 \\@void = primitive(void)
213213 \\@usize = primitive(usize)
......@@ -265,7 +265,7 @@ pub fn addCases(ctx: *TestContext) void {
265265 \\
266266 );
267267
268 ctx.addZIRCompareOutput("function call with no args no return value",
268 try ctx.addCompareOutput("function call with no args no return value", .ZIR,
269269 \\@noreturn = primitive(noreturn)
270270 \\@void = primitive(void)
271271 \\@usize = primitive(usize)