authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-05 14:13:26+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-23 00:52:50+00:00
loga3b3a33d7a365bde9cb71cd5a5bf6663754b5ade
tree2a10d4b7611bc39a434d5ae2500aa2f783e1510e
parent5e203e157b0188a57ce43b876ab4d1877f1dfea1

cases: remove old incremental case system

We now run incremental tests with `tools/incr-check.zig` (with the actual cases being in `test/incremental/`).

3 files changed, 68 insertions(+), 277 deletions(-)

test/compile_errors.zig+4-2
...@@ -4,8 +4,7 @@ const Cases = @import("src/Cases.zig");...@@ -4,8 +4,7 @@ const Cases = @import("src/Cases.zig");
44
5pub fn addCases(ctx: *Cases, b: *std.Build) !void {5pub fn addCases(ctx: *Cases, b: *std.Build) !void {
6 {6 {
7 const case = ctx.obj("multiline error messages", b.graph.host);7 const case = ctx.obj("multiline error message", b.graph.host);
8
9 case.addError(8 case.addError(
10 \\comptime {9 \\comptime {
11 \\ @compileError("hello\nworld");10 \\ @compileError("hello\nworld");
...@@ -14,7 +13,10 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -14,7 +13,10 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
14 \\:2:5: error: hello13 \\:2:5: error: hello
15 \\ world14 \\ world
16 });15 });
16 }
1717
18 {
19 const case = ctx.obj("multiline error message with trailing newline", b.graph.host);
18 case.addError(20 case.addError(
19 \\comptime {21 \\comptime {
20 \\ @compileError(22 \\ @compileError(
test/nvptx.zig+3-2
...@@ -91,9 +91,10 @@ fn addPtx(ctx: *Cases, target: std.Build.ResolvedTarget, name: []const u8) *Case...@@ -91,9 +91,10 @@ fn addPtx(ctx: *Cases, target: std.Build.ResolvedTarget, name: []const u8) *Case
91 ctx.cases.append(.{91 ctx.cases.append(.{
92 .name = name,92 .name = name,
93 .target = target,93 .target = target,
94 .updates = std.ArrayList(Cases.Update).init(ctx.cases.allocator),94 .files = .init(ctx.arena),
95 .case = null,
95 .output_mode = .Obj,96 .output_mode = .Obj,
96 .deps = std.ArrayList(Cases.DepModule).init(ctx.cases.allocator),97 .deps = .init(ctx.arena),
97 .link_libc = false,98 .link_libc = false,
98 .emit_bin = false,99 .emit_bin = false,
99 .backend = .llvm,100 .backend = .llvm,
test/src/Cases.zig+61-273
...@@ -2,50 +2,11 @@ gpa: Allocator,...@@ -2,50 +2,11 @@ gpa: Allocator,
2arena: Allocator,2arena: Allocator,
3cases: std.ArrayList(Case),3cases: std.ArrayList(Case),
4translate: std.ArrayList(Translate),4translate: std.ArrayList(Translate),
5incremental_cases: std.ArrayList(IncrementalCase),
65
7pub const IncrementalCase = struct {6pub const IncrementalCase = struct {
8 base_path: []const u8,7 base_path: []const u8,
9};8};
109
11pub const Update = struct {
12 /// The input to the current update. We simulate an incremental update
13 /// with the file's contents changed to this value each update.
14 ///
15 /// This value can change entirely between updates, which would be akin
16 /// to deleting the source file and creating a new one from scratch; or
17 /// you can keep it mostly consistent, with small changes, testing the
18 /// effects of the incremental compilation.
19 files: std.ArrayList(File),
20 /// This is a description of what happens with the update, for debugging
21 /// purposes.
22 name: []const u8,
23 case: union(enum) {
24 /// Check that it compiles with no errors.
25 Compile: void,
26 /// Check the main binary output file against an expected set of bytes.
27 /// This is most useful with, for example, `-ofmt=c`.
28 CompareObjectFile: []const u8,
29 /// An error update attempts to compile bad code, and ensures that it
30 /// fails to compile, and for the expected reasons.
31 /// A slice containing the expected stderr template, which
32 /// gets some values substituted.
33 Error: []const []const u8,
34 /// An execution update compiles and runs the input, testing the
35 /// stdout against the expected results
36 /// This is a slice containing the expected message.
37 Execution: []const u8,
38 /// A header update compiles the input with the equivalent of
39 /// `-femit-h` and tests the produced header against the
40 /// expected result.
41 Header: []const u8,
42 },
43
44 pub fn addSourceFile(update: *Update, name: []const u8, src: [:0]const u8) void {
45 update.files.append(.{ .path = name, .src = src }) catch @panic("out of memory");
46 }
47};
48
49pub const File = struct {10pub const File = struct {
50 src: [:0]const u8,11 src: [:0]const u8,
51 path: []const u8,12 path: []const u8,
...@@ -67,9 +28,6 @@ pub const CFrontend = enum {...@@ -67,9 +28,6 @@ pub const CFrontend = enum {
67 aro,28 aro,
68};29};
6930
70/// A `Case` consists of a list of `Update`. The same `Compilation` is used for each
71/// update, so each update's source is treated as a single file being
72/// updated by the test harness and incrementally compiled.
73pub const Case = struct {31pub const Case = struct {
74 /// The name of the test case. This is shown if a test fails, and32 /// The name of the test case. This is shown if a test fails, and
75 /// otherwise ignored.33 /// otherwise ignored.
...@@ -81,7 +39,29 @@ pub const Case = struct {...@@ -81,7 +39,29 @@ pub const Case = struct {
81 /// to Executable.39 /// to Executable.
82 output_mode: std.builtin.OutputMode,40 output_mode: std.builtin.OutputMode,
83 optimize_mode: std.builtin.OptimizeMode = .Debug,41 optimize_mode: std.builtin.OptimizeMode = .Debug,
84 updates: std.ArrayList(Update),42
43 files: std.ArrayList(File),
44 case: ?union(enum) {
45 /// Check that it compiles with no errors.
46 Compile: void,
47 /// Check the main binary output file against an expected set of bytes.
48 /// This is most useful with, for example, `-ofmt=c`.
49 CompareObjectFile: []const u8,
50 /// An error update attempts to compile bad code, and ensures that it
51 /// fails to compile, and for the expected reasons.
52 /// A slice containing the expected stderr template, which
53 /// gets some values substituted.
54 Error: []const []const u8,
55 /// An execution update compiles and runs the input, testing the
56 /// stdout against the expected results
57 /// This is a slice containing the expected message.
58 Execution: []const u8,
59 /// A header update compiles the input with the equivalent of
60 /// `-femit-h` and tests the produced header against the
61 /// expected result.
62 Header: []const u8,
63 },
64
85 emit_bin: bool = true,65 emit_bin: bool = true,
86 emit_h: bool = false,66 emit_h: bool = false,
87 is_test: bool = false,67 is_test: bool = false,
...@@ -99,8 +79,7 @@ pub const Case = struct {...@@ -99,8 +79,7 @@ pub const Case = struct {
99 deps: std.ArrayList(DepModule),79 deps: std.ArrayList(DepModule),
10080
101 pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {81 pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {
102 const update = &case.updates.items[case.updates.items.len - 1];82 case.files.append(.{ .path = name, .src = src }) catch @panic("OOM");
103 update.files.append(.{ .path = name, .src = src }) catch @panic("OOM");
104 }83 }
10584
106 pub fn addDepModule(case: *Case, name: []const u8, path: []const u8) void {85 pub fn addDepModule(case: *Case, name: []const u8, path: []const u8) void {
...@@ -113,46 +92,28 @@ pub const Case = struct {...@@ -113,46 +92,28 @@ pub const Case = struct {
113 /// Adds a subcase in which the module is updated with `src`, compiled,92 /// Adds a subcase in which the module is updated with `src`, compiled,
114 /// run, and the output is tested against `result`.93 /// run, and the output is tested against `result`.
115 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void {94 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void {
116 self.updates.append(.{95 assert(self.case == null);
117 .files = std.ArrayList(File).init(self.updates.allocator),96 self.case = .{ .Execution = result };
118 .name = "update",97 self.addSourceFile("tmp.zig", src);
119 .case = .{ .Execution = result },
120 }) catch @panic("out of memory");
121 addSourceFile(self, "tmp.zig", src);
122 }
123
124 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
125 return self.addErrorNamed("update", src, errors);
126 }98 }
12799
128 /// Adds a subcase in which the module is updated with `src`, which100 /// Adds a subcase in which the module is updated with `src`, which
129 /// should contain invalid input, and ensures that compilation fails101 /// should contain invalid input, and ensures that compilation fails
130 /// for the expected reasons, given in sequential order in `errors` in102 /// for the expected reasons, given in sequential order in `errors` in
131 /// the form `:line:column: error: message`.103 /// the form `:line:column: error: message`.
132 pub fn addErrorNamed(104 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
133 self: *Case,
134 name: []const u8,
135 src: [:0]const u8,
136 errors: []const []const u8,
137 ) void {
138 assert(errors.len != 0);105 assert(errors.len != 0);
139 self.updates.append(.{106 assert(self.case == null);
140 .files = std.ArrayList(File).init(self.updates.allocator),107 self.case = .{ .Error = errors };
141 .name = name,108 self.addSourceFile("tmp.zig", src);
142 .case = .{ .Error = errors },
143 }) catch @panic("out of memory");
144 addSourceFile(self, "tmp.zig", src);
145 }109 }
146110
147 /// Adds a subcase in which the module is updated with `src`, and111 /// Adds a subcase in which the module is updated with `src`, and
148 /// asserts that it compiles without issue112 /// asserts that it compiles without issue
149 pub fn addCompile(self: *Case, src: [:0]const u8) void {113 pub fn addCompile(self: *Case, src: [:0]const u8) void {
150 self.updates.append(.{114 assert(self.case == null);
151 .files = std.ArrayList(File).init(self.updates.allocator),115 self.case = .Compile;
152 .name = "compile",116 self.addSourceFile("tmp.zig", src);
153 .case = .{ .Compile = {} },
154 }) catch @panic("out of memory");
155 addSourceFile(self, "tmp.zig", src);
156 }117 }
157};118};
158119
...@@ -180,10 +141,11 @@ pub fn addExe(...@@ -180,10 +141,11 @@ pub fn addExe(
180 name: []const u8,141 name: []const u8,
181 target: std.Build.ResolvedTarget,142 target: std.Build.ResolvedTarget,
182) *Case {143) *Case {
183 ctx.cases.append(Case{144 ctx.cases.append(.{
184 .name = name,145 .name = name,
185 .target = target,146 .target = target,
186 .updates = std.ArrayList(Update).init(ctx.cases.allocator),147 .files = .init(ctx.arena),
148 .case = null,
187 .output_mode = .Exe,149 .output_mode = .Exe,
188 .deps = std.ArrayList(DepModule).init(ctx.arena),150 .deps = std.ArrayList(DepModule).init(ctx.arena),
189 }) catch @panic("out of memory");151 }) catch @panic("out of memory");
...@@ -198,10 +160,11 @@ pub fn exe(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Cas...@@ -198,10 +160,11 @@ pub fn exe(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Cas
198pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.Query, b: *std.Build) *Case {160pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.Query, b: *std.Build) *Case {
199 var adjusted_query = target_query;161 var adjusted_query = target_query;
200 adjusted_query.ofmt = .c;162 adjusted_query.ofmt = .c;
201 ctx.cases.append(Case{163 ctx.cases.append(.{
202 .name = name,164 .name = name,
203 .target = b.resolveTargetQuery(adjusted_query),165 .target = b.resolveTargetQuery(adjusted_query),
204 .updates = std.ArrayList(Update).init(ctx.cases.allocator),166 .files = .init(ctx.arena),
167 .case = null,
205 .output_mode = .Exe,168 .output_mode = .Exe,
206 .deps = std.ArrayList(DepModule).init(ctx.arena),169 .deps = std.ArrayList(DepModule).init(ctx.arena),
207 .link_libc = true,170 .link_libc = true,
...@@ -210,10 +173,11 @@ pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target....@@ -210,10 +173,11 @@ pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.
210}173}
211174
212pub fn addObjLlvm(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {175pub fn addObjLlvm(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
213 ctx.cases.append(Case{176 ctx.cases.append(.{
214 .name = name,177 .name = name,
215 .target = target,178 .target = target,
216 .updates = std.ArrayList(Update).init(ctx.cases.allocator),179 .files = .init(ctx.arena),
180 .case = null,
217 .output_mode = .Obj,181 .output_mode = .Obj,
218 .deps = std.ArrayList(DepModule).init(ctx.arena),182 .deps = std.ArrayList(DepModule).init(ctx.arena),
219 .backend = .llvm,183 .backend = .llvm,
...@@ -226,10 +190,11 @@ pub fn addObj(...@@ -226,10 +190,11 @@ pub fn addObj(
226 name: []const u8,190 name: []const u8,
227 target: std.Build.ResolvedTarget,191 target: std.Build.ResolvedTarget,
228) *Case {192) *Case {
229 ctx.cases.append(Case{193 ctx.cases.append(.{
230 .name = name,194 .name = name,
231 .target = target,195 .target = target,
232 .updates = std.ArrayList(Update).init(ctx.cases.allocator),196 .files = .init(ctx.arena),
197 .case = null,
233 .output_mode = .Obj,198 .output_mode = .Obj,
234 .deps = std.ArrayList(DepModule).init(ctx.arena),199 .deps = std.ArrayList(DepModule).init(ctx.arena),
235 }) catch @panic("out of memory");200 }) catch @panic("out of memory");
...@@ -241,10 +206,11 @@ pub fn addTest(...@@ -241,10 +206,11 @@ pub fn addTest(
241 name: []const u8,206 name: []const u8,
242 target: std.Build.ResolvedTarget,207 target: std.Build.ResolvedTarget,
243) *Case {208) *Case {
244 ctx.cases.append(Case{209 ctx.cases.append(.{
245 .name = name,210 .name = name,
246 .target = target,211 .target = target,
247 .updates = std.ArrayList(Update).init(ctx.cases.allocator),212 .files = .init(ctx.arena),
213 .case = null,
248 .output_mode = .Exe,214 .output_mode = .Exe,
249 .is_test = true,215 .is_test = true,
250 .deps = std.ArrayList(DepModule).init(ctx.arena),216 .deps = std.ArrayList(DepModule).init(ctx.arena),
...@@ -266,10 +232,11 @@ pub fn objZIR(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *...@@ -266,10 +232,11 @@ pub fn objZIR(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *
266pub fn addC(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {232pub fn addC(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
267 var target_adjusted = target;233 var target_adjusted = target;
268 target_adjusted.ofmt = std.Target.ObjectFormat.c;234 target_adjusted.ofmt = std.Target.ObjectFormat.c;
269 ctx.cases.append(Case{235 ctx.cases.append(.{
270 .name = name,236 .name = name,
271 .target = target_adjusted,237 .target = target_adjusted,
272 .updates = std.ArrayList(Update).init(ctx.cases.allocator),238 .files = .init(ctx.arena),
239 .case = null,
273 .output_mode = .Obj,240 .output_mode = .Obj,
274 .deps = std.ArrayList(DepModule).init(ctx.arena),241 .deps = std.ArrayList(DepModule).init(ctx.arena),
275 }) catch @panic("out of memory");242 }) catch @panic("out of memory");
...@@ -352,9 +319,7 @@ pub fn addCompile(...@@ -352,9 +319,7 @@ pub fn addCompile(
352 ctx.addObj(name, target).addCompile(src);319 ctx.addObj(name, target).addCompile(src);
353}320}
354321
355/// Adds a test for each file in the provided directory.322/// Adds a test for each file in the provided directory. Recurses nested directories.
356/// Testing strategy (TestStrategy) is inferred automatically from filenames.
357/// Recurses nested directories.
358///323///
359/// Each file should include a test manifest as a contiguous block of comments at324/// Each file should include a test manifest as a contiguous block of comments at
360/// the end of the file. The first line should be the test type, followed by a set of325/// the end of the file. The first line should be the test type, followed by a set of
...@@ -379,29 +344,18 @@ fn addFromDirInner(...@@ -379,29 +344,18 @@ fn addFromDirInner(
379 b: *std.Build,344 b: *std.Build,
380) !void {345) !void {
381 var it = try iterable_dir.walk(ctx.arena);346 var it = try iterable_dir.walk(ctx.arena);
382 var filenames = std.ArrayList([]const u8).init(ctx.arena);347 var filenames: std.ArrayListUnmanaged([]const u8) = .empty;
383348
384 while (try it.next()) |entry| {349 while (try it.next()) |entry| {
385 if (entry.kind != .file) continue;350 if (entry.kind != .file) continue;
386351
387 // Ignore stuff such as .swp files352 // Ignore stuff such as .swp files
388 if (!knownFileExtension(entry.basename)) continue;353 if (!knownFileExtension(entry.basename)) continue;
389 try filenames.append(try ctx.arena.dupe(u8, entry.path));354 try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path));
390 }355 }
391356
392 // Sort filenames, so that incremental tests are contiguous and in-order357 for (filenames.items) |filename| {
393 sortTestFilenames(filenames.items);
394
395 var test_it = TestIterator{ .filenames = filenames.items };
396 while (test_it.next()) |maybe_batch| {
397 const batch = maybe_batch orelse break;
398 const strategy: TestStrategy = if (batch.len > 1) .incremental else .independent;
399 const filename = batch[0];
400 current_file.* = filename;358 current_file.* = filename;
401 if (strategy == .incremental) {
402 try ctx.incremental_cases.append(.{ .base_path = filename });
403 continue;
404 }
405359
406 const max_file_size = 10 * 1024 * 1024;360 const max_file_size = 10 * 1024 * 1024;
407 const src = try iterable_dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0);361 const src = try iterable_dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0);
...@@ -482,7 +436,8 @@ fn addFromDirInner(...@@ -482,7 +436,8 @@ fn addFromDirInner(
482 .name = std.fs.path.stem(filename),436 .name = std.fs.path.stem(filename),
483 .import_path = std.fs.path.dirname(filename),437 .import_path = std.fs.path.dirname(filename),
484 .backend = backend,438 .backend = backend,
485 .updates = std.ArrayList(Cases.Update).init(ctx.cases.allocator),439 .files = .init(ctx.arena),
440 .case = null,
486 .emit_bin = emit_bin,441 .emit_bin = emit_bin,
487 .is_test = is_test,442 .is_test = is_test,
488 .output_mode = output_mode,443 .output_mode = output_mode,
...@@ -516,10 +471,6 @@ fn addFromDirInner(...@@ -516,10 +471,6 @@ fn addFromDirInner(
516 .cli => @panic("TODO cli tests"),471 .cli => @panic("TODO cli tests"),
517 }472 }
518 }473 }
519 } else |err| {
520 // make sure the current file is set to the file that produced an error
521 current_file.* = test_it.currentFilename();
522 return err;
523 }474 }
524}475}
525476
...@@ -528,7 +479,6 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases {...@@ -528,7 +479,6 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases {
528 .gpa = gpa,479 .gpa = gpa,
529 .cases = std.ArrayList(Case).init(gpa),480 .cases = std.ArrayList(Case).init(gpa),
530 .translate = std.ArrayList(Translate).init(gpa),481 .translate = std.ArrayList(Translate).init(gpa),
531 .incremental_cases = std.ArrayList(IncrementalCase).init(gpa),
532 .arena = arena,482 .arena = arena,
533 };483 };
534}484}
...@@ -633,26 +583,7 @@ pub fn lowerToBuildSteps(...@@ -633,26 +583,7 @@ pub fn lowerToBuildSteps(
633 std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)});583 std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)});
634 const cases_dir_path = b.build_root.join(b.allocator, &.{ "test", "cases" }) catch @panic("OOM");584 const cases_dir_path = b.build_root.join(b.allocator, &.{ "test", "cases" }) catch @panic("OOM");
635585
636 for (self.incremental_cases.items) |incr_case| {
637 if (true) {
638 // TODO: incremental tests are disabled for now, as incremental compilation bugs were
639 // getting in the way of practical improvements to the compiler, and incremental
640 // compilation is not currently used. They should be re-enabled once incremental
641 // compilation is in a happier state.
642 continue;
643 }
644 // TODO: the logic for running these was bad, so I've ripped it out. Rewrite this
645 // in a way that actually spawns the compiler, communicating with it over the
646 // compiler server protocol.
647 _ = incr_case;
648 @panic("TODO implement incremental test case executor");
649 }
650
651 for (self.cases.items) |case| {586 for (self.cases.items) |case| {
652 if (case.updates.items.len != 1) continue; // handled with incremental_cases above
653 assert(case.updates.items.len == 1);
654 const update = case.updates.items[0];
655
656 for (test_filters) |test_filter| {587 for (test_filters) |test_filter| {
657 if (std.mem.indexOf(u8, case.name, test_filter)) |_| break;588 if (std.mem.indexOf(u8, case.name, test_filter)) |_| break;
658 } else if (test_filters.len > 0) continue;589 } else if (test_filters.len > 0) continue;
...@@ -668,10 +599,10 @@ pub fn lowerToBuildSteps(...@@ -668,10 +599,10 @@ pub fn lowerToBuildSteps(
668 const writefiles = b.addWriteFiles();599 const writefiles = b.addWriteFiles();
669 var file_sources = std.StringHashMap(std.Build.LazyPath).init(b.allocator);600 var file_sources = std.StringHashMap(std.Build.LazyPath).init(b.allocator);
670 defer file_sources.deinit();601 defer file_sources.deinit();
671 const first_file = update.files.items[0];602 const first_file = case.files.items[0];
672 const root_source_file = writefiles.add(first_file.path, first_file.src);603 const root_source_file = writefiles.add(first_file.path, first_file.src);
673 file_sources.put(first_file.path, root_source_file) catch @panic("OOM");604 file_sources.put(first_file.path, root_source_file) catch @panic("OOM");
674 for (update.files.items[1..]) |file| {605 for (case.files.items[1..]) |file| {
675 file_sources.put(file.path, writefiles.add(file.path, file.src)) catch @panic("OOM");606 file_sources.put(file.path, writefiles.add(file.path, file.src)) catch @panic("OOM");
676 }607 }
677608
...@@ -730,7 +661,7 @@ pub fn lowerToBuildSteps(...@@ -730,7 +661,7 @@ pub fn lowerToBuildSteps(
730 },661 },
731 }662 }
732663
733 switch (update.case) {664 switch (case.case.?) {
734 .Compile => {665 .Compile => {
735 // Force the binary to be emitted if requested.666 // Force the binary to be emitted if requested.
736 if (case.emit_bin) {667 if (case.emit_bin) {
...@@ -787,149 +718,6 @@ pub fn lowerToBuildSteps(...@@ -787,149 +718,6 @@ pub fn lowerToBuildSteps(
787 }718 }
788}719}
789720
790/// Sort test filenames in-place, so that incremental test cases ("foo.0.zig",
791/// "foo.1.zig", etc.) are contiguous and appear in numerical order.
792fn sortTestFilenames(filenames: [][]const u8) void {
793 const Context = struct {
794 pub fn lessThan(_: @This(), a: []const u8, b: []const u8) bool {
795 const a_parts = getTestFileNameParts(a);
796 const b_parts = getTestFileNameParts(b);
797
798 // Sort "<base_name>.X.<file_ext>" based on "<base_name>" and "<file_ext>" first
799 return switch (std.mem.order(u8, a_parts.base_name, b_parts.base_name)) {
800 .lt => true,
801 .gt => false,
802 .eq => switch (std.mem.order(u8, a_parts.file_ext, b_parts.file_ext)) {
803 .lt => true,
804 .gt => false,
805 .eq => {
806 // a and b differ only in their ".X" part
807
808 // Sort "<base_name>.<file_ext>" before any "<base_name>.X.<file_ext>"
809 if (a_parts.test_index) |a_index| {
810 if (b_parts.test_index) |b_index| {
811 // Make sure that incremental tests appear in linear order
812 return a_index < b_index;
813 } else {
814 return false;
815 }
816 } else {
817 return b_parts.test_index != null;
818 }
819 },
820 },
821 };
822 }
823 };
824 std.mem.sort([]const u8, filenames, Context{}, Context.lessThan);
825}
826
827/// Iterates a set of filenames extracting batches that are either incremental
828/// ("foo.0.zig", "foo.1.zig", etc.) or independent ("foo.zig", "bar.zig", etc.).
829/// Assumes filenames are sorted.
830const TestIterator = struct {
831 start: usize = 0,
832 end: usize = 0,
833 filenames: []const []const u8,
834 /// reset on each call to `next`
835 index: usize = 0,
836
837 const Error = error{InvalidIncrementalTestIndex};
838
839 fn next(it: *TestIterator) Error!?[]const []const u8 {
840 try it.nextInner();
841 if (it.start == it.end) return null;
842 return it.filenames[it.start..it.end];
843 }
844
845 fn nextInner(it: *TestIterator) Error!void {
846 it.start = it.end;
847 if (it.end == it.filenames.len) return;
848 if (it.end + 1 == it.filenames.len) {
849 it.end += 1;
850 return;
851 }
852
853 const remaining = it.filenames[it.end..];
854 it.index = 0;
855 while (it.index < remaining.len - 1) : (it.index += 1) {
856 // First, check if this file is part of an incremental update sequence
857 // Split filename into "<base_name>.<index>.<file_ext>"
858 const prev_parts = getTestFileNameParts(remaining[it.index]);
859 const new_parts = getTestFileNameParts(remaining[it.index + 1]);
860
861 // If base_name and file_ext match, these files are in the same test sequence
862 // and the new one should be the incremented version of the previous test
863 if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and
864 std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext))
865 {
866 // This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1
867 if (prev_parts.test_index == null)
868 return error.InvalidIncrementalTestIndex;
869 if (new_parts.test_index == null)
870 return error.InvalidIncrementalTestIndex;
871 if (new_parts.test_index.? != prev_parts.test_index.? + 1)
872 return error.InvalidIncrementalTestIndex;
873 } else {
874 // This is not the same test sequence, so the new file must be the first file
875 // in a new sequence ("*.0.zig") or an independent test file ("*.zig")
876 if (new_parts.test_index != null and new_parts.test_index.? != 0)
877 return error.InvalidIncrementalTestIndex;
878
879 it.end += it.index + 1;
880 break;
881 }
882 } else {
883 it.end += remaining.len;
884 }
885 }
886
887 /// In the event of an `error.InvalidIncrementalTestIndex`, this function can
888 /// be used to find the current filename that was being processed.
889 /// Asserts the iterator hasn't reached the end.
890 fn currentFilename(it: TestIterator) []const u8 {
891 assert(it.end != it.filenames.len);
892 const remaining = it.filenames[it.end..];
893 return remaining[it.index + 1];
894 }
895};
896
897/// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns
898/// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or
899/// cannot be parsed as a decimal number, it is treated as part of <filename>
900fn getTestFileNameParts(name: []const u8) struct {
901 base_name: []const u8,
902 file_ext: []const u8,
903 test_index: ?usize,
904} {
905 const file_ext = std.fs.path.extension(name);
906 const trimmed = name[0 .. name.len - file_ext.len]; // Trim off ".<ext>"
907 const maybe_index = std.fs.path.extension(trimmed); // Extract ".X"
908
909 // Attempt to parse index
910 const index: ?usize = if (maybe_index.len > 0)
911 std.fmt.parseInt(usize, maybe_index[1..], 10) catch null
912 else
913 null;
914
915 // Adjust "<filename>" extent based on parsing success
916 const base_name_end = trimmed.len - if (index != null) maybe_index.len else 0;
917 return .{
918 .base_name = name[0..base_name_end],
919 .file_ext = if (file_ext.len > 0) file_ext[1..] else file_ext,
920 .test_index = index,
921 };
922}
923
924const TestStrategy = enum {
925 /// Execute tests as independent compilations, unless they are explicitly
926 /// incremental ("foo.0.zig", "foo.1.zig", etc.)
927 independent,
928 /// Execute all tests as incremental updates to a single compilation. Explicitly
929 /// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order
930 incremental,
931};
932
933/// Default config values for known test manifest key-value pairings.721/// Default config values for known test manifest key-value pairings.
934/// Currently handled defaults are:722/// Currently handled defaults are:
935/// * backend723/// * backend