authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-05 14:13:26+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-02-23 00:52:50+00:00
loga3b3a33d7a365bde9cb71cd5a5bf6663754b5ade
tree2a10d4b7611bc39a434d5ae2500aa2f783e1510e
parent5e203e157b0188a57ce43b876ab4d1877f1dfea1

cases: remove old incremental case system

We now run incremental tests with `tools/incr-check.zig` (with the actual cases being in `test/incremental/`).

3 files changed, 68 insertions(+), 277 deletions(-)

test/compile_errors.zig+4-2
......@@ -4,8 +4,7 @@ const Cases = @import("src/Cases.zig");
44
55pub fn addCases(ctx: *Cases, b: *std.Build) !void {
66 {
7 const case = ctx.obj("multiline error messages", b.graph.host);
8
7 const case = ctx.obj("multiline error message", b.graph.host);
98 case.addError(
109 \\comptime {
1110 \\ @compileError("hello\nworld");
......@@ -14,7 +13,10 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
1413 \\:2:5: error: hello
1514 \\ world
1615 });
16 }
1717
18 {
19 const case = ctx.obj("multiline error message with trailing newline", b.graph.host);
1820 case.addError(
1921 \\comptime {
2022 \\ @compileError(
test/nvptx.zig+3-2
......@@ -91,9 +91,10 @@ fn addPtx(ctx: *Cases, target: std.Build.ResolvedTarget, name: []const u8) *Case
9191 ctx.cases.append(.{
9292 .name = name,
9393 .target = target,
94 .updates = std.ArrayList(Cases.Update).init(ctx.cases.allocator),
94 .files = .init(ctx.arena),
95 .case = null,
9596 .output_mode = .Obj,
96 .deps = std.ArrayList(Cases.DepModule).init(ctx.cases.allocator),
97 .deps = .init(ctx.arena),
9798 .link_libc = false,
9899 .emit_bin = false,
99100 .backend = .llvm,
test/src/Cases.zig+61-273
......@@ -2,50 +2,11 @@ gpa: Allocator,
22arena: Allocator,
33cases: std.ArrayList(Case),
44translate: std.ArrayList(Translate),
5incremental_cases: std.ArrayList(IncrementalCase),
65
76pub const IncrementalCase = struct {
87 base_path: []const u8,
98};
109
11pub const Update = struct {
12 /// The input to the current update. We simulate an incremental update
13 /// with the file's contents changed to this value each update.
14 ///
15 /// This value can change entirely between updates, which would be akin
16 /// to deleting the source file and creating a new one from scratch; or
17 /// you can keep it mostly consistent, with small changes, testing the
18 /// effects of the incremental compilation.
19 files: std.ArrayList(File),
20 /// This is a description of what happens with the update, for debugging
21 /// purposes.
22 name: []const u8,
23 case: union(enum) {
24 /// Check that it compiles with no errors.
25 Compile: void,
26 /// Check the main binary output file against an expected set of bytes.
27 /// This is most useful with, for example, `-ofmt=c`.
28 CompareObjectFile: []const u8,
29 /// An error update attempts to compile bad code, and ensures that it
30 /// fails to compile, and for the expected reasons.
31 /// A slice containing the expected stderr template, which
32 /// gets some values substituted.
33 Error: []const []const u8,
34 /// An execution update compiles and runs the input, testing the
35 /// stdout against the expected results
36 /// This is a slice containing the expected message.
37 Execution: []const u8,
38 /// A header update compiles the input with the equivalent of
39 /// `-femit-h` and tests the produced header against the
40 /// expected result.
41 Header: []const u8,
42 },
43
44 pub fn addSourceFile(update: *Update, name: []const u8, src: [:0]const u8) void {
45 update.files.append(.{ .path = name, .src = src }) catch @panic("out of memory");
46 }
47};
48
4910pub const File = struct {
5011 src: [:0]const u8,
5112 path: []const u8,
......@@ -67,9 +28,6 @@ pub const CFrontend = enum {
6728 aro,
6829};
6930
70/// A `Case` consists of a list of `Update`. The same `Compilation` is used for each
71/// update, so each update's source is treated as a single file being
72/// updated by the test harness and incrementally compiled.
7331pub const Case = struct {
7432 /// The name of the test case. This is shown if a test fails, and
7533 /// otherwise ignored.
......@@ -81,7 +39,29 @@ pub const Case = struct {
8139 /// to Executable.
8240 output_mode: std.builtin.OutputMode,
8341 optimize_mode: std.builtin.OptimizeMode = .Debug,
84 updates: std.ArrayList(Update),
42
43 files: std.ArrayList(File),
44 case: ?union(enum) {
45 /// Check that it compiles with no errors.
46 Compile: void,
47 /// Check the main binary output file against an expected set of bytes.
48 /// This is most useful with, for example, `-ofmt=c`.
49 CompareObjectFile: []const u8,
50 /// An error update attempts to compile bad code, and ensures that it
51 /// fails to compile, and for the expected reasons.
52 /// A slice containing the expected stderr template, which
53 /// gets some values substituted.
54 Error: []const []const u8,
55 /// An execution update compiles and runs the input, testing the
56 /// stdout against the expected results
57 /// This is a slice containing the expected message.
58 Execution: []const u8,
59 /// A header update compiles the input with the equivalent of
60 /// `-femit-h` and tests the produced header against the
61 /// expected result.
62 Header: []const u8,
63 },
64
8565 emit_bin: bool = true,
8666 emit_h: bool = false,
8767 is_test: bool = false,
......@@ -99,8 +79,7 @@ pub const Case = struct {
9979 deps: std.ArrayList(DepModule),
10080
10181 pub fn addSourceFile(case: *Case, name: []const u8, src: [:0]const u8) void {
102 const update = &case.updates.items[case.updates.items.len - 1];
103 update.files.append(.{ .path = name, .src = src }) catch @panic("OOM");
82 case.files.append(.{ .path = name, .src = src }) catch @panic("OOM");
10483 }
10584
10685 pub fn addDepModule(case: *Case, name: []const u8, path: []const u8) void {
......@@ -113,46 +92,28 @@ pub const Case = struct {
11392 /// Adds a subcase in which the module is updated with `src`, compiled,
11493 /// run, and the output is tested against `result`.
11594 pub fn addCompareOutput(self: *Case, src: [:0]const u8, result: []const u8) void {
116 self.updates.append(.{
117 .files = std.ArrayList(File).init(self.updates.allocator),
118 .name = "update",
119 .case = .{ .Execution = result },
120 }) catch @panic("out of memory");
121 addSourceFile(self, "tmp.zig", src);
122 }
123
124 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
125 return self.addErrorNamed("update", src, errors);
95 assert(self.case == null);
96 self.case = .{ .Execution = result };
97 self.addSourceFile("tmp.zig", src);
12698 }
12799
128100 /// Adds a subcase in which the module is updated with `src`, which
129101 /// should contain invalid input, and ensures that compilation fails
130102 /// for the expected reasons, given in sequential order in `errors` in
131103 /// the form `:line:column: error: message`.
132 pub fn addErrorNamed(
133 self: *Case,
134 name: []const u8,
135 src: [:0]const u8,
136 errors: []const []const u8,
137 ) void {
104 pub fn addError(self: *Case, src: [:0]const u8, errors: []const []const u8) void {
138105 assert(errors.len != 0);
139 self.updates.append(.{
140 .files = std.ArrayList(File).init(self.updates.allocator),
141 .name = name,
142 .case = .{ .Error = errors },
143 }) catch @panic("out of memory");
144 addSourceFile(self, "tmp.zig", src);
106 assert(self.case == null);
107 self.case = .{ .Error = errors };
108 self.addSourceFile("tmp.zig", src);
145109 }
146110
147111 /// Adds a subcase in which the module is updated with `src`, and
148112 /// asserts that it compiles without issue
149113 pub fn addCompile(self: *Case, src: [:0]const u8) void {
150 self.updates.append(.{
151 .files = std.ArrayList(File).init(self.updates.allocator),
152 .name = "compile",
153 .case = .{ .Compile = {} },
154 }) catch @panic("out of memory");
155 addSourceFile(self, "tmp.zig", src);
114 assert(self.case == null);
115 self.case = .Compile;
116 self.addSourceFile("tmp.zig", src);
156117 }
157118};
158119
......@@ -180,10 +141,11 @@ pub fn addExe(
180141 name: []const u8,
181142 target: std.Build.ResolvedTarget,
182143) *Case {
183 ctx.cases.append(Case{
144 ctx.cases.append(.{
184145 .name = name,
185146 .target = target,
186 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
147 .files = .init(ctx.arena),
148 .case = null,
187149 .output_mode = .Exe,
188150 .deps = std.ArrayList(DepModule).init(ctx.arena),
189151 }) catch @panic("out of memory");
......@@ -198,10 +160,11 @@ pub fn exe(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Cas
198160pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.Query, b: *std.Build) *Case {
199161 var adjusted_query = target_query;
200162 adjusted_query.ofmt = .c;
201 ctx.cases.append(Case{
163 ctx.cases.append(.{
202164 .name = name,
203165 .target = b.resolveTargetQuery(adjusted_query),
204 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
166 .files = .init(ctx.arena),
167 .case = null,
205168 .output_mode = .Exe,
206169 .deps = std.ArrayList(DepModule).init(ctx.arena),
207170 .link_libc = true,
......@@ -210,10 +173,11 @@ pub fn exeFromCompiledC(ctx: *Cases, name: []const u8, target_query: std.Target.
210173}
211174
212175pub fn addObjLlvm(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
213 ctx.cases.append(Case{
176 ctx.cases.append(.{
214177 .name = name,
215178 .target = target,
216 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
179 .files = .init(ctx.arena),
180 .case = null,
217181 .output_mode = .Obj,
218182 .deps = std.ArrayList(DepModule).init(ctx.arena),
219183 .backend = .llvm,
......@@ -226,10 +190,11 @@ pub fn addObj(
226190 name: []const u8,
227191 target: std.Build.ResolvedTarget,
228192) *Case {
229 ctx.cases.append(Case{
193 ctx.cases.append(.{
230194 .name = name,
231195 .target = target,
232 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
196 .files = .init(ctx.arena),
197 .case = null,
233198 .output_mode = .Obj,
234199 .deps = std.ArrayList(DepModule).init(ctx.arena),
235200 }) catch @panic("out of memory");
......@@ -241,10 +206,11 @@ pub fn addTest(
241206 name: []const u8,
242207 target: std.Build.ResolvedTarget,
243208) *Case {
244 ctx.cases.append(Case{
209 ctx.cases.append(.{
245210 .name = name,
246211 .target = target,
247 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
212 .files = .init(ctx.arena),
213 .case = null,
248214 .output_mode = .Exe,
249215 .is_test = true,
250216 .deps = std.ArrayList(DepModule).init(ctx.arena),
......@@ -266,10 +232,11 @@ pub fn objZIR(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *
266232pub fn addC(ctx: *Cases, name: []const u8, target: std.Build.ResolvedTarget) *Case {
267233 var target_adjusted = target;
268234 target_adjusted.ofmt = std.Target.ObjectFormat.c;
269 ctx.cases.append(Case{
235 ctx.cases.append(.{
270236 .name = name,
271237 .target = target_adjusted,
272 .updates = std.ArrayList(Update).init(ctx.cases.allocator),
238 .files = .init(ctx.arena),
239 .case = null,
273240 .output_mode = .Obj,
274241 .deps = std.ArrayList(DepModule).init(ctx.arena),
275242 }) catch @panic("out of memory");
......@@ -352,9 +319,7 @@ pub fn addCompile(
352319 ctx.addObj(name, target).addCompile(src);
353320}
354321
355/// Adds a test for each file in the provided directory.
356/// Testing strategy (TestStrategy) is inferred automatically from filenames.
357/// Recurses nested directories.
322/// Adds a test for each file in the provided directory. Recurses nested directories.
358323///
359324/// Each file should include a test manifest as a contiguous block of comments at
360325/// the end of the file. The first line should be the test type, followed by a set of
......@@ -379,29 +344,18 @@ fn addFromDirInner(
379344 b: *std.Build,
380345) !void {
381346 var it = try iterable_dir.walk(ctx.arena);
382 var filenames = std.ArrayList([]const u8).init(ctx.arena);
347 var filenames: std.ArrayListUnmanaged([]const u8) = .empty;
383348
384349 while (try it.next()) |entry| {
385350 if (entry.kind != .file) continue;
386351
387352 // Ignore stuff such as .swp files
388353 if (!knownFileExtension(entry.basename)) continue;
389 try filenames.append(try ctx.arena.dupe(u8, entry.path));
354 try filenames.append(ctx.arena, try ctx.arena.dupe(u8, entry.path));
390355 }
391356
392 // Sort filenames, so that incremental tests are contiguous and in-order
393 sortTestFilenames(filenames.items);
394
395 var test_it = TestIterator{ .filenames = filenames.items };
396 while (test_it.next()) |maybe_batch| {
397 const batch = maybe_batch orelse break;
398 const strategy: TestStrategy = if (batch.len > 1) .incremental else .independent;
399 const filename = batch[0];
357 for (filenames.items) |filename| {
400358 current_file.* = filename;
401 if (strategy == .incremental) {
402 try ctx.incremental_cases.append(.{ .base_path = filename });
403 continue;
404 }
405359
406360 const max_file_size = 10 * 1024 * 1024;
407361 const src = try iterable_dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0);
......@@ -482,7 +436,8 @@ fn addFromDirInner(
482436 .name = std.fs.path.stem(filename),
483437 .import_path = std.fs.path.dirname(filename),
484438 .backend = backend,
485 .updates = std.ArrayList(Cases.Update).init(ctx.cases.allocator),
439 .files = .init(ctx.arena),
440 .case = null,
486441 .emit_bin = emit_bin,
487442 .is_test = is_test,
488443 .output_mode = output_mode,
......@@ -516,10 +471,6 @@ fn addFromDirInner(
516471 .cli => @panic("TODO cli tests"),
517472 }
518473 }
519 } else |err| {
520 // make sure the current file is set to the file that produced an error
521 current_file.* = test_it.currentFilename();
522 return err;
523474 }
524475}
525476
......@@ -528,7 +479,6 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases {
528479 .gpa = gpa,
529480 .cases = std.ArrayList(Case).init(gpa),
530481 .translate = std.ArrayList(Translate).init(gpa),
531 .incremental_cases = std.ArrayList(IncrementalCase).init(gpa),
532482 .arena = arena,
533483 };
534484}
......@@ -633,26 +583,7 @@ pub fn lowerToBuildSteps(
633583 std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)});
634584 const cases_dir_path = b.build_root.join(b.allocator, &.{ "test", "cases" }) catch @panic("OOM");
635585
636 for (self.incremental_cases.items) |incr_case| {
637 if (true) {
638 // TODO: incremental tests are disabled for now, as incremental compilation bugs were
639 // getting in the way of practical improvements to the compiler, and incremental
640 // compilation is not currently used. They should be re-enabled once incremental
641 // compilation is in a happier state.
642 continue;
643 }
644 // TODO: the logic for running these was bad, so I've ripped it out. Rewrite this
645 // in a way that actually spawns the compiler, communicating with it over the
646 // compiler server protocol.
647 _ = incr_case;
648 @panic("TODO implement incremental test case executor");
649 }
650
651586 for (self.cases.items) |case| {
652 if (case.updates.items.len != 1) continue; // handled with incremental_cases above
653 assert(case.updates.items.len == 1);
654 const update = case.updates.items[0];
655
656587 for (test_filters) |test_filter| {
657588 if (std.mem.indexOf(u8, case.name, test_filter)) |_| break;
658589 } else if (test_filters.len > 0) continue;
......@@ -668,10 +599,10 @@ pub fn lowerToBuildSteps(
668599 const writefiles = b.addWriteFiles();
669600 var file_sources = std.StringHashMap(std.Build.LazyPath).init(b.allocator);
670601 defer file_sources.deinit();
671 const first_file = update.files.items[0];
602 const first_file = case.files.items[0];
672603 const root_source_file = writefiles.add(first_file.path, first_file.src);
673604 file_sources.put(first_file.path, root_source_file) catch @panic("OOM");
674 for (update.files.items[1..]) |file| {
605 for (case.files.items[1..]) |file| {
675606 file_sources.put(file.path, writefiles.add(file.path, file.src)) catch @panic("OOM");
676607 }
677608
......@@ -730,7 +661,7 @@ pub fn lowerToBuildSteps(
730661 },
731662 }
732663
733 switch (update.case) {
664 switch (case.case.?) {
734665 .Compile => {
735666 // Force the binary to be emitted if requested.
736667 if (case.emit_bin) {
......@@ -787,149 +718,6 @@ pub fn lowerToBuildSteps(
787718 }
788719}
789720
790/// Sort test filenames in-place, so that incremental test cases ("foo.0.zig",
791/// "foo.1.zig", etc.) are contiguous and appear in numerical order.
792fn sortTestFilenames(filenames: [][]const u8) void {
793 const Context = struct {
794 pub fn lessThan(_: @This(), a: []const u8, b: []const u8) bool {
795 const a_parts = getTestFileNameParts(a);
796 const b_parts = getTestFileNameParts(b);
797
798 // Sort "<base_name>.X.<file_ext>" based on "<base_name>" and "<file_ext>" first
799 return switch (std.mem.order(u8, a_parts.base_name, b_parts.base_name)) {
800 .lt => true,
801 .gt => false,
802 .eq => switch (std.mem.order(u8, a_parts.file_ext, b_parts.file_ext)) {
803 .lt => true,
804 .gt => false,
805 .eq => {
806 // a and b differ only in their ".X" part
807
808 // Sort "<base_name>.<file_ext>" before any "<base_name>.X.<file_ext>"
809 if (a_parts.test_index) |a_index| {
810 if (b_parts.test_index) |b_index| {
811 // Make sure that incremental tests appear in linear order
812 return a_index < b_index;
813 } else {
814 return false;
815 }
816 } else {
817 return b_parts.test_index != null;
818 }
819 },
820 },
821 };
822 }
823 };
824 std.mem.sort([]const u8, filenames, Context{}, Context.lessThan);
825}
826
827/// Iterates a set of filenames extracting batches that are either incremental
828/// ("foo.0.zig", "foo.1.zig", etc.) or independent ("foo.zig", "bar.zig", etc.).
829/// Assumes filenames are sorted.
830const TestIterator = struct {
831 start: usize = 0,
832 end: usize = 0,
833 filenames: []const []const u8,
834 /// reset on each call to `next`
835 index: usize = 0,
836
837 const Error = error{InvalidIncrementalTestIndex};
838
839 fn next(it: *TestIterator) Error!?[]const []const u8 {
840 try it.nextInner();
841 if (it.start == it.end) return null;
842 return it.filenames[it.start..it.end];
843 }
844
845 fn nextInner(it: *TestIterator) Error!void {
846 it.start = it.end;
847 if (it.end == it.filenames.len) return;
848 if (it.end + 1 == it.filenames.len) {
849 it.end += 1;
850 return;
851 }
852
853 const remaining = it.filenames[it.end..];
854 it.index = 0;
855 while (it.index < remaining.len - 1) : (it.index += 1) {
856 // First, check if this file is part of an incremental update sequence
857 // Split filename into "<base_name>.<index>.<file_ext>"
858 const prev_parts = getTestFileNameParts(remaining[it.index]);
859 const new_parts = getTestFileNameParts(remaining[it.index + 1]);
860
861 // If base_name and file_ext match, these files are in the same test sequence
862 // and the new one should be the incremented version of the previous test
863 if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and
864 std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext))
865 {
866 // This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1
867 if (prev_parts.test_index == null)
868 return error.InvalidIncrementalTestIndex;
869 if (new_parts.test_index == null)
870 return error.InvalidIncrementalTestIndex;
871 if (new_parts.test_index.? != prev_parts.test_index.? + 1)
872 return error.InvalidIncrementalTestIndex;
873 } else {
874 // This is not the same test sequence, so the new file must be the first file
875 // in a new sequence ("*.0.zig") or an independent test file ("*.zig")
876 if (new_parts.test_index != null and new_parts.test_index.? != 0)
877 return error.InvalidIncrementalTestIndex;
878
879 it.end += it.index + 1;
880 break;
881 }
882 } else {
883 it.end += remaining.len;
884 }
885 }
886
887 /// In the event of an `error.InvalidIncrementalTestIndex`, this function can
888 /// be used to find the current filename that was being processed.
889 /// Asserts the iterator hasn't reached the end.
890 fn currentFilename(it: TestIterator) []const u8 {
891 assert(it.end != it.filenames.len);
892 const remaining = it.filenames[it.end..];
893 return remaining[it.index + 1];
894 }
895};
896
897/// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns
898/// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or
899/// cannot be parsed as a decimal number, it is treated as part of <filename>
900fn getTestFileNameParts(name: []const u8) struct {
901 base_name: []const u8,
902 file_ext: []const u8,
903 test_index: ?usize,
904} {
905 const file_ext = std.fs.path.extension(name);
906 const trimmed = name[0 .. name.len - file_ext.len]; // Trim off ".<ext>"
907 const maybe_index = std.fs.path.extension(trimmed); // Extract ".X"
908
909 // Attempt to parse index
910 const index: ?usize = if (maybe_index.len > 0)
911 std.fmt.parseInt(usize, maybe_index[1..], 10) catch null
912 else
913 null;
914
915 // Adjust "<filename>" extent based on parsing success
916 const base_name_end = trimmed.len - if (index != null) maybe_index.len else 0;
917 return .{
918 .base_name = name[0..base_name_end],
919 .file_ext = if (file_ext.len > 0) file_ext[1..] else file_ext,
920 .test_index = index,
921 };
922}
923
924const TestStrategy = enum {
925 /// Execute tests as independent compilations, unless they are explicitly
926 /// incremental ("foo.0.zig", "foo.1.zig", etc.)
927 independent,
928 /// Execute all tests as incremental updates to a single compilation. Explicitly
929 /// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order
930 incremental,
931};
932
933721/// Default config values for known test manifest key-value pairings.
934722/// Currently handled defaults are:
935723/// * backend