authorgravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-04-26 22:55:11+02:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2022-04-28 18:35:01+02:00
log46db5e2a44b38c92e97a438c4b6a67627cc12d16
tree0e0ea203a5af85654647f5aecee644e35a364e9d
parentbc370311cb76f570debf38f5d822a268f1dace83

test: unroll into multiple cases, provide default parsers

Provide default parsers for obvious config options such as `CrossTarget` or `Backend` (or any enum for that matter). Unroll iterator loops into multiple cases - we need to create a Cartesian product for all possibilities specified in the test manifest.

2 files changed, 98 insertions(+), 70 deletions(-)

src/test.zig+98-69
...@@ -237,10 +237,10 @@ const TestManifest = struct {...@@ -237,10 +237,10 @@ const TestManifest = struct {
237 }237 }
238 };238 };
239239
240 fn ConfigValueIterator(comptime T: type, comptime ParseFn: type) type {240 fn ConfigValueIterator(comptime T: type) type {
241 return struct {241 return struct {
242 inner: std.mem.SplitIterator(u8),242 inner: std.mem.SplitIterator(u8),
243 parse_fn: ParseFn,243 parse_fn: ParseFn(T),
244244
245 fn next(self: *@This()) ?T {245 fn next(self: *@This()) ?T {
246 const next_raw = self.inner.next() orelse return null;246 const next_raw = self.inner.next() orelse return null;
...@@ -320,28 +320,34 @@ const TestManifest = struct {...@@ -320,28 +320,34 @@ const TestManifest = struct {
320 return manifest;320 return manifest;
321 }321 }
322322
323 fn getConfigForKey(323 fn getConfigForKeyCustomParser(
324 self: TestManifest,324 self: TestManifest,
325 key: []const u8,325 key: []const u8,
326 comptime T: type,326 comptime T: type,
327 parse_fn: fn ([]const u8) ?T,327 parse_fn: ParseFn(T),
328 ) ConfigValueIterator(T, @TypeOf(parse_fn)) {328 ) ConfigValueIterator(T) {
329 const delimiter = ",";329 const bytes = self.config_map.get(key) orelse TestManifestConfigDefaults.get(self.@"type", key);
330 var inner: std.mem.SplitIterator(u8) = if (self.config_map.get(key)) |bytes| .{330 return ConfigValueIterator(T){
331 .buffer = bytes,331 .inner = std.mem.split(u8, bytes, ","),
332 .delimiter = delimiter,
333 .index = 0,
334 } else .{
335 .buffer = undefined,
336 .delimiter = delimiter,
337 .index = null,
338 };
339 return ConfigValueIterator(T, @TypeOf(parse_fn)){
340 .inner = inner,
341 .parse_fn = parse_fn,332 .parse_fn = parse_fn,
342 };333 };
343 }334 }
344335
336 fn getConfigForKey(
337 self: TestManifest,
338 key: []const u8,
339 comptime T: type,
340 ) ConfigValueIterator(T) {
341 return self.getConfigForKeyCustomParser(key, T, getDefaultParser(T));
342 }
343
344 fn getConfigForKeyAssertSingle(self: TestManifest, key: []const u8, comptime T: type) T {
345 var it = self.getConfigForKey(key, T);
346 const res = it.next().?;
347 assert(it.next() == null);
348 return res;
349 }
350
345 fn trailing(self: TestManifest) TrailingIterator {351 fn trailing(self: TestManifest) TrailingIterator {
346 return .{352 return .{
347 .inner = std.mem.tokenize(u8, self.trailing_bytes, "\r\n"),353 .inner = std.mem.tokenize(u8, self.trailing_bytes, "\r\n"),
...@@ -356,6 +362,40 @@ const TestManifest = struct {...@@ -356,6 +362,40 @@ const TestManifest = struct {
356 }362 }
357 return out.toOwnedSlice();363 return out.toOwnedSlice();
358 }364 }
365
366 fn ParseFn(comptime T: type) type {
367 return fn ([]const u8) ?T;
368 }
369
370 fn getDefaultParser(comptime T: type) ParseFn(T) {
371 switch (@typeInfo(T)) {
372 .Int => return struct {
373 fn parse(str: []const u8) ?T {
374 return std.fmt.parseInt(T, str, 0) catch null;
375 }
376 }.parse,
377 .Bool => return struct {
378 fn parse(str: []const u8) ?T {
379 const as_int = std.fmt.parseInt(u1, str, 0) catch return null;
380 return as_int > 0;
381 }
382 }.parse,
383 .Enum => return struct {
384 fn parse(str: []const u8) ?T {
385 return std.meta.stringToEnum(T, str);
386 }
387 }.parse,
388 .Struct => if (comptime std.mem.eql(u8, @typeName(T), "CrossTarget")) return struct {
389 fn parse(str: []const u8) ?T {
390 var opts = CrossTarget.ParseOptions{
391 .arch_os_abi = str,
392 };
393 return CrossTarget.parse(opts) catch null;
394 }
395 }.parse else @compileError("no default parser for " ++ @typeName(T)),
396 else => @compileError("no default parser for " ++ @typeName(T)),
397 }
398 }
359};399};
360400
361pub const TestContext = struct {401pub const TestContext = struct {
...@@ -401,10 +441,6 @@ pub const TestContext = struct {...@@ -401,10 +441,6 @@ pub const TestContext = struct {
401 stage1,441 stage1,
402 stage2,442 stage2,
403 llvm,443 llvm,
404
405 fn parse(str: []const u8) ?Backend {
406 return std.meta.stringToEnum(Backend, str);
407 }
408 };444 };
409445
410 /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each446 /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each
...@@ -899,7 +935,7 @@ pub const TestContext = struct {...@@ -899,7 +935,7 @@ pub const TestContext = struct {
899935
900 pub fn addTestCasesFromDir(ctx: *TestContext, dir: std.fs.Dir, strategy: Strategy) void {936 pub fn addTestCasesFromDir(ctx: *TestContext, dir: std.fs.Dir, strategy: Strategy) void {
901 var current_file: []const u8 = "none";937 var current_file: []const u8 = "none";
902 addTestCasesFromDirInner(ctx, dir, strategy, &current_file) catch |err| {938 ctx.addTestCasesFromDirInner(dir, strategy, &current_file) catch |err| {
903 std.debug.panic("test harness failed to process file '{s}': {s}\n", .{939 std.debug.panic("test harness failed to process file '{s}': {s}\n", .{
904 current_file, @errorName(err),940 current_file, @errorName(err),
905 });941 });
...@@ -974,11 +1010,10 @@ pub const TestContext = struct {...@@ -974,11 +1010,10 @@ pub const TestContext = struct {
974 /// that if any errors occur the caller knows it happened during this file.1010 /// that if any errors occur the caller knows it happened during this file.
975 current_file: *[]const u8,1011 current_file: *[]const u8,
976 ) !void {1012 ) !void {
977 var opt_case: ?*Case = null;1013 var cases = std.ArrayList(*Case).init(ctx.arena);
9781014
979 var it = dir.iterate();1015 var it = dir.iterate();
980 var filenames = std.ArrayList([]const u8).init(ctx.arena);1016 var filenames = std.ArrayList([]const u8).init(ctx.arena);
981 defer filenames.deinit();
9821017
983 while (try it.next()) |entry| {1018 while (try it.next()) |entry| {
984 if (entry.kind != .File) continue;1019 if (entry.kind != .File) continue;
...@@ -1021,7 +1056,7 @@ pub const TestContext = struct {...@@ -1021,7 +1056,7 @@ pub const TestContext = struct {
1021 if (new_parts.test_index != null and new_parts.test_index.? != 0) return error.InvalidIncrementalTestIndex;1056 if (new_parts.test_index != null and new_parts.test_index.? != 0) return error.InvalidIncrementalTestIndex;
10221057
1023 if (strategy == .independent)1058 if (strategy == .independent)
1024 opt_case = null; // Generate a new independent test case for this update1059 cases.clearRetainingCapacity(); // Generate a new independent test case for this update
1025 }1060 }
1026 }1061 }
1027 prev_filename = filename;1062 prev_filename = filename;
...@@ -1032,59 +1067,53 @@ pub const TestContext = struct {...@@ -1032,59 +1067,53 @@ pub const TestContext = struct {
1032 // Parse the manifest1067 // Parse the manifest
1033 var manifest = try TestManifest.parse(ctx.arena, src);1068 var manifest = try TestManifest.parse(ctx.arena, src);
10341069
1035 switch (manifest.@"type") {1070 if (cases.items.len == 0) {
1036 .@"error" => {1071 var backends = manifest.getConfigForKey("backend", Backend);
1037 const case = opt_case orelse case: {1072 var targets = manifest.getConfigForKey("target", CrossTarget);
1038 const case = try ctx.cases.addOne();1073 const is_test = manifest.getConfigForKeyAssertSingle("is_test", bool);
1039 const backend = manifest.getConfigForKey("backend", Backend, Backend.parse).next().?;1074 const output_mode = manifest.getConfigForKeyAssertSingle("output_mode", std.builtin.OutputMode);
1040 case.* = .{
1041 .name = "none",
1042 .target = .{},
1043 .backend = backend,
1044 .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator),
1045 .is_test = false,
1046 .output_mode = .Obj,
1047 .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator),
1048 };
1049 opt_case = case;
1050 break :case case;
1051 };
1052 const errors = try manifest.trailingAlloc(ctx.arena);
10531075
1054 switch (strategy) {1076 // Cross-product to get all possible test combinations
1055 .independent => {1077 while (backends.next()) |backend| {
1056 case.addError(src, errors);1078 while (targets.next()) |target| {
1057 },
1058 .incremental => {
1059 case.addErrorNamed("update", src, errors);
1060 },
1061 }
1062 },
1063 .run => {
1064 const case = opt_case orelse case: {
1065 const case = try ctx.cases.addOne();1079 const case = try ctx.cases.addOne();
1066 const backend = manifest.getConfigForKey("backend", Backend, Backend.parse).next().?;
1067 case.* = .{1080 case.* = .{
1068 .name = "none",1081 .name = "none",
1069 .target = .{},1082 .target = target,
1070 .backend = backend,1083 .backend = backend,
1071 .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator),1084 .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator),
1072 .is_test = false,1085 .is_test = is_test,
1073 .output_mode = .Exe,1086 .output_mode = output_mode,
1074 .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator),1087 .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator),
1075 };1088 };
1076 opt_case = case;1089 try cases.append(case);
1077 break :case case;
1078 };
1079
1080 var output = std.ArrayList(u8).init(ctx.arena);
1081 var trailing_it = manifest.trailing();
1082 while (trailing_it.next()) |line| {
1083 try output.appendSlice(line);
1084 }1090 }
1085 case.addCompareOutput(src, output.toOwnedSlice());1091 }
1086 },1092 }
1087 .cli => @panic("TODO cli tests"),1093
1094 for (cases.items) |case| {
1095 switch (manifest.@"type") {
1096 .@"error" => {
1097 const errors = try manifest.trailingAlloc(ctx.arena);
1098 switch (strategy) {
1099 .independent => {
1100 case.addError(src, errors);
1101 },
1102 .incremental => {
1103 case.addErrorNamed("update", src, errors);
1104 },
1105 }
1106 },
1107 .run => {
1108 var output = std.ArrayList(u8).init(ctx.arena);
1109 var trailing_it = manifest.trailing();
1110 while (trailing_it.next()) |line| {
1111 try output.appendSlice(line);
1112 }
1113 case.addCompareOutput(src, output.toOwnedSlice());
1114 },
1115 .cli => @panic("TODO cli tests"),
1116 }
1088 }1117 }
1089 }1118 }
1090 }1119 }
test/incremental/add.0.zig-1
...@@ -7,5 +7,4 @@ fn add(a: u32, b: u32) void {...@@ -7,5 +7,4 @@ fn add(a: u32, b: u32) void {
7}7}
88
9// run9// run
10// backend=stage2
11//10//