| ... | @@ -72,6 +72,17 @@ test { | ... | @@ -72,6 +72,17 @@ test { |
| 72 | } | 72 | } |
| 73 | } | 73 | } |
| 74 | | 74 | |
| | 75 | { |
| | 76 | const dir_path = try std.fs.path.join(arena, &.{ |
| | 77 | std.fs.path.dirname(@src().file).?, "..", "test", "incremental", |
| | 78 | }); |
| | 79 | |
| | 80 | var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true }); |
| | 81 | defer dir.close(); |
| | 82 | |
| | 83 | ctx.addTestCasesFromDir(dir); |
| | 84 | } |
| | 85 | |
| 75 | try @import("test_cases").addCases(&ctx); | 86 | try @import("test_cases").addCases(&ctx); |
| 76 | | 87 | |
| 77 | try ctx.run(); | 88 | try ctx.run(); |
| ... | @@ -154,6 +165,125 @@ const ErrorMsg = union(enum) { | ... | @@ -154,6 +165,125 @@ const ErrorMsg = union(enum) { |
| 154 | } | 165 | } |
| 155 | }; | 166 | }; |
| 156 | | 167 | |
| | 168 | /// Manifest syntax example: |
| | 169 | /// (see https://github.com/ziglang/zig/issues/11288) |
| | 170 | /// |
| | 171 | /// error |
| | 172 | /// backend=stage1,stage2 |
| | 173 | /// output_mode=exe |
| | 174 | /// |
| | 175 | /// :3:19: error: foo |
| | 176 | /// |
| | 177 | /// run |
| | 178 | /// target=x86_64-linux,aarch64-macos |
| | 179 | /// |
| | 180 | /// I am expected stdout! Hello! |
| | 181 | /// |
| | 182 | /// cli |
| | 183 | /// |
| | 184 | /// build test |
| | 185 | const TestManifest = struct { |
| | 186 | @"type": Type, |
| | 187 | config_map: std.StringHashMap([]const u8), |
| | 188 | trailing_bytes: []const u8 = "", |
| | 189 | |
| | 190 | const Type = enum { |
| | 191 | @"error", |
| | 192 | run, |
| | 193 | cli, |
| | 194 | }; |
| | 195 | |
| | 196 | const TrailingIterator = struct { |
| | 197 | inner: std.mem.TokenIterator(u8), |
| | 198 | |
| | 199 | fn next(self: *TrailingIterator) ?[]const u8 { |
| | 200 | const next_inner = self.inner.next() orelse return null; |
| | 201 | return std.mem.trim(u8, next_inner, " \t"); |
| | 202 | } |
| | 203 | }; |
| | 204 | |
| | 205 | fn ConfigValueIterator(comptime T: type, comptime ParseFn: type) type { |
| | 206 | return struct { |
| | 207 | inner: std.mem.SplitIterator(u8), |
| | 208 | parse_fn: ParseFn, |
| | 209 | |
| | 210 | fn next(self: *@This()) ?T { |
| | 211 | const next_raw = self.inner.next() orelse return null; |
| | 212 | return self.parse_fn(next_raw); |
| | 213 | } |
| | 214 | }; |
| | 215 | } |
| | 216 | |
| | 217 | fn parse(arena: Allocator, bytes: []const u8) !TestManifest { |
| | 218 | var it = std.mem.tokenize(u8, bytes, "\r\n"); |
| | 219 | |
| | 220 | // First line is the test type |
| | 221 | const tt: Type = blk: { |
| | 222 | const line = it.next() orelse return error.MissingTestCaseType; |
| | 223 | const raw = std.mem.trim(u8, line[2..], " \t"); |
| | 224 | if (std.mem.eql(u8, raw, "error")) { |
| | 225 | break :blk .@"error"; |
| | 226 | } else if (std.mem.eql(u8, raw, "run")) { |
| | 227 | break :blk .run; |
| | 228 | } else if (std.mem.eql(u8, raw, "cli")) { |
| | 229 | break :blk .cli; |
| | 230 | } else { |
| | 231 | std.log.warn("unknown test case type requested: {s}", .{raw}); |
| | 232 | return error.UnknownTestCaseType; |
| | 233 | } |
| | 234 | }; |
| | 235 | |
| | 236 | var manifest: TestManifest = .{ |
| | 237 | .@"type" = tt, |
| | 238 | .config_map = std.StringHashMap([]const u8).init(arena), |
| | 239 | }; |
| | 240 | |
| | 241 | // Any subsequent line until a blank comment line is key=value(s) pair |
| | 242 | while (it.next()) |line| { |
| | 243 | const trimmed = std.mem.trim(u8, line[2..], " \t"); |
| | 244 | if (trimmed.len == 0) break; |
| | 245 | |
| | 246 | // Parse key=value(s) |
| | 247 | var kv_it = std.mem.split(u8, trimmed, "="); |
| | 248 | const key = kv_it.next() orelse return error.MissingKeyForConfig; |
| | 249 | try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig); |
| | 250 | } |
| | 251 | |
| | 252 | // Finally, trailing is expected output |
| | 253 | manifest.trailing_bytes = bytes[it.index..]; |
| | 254 | |
| | 255 | return manifest; |
| | 256 | } |
| | 257 | |
| | 258 | fn getConfigValues( |
| | 259 | self: TestManifest, |
| | 260 | key: []const u8, |
| | 261 | comptime T: type, |
| | 262 | parse_fn: anytype, |
| | 263 | ) ?ConfigValueIterator(T, @TypeOf(parse_fn)) { |
| | 264 | const bytes = self.config_map.get(key) orelse return null; |
| | 265 | return ConfigValueIterator(T, @TypeOf(parse_fn)){ |
| | 266 | .inner = std.mem.split(u8, bytes, ","), |
| | 267 | .parse_fn = parse_fn, |
| | 268 | }; |
| | 269 | } |
| | 270 | |
| | 271 | fn trailing(self: TestManifest) TrailingIterator { |
| | 272 | return .{ |
| | 273 | .inner = std.mem.tokenize(u8, self.trailing_bytes, "\r\n"), |
| | 274 | }; |
| | 275 | } |
| | 276 | |
| | 277 | fn trailingAlloc(self: TestManifest, arena: Allocator) ![]const []const u8 { |
| | 278 | var out = std.ArrayList([]const u8).init(arena); |
| | 279 | var it = self.trailing(); |
| | 280 | while (it.next()) |line| { |
| | 281 | try out.append(line); |
| | 282 | } |
| | 283 | return out.toOwnedSlice(); |
| | 284 | } |
| | 285 | }; |
| | 286 | |
| 157 | pub const TestContext = struct { | 287 | pub const TestContext = struct { |
| 158 | arena: Allocator, | 288 | arena: Allocator, |
| 159 | cases: std.ArrayList(Case), | 289 | cases: std.ArrayList(Case), |
| ... | @@ -197,6 +327,10 @@ pub const TestContext = struct { | ... | @@ -197,6 +327,10 @@ pub const TestContext = struct { |
| 197 | stage1, | 327 | stage1, |
| 198 | stage2, | 328 | stage2, |
| 199 | llvm, | 329 | llvm, |
| | 330 | |
| | 331 | fn parse(str: []const u8) ?Backend { |
| | 332 | return std.meta.stringToEnum(Backend, str); |
| | 333 | } |
| 200 | }; | 334 | }; |
| 201 | | 335 | |
| 202 | /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each | 336 | /// A `Case` consists of a list of `Update`. The same `Compilation` is used for each |
| ... | @@ -661,6 +795,10 @@ pub const TestContext = struct { | ... | @@ -661,6 +795,10 @@ pub const TestContext = struct { |
| 661 | /// Execute all tests as incremental updates to a single compilation. Explicitly | 795 | /// Execute all tests as incremental updates to a single compilation. Explicitly |
| 662 | /// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order | 796 | /// incremental tests ("foo.0.zig", "foo.1.zig", etc.) still execute in order |
| 663 | incremental, | 797 | incremental, |
| | 798 | |
| | 799 | fn parse(str: []const u8) ?Strategy { |
| | 800 | return std.meta.stringToEnum(Strategy, str); |
| | 801 | } |
| 664 | }; | 802 | }; |
| 665 | | 803 | |
| 666 | /// Adds a compile-error test for each file in the provided directory, using the | 804 | /// Adds a compile-error test for each file in the provided directory, using the |
| ... | @@ -689,6 +827,15 @@ pub const TestContext = struct { | ... | @@ -689,6 +827,15 @@ pub const TestContext = struct { |
| 689 | }; | 827 | }; |
| 690 | } | 828 | } |
| 691 | | 829 | |
| | 830 | pub fn addTestCasesFromDir(ctx: *TestContext, dir: std.fs.Dir) void { |
| | 831 | var current_file: []const u8 = "none"; |
| | 832 | addTestCasesFromDirInner(ctx, dir, ¤t_file) catch |err| { |
| | 833 | std.debug.panic("test harness failed to process file '{s}': {s}\n", .{ |
| | 834 | current_file, @errorName(err), |
| | 835 | }); |
| | 836 | }; |
| | 837 | } |
| | 838 | |
| 692 | /// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns | 839 | /// For a filename in the format "<filename>.X.<ext>" or "<filename>.<ext>", returns |
| 693 | /// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or | 840 | /// "<filename>", "<ext>" and X parsed as a decimal number. If X is not present, or |
| 694 | /// cannot be parsed as a decimal number, it is treated as part of <filename> | 841 | /// cannot be parsed as a decimal number, it is treated as part of <filename> |
| ... | @@ -749,6 +896,159 @@ pub const TestContext = struct { | ... | @@ -749,6 +896,159 @@ pub const TestContext = struct { |
| 749 | std.sort.sort([]const u8, filenames, Context{}, Context.lessThan); | 896 | std.sort.sort([]const u8, filenames, Context{}, Context.lessThan); |
| 750 | } | 897 | } |
| 751 | | 898 | |
| | 899 | fn addTestCasesFromDirInner( |
| | 900 | ctx: *TestContext, |
| | 901 | dir: std.fs.Dir, |
| | 902 | /// This is kept up to date with the currently being processed file so |
| | 903 | /// that if any errors occur the caller knows it happened during this file. |
| | 904 | current_file: *[]const u8, |
| | 905 | ) !void { |
| | 906 | var opt_case: ?*Case = null; |
| | 907 | |
| | 908 | var it = dir.iterate(); |
| | 909 | var filenames = std.ArrayList([]const u8).init(ctx.arena); |
| | 910 | defer filenames.deinit(); |
| | 911 | |
| | 912 | while (try it.next()) |entry| { |
| | 913 | if (entry.kind != .File) continue; |
| | 914 | |
| | 915 | // Ignore stuff such as .swp files |
| | 916 | switch (Compilation.classifyFileExt(entry.name)) { |
| | 917 | .unknown => continue, |
| | 918 | else => {}, |
| | 919 | } |
| | 920 | try filenames.append(try ctx.arena.dupe(u8, entry.name)); |
| | 921 | } |
| | 922 | |
| | 923 | // Sort filenames, so that incremental tests are contiguous and in-order |
| | 924 | sortTestFilenames(filenames.items); |
| | 925 | |
| | 926 | var prev_filename: []const u8 = ""; |
| | 927 | for (filenames.items) |filename| { |
| | 928 | current_file.* = filename; |
| | 929 | |
| | 930 | { // First, check if this file is part of an incremental update sequence |
| | 931 | |
| | 932 | // Split filename into "<base_name>.<index>.<file_ext>" |
| | 933 | const prev_parts = getTestFileNameParts(prev_filename); |
| | 934 | const new_parts = getTestFileNameParts(filename); |
| | 935 | |
| | 936 | // If base_name and file_ext match, these files are in the same test sequence |
| | 937 | // and the new one should be the incremented version of the previous test |
| | 938 | if (std.mem.eql(u8, prev_parts.base_name, new_parts.base_name) and |
| | 939 | std.mem.eql(u8, prev_parts.file_ext, new_parts.file_ext)) |
| | 940 | { |
| | 941 | |
| | 942 | // This is "foo.X.zig" followed by "foo.Y.zig". Make sure that X = Y + 1 |
| | 943 | if (prev_parts.test_index == null) return error.InvalidIncrementalTestIndex; |
| | 944 | if (new_parts.test_index == null) return error.InvalidIncrementalTestIndex; |
| | 945 | if (new_parts.test_index.? != prev_parts.test_index.? + 1) return error.InvalidIncrementalTestIndex; |
| | 946 | } else { |
| | 947 | |
| | 948 | // This is not the same test sequence, so the new file must be the first file |
| | 949 | // in a new sequence ("*.0.zig") or an independent test file ("*.zig") |
| | 950 | if (new_parts.test_index != null and new_parts.test_index.? != 0) return error.InvalidIncrementalTestIndex; |
| | 951 | |
| | 952 | // if (strategy == .independent) |
| | 953 | // opt_case = null; // Generate a new independent test case for this update |
| | 954 | } |
| | 955 | } |
| | 956 | prev_filename = filename; |
| | 957 | |
| | 958 | const max_file_size = 10 * 1024 * 1024; |
| | 959 | const src = try dir.readFileAllocOptions(ctx.arena, filename, max_file_size, null, 1, 0); |
| | 960 | |
| | 961 | // The manifest is the last contiguous block of comments in the file |
| | 962 | // We scan for the beginning by searching backward for the first non-empty line that does not start with "//" |
| | 963 | var manifest_start: ?usize = null; |
| | 964 | var manifest_end: usize = src.len; |
| | 965 | if (src.len > 0) { |
| | 966 | var cursor: usize = src.len - 1; |
| | 967 | while (true) { |
| | 968 | // Move to beginning of line |
| | 969 | while (cursor > 0 and src[cursor - 1] != '\n') cursor -= 1; |
| | 970 | |
| | 971 | if (std.mem.startsWith(u8, src[cursor..], "//")) { |
| | 972 | manifest_start = cursor; // Contiguous comment line, include in manifest |
| | 973 | } else { |
| | 974 | if (manifest_start != null) break; // Encountered non-comment line, end of manifest |
| | 975 | |
| | 976 | // We ignore all-whitespace lines following the comment block, but anything else |
| | 977 | // means that there is no manifest present. |
| | 978 | if (std.mem.trim(u8, src[cursor..manifest_end], " \r\n\t").len == 0) { |
| | 979 | manifest_end = cursor; |
| | 980 | } else break; // If it's not whitespace, there is no manifest |
| | 981 | } |
| | 982 | |
| | 983 | // Move to previous line |
| | 984 | if (cursor != 0) cursor -= 1 else break; |
| | 985 | } |
| | 986 | } |
| | 987 | |
| | 988 | if (manifest_start) |start| { |
| | 989 | // Parse the manifest |
| | 990 | var mani = try TestManifest.parse(ctx.arena, src[start..manifest_end]); |
| | 991 | const strategy = mani.getConfigValues("strategy", Strategy, Strategy.parse).?.next().?; |
| | 992 | const backend = mani.getConfigValues("backend", Backend, Backend.parse).?.next().?; |
| | 993 | |
| | 994 | switch (mani.@"type") { |
| | 995 | .@"error" => { |
| | 996 | const case = opt_case orelse case: { |
| | 997 | const case = try ctx.cases.addOne(); |
| | 998 | case.* = .{ |
| | 999 | .name = "none", |
| | 1000 | .target = .{}, |
| | 1001 | .backend = backend, |
| | 1002 | .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator), |
| | 1003 | .is_test = false, |
| | 1004 | .output_mode = .Obj, |
| | 1005 | .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator), |
| | 1006 | }; |
| | 1007 | opt_case = case; |
| | 1008 | break :case case; |
| | 1009 | }; |
| | 1010 | const errors = try mani.trailingAlloc(ctx.arena); |
| | 1011 | |
| | 1012 | switch (strategy) { |
| | 1013 | .independent => { |
| | 1014 | case.addError(src, errors); |
| | 1015 | }, |
| | 1016 | .incremental => { |
| | 1017 | case.addErrorNamed("update", src, errors); |
| | 1018 | }, |
| | 1019 | } |
| | 1020 | }, |
| | 1021 | .run => { |
| | 1022 | const case = opt_case orelse case: { |
| | 1023 | const case = try ctx.cases.addOne(); |
| | 1024 | case.* = .{ |
| | 1025 | .name = "none", |
| | 1026 | .target = .{}, |
| | 1027 | .backend = backend, |
| | 1028 | .updates = std.ArrayList(TestContext.Update).init(ctx.cases.allocator), |
| | 1029 | .is_test = false, |
| | 1030 | .output_mode = .Exe, |
| | 1031 | .files = std.ArrayList(TestContext.File).init(ctx.cases.allocator), |
| | 1032 | }; |
| | 1033 | opt_case = case; |
| | 1034 | break :case case; |
| | 1035 | }; |
| | 1036 | |
| | 1037 | var output = std.ArrayList(u8).init(ctx.arena); |
| | 1038 | var trailing_it = mani.trailing(); |
| | 1039 | while (trailing_it.next()) |line| { |
| | 1040 | try output.appendSlice(line); |
| | 1041 | } |
| | 1042 | case.addCompareOutput(src, output.toOwnedSlice()); |
| | 1043 | }, |
| | 1044 | .cli => @panic("TODO cli tests"), |
| | 1045 | } |
| | 1046 | } else { |
| | 1047 | return error.MissingManifest; |
| | 1048 | } |
| | 1049 | } |
| | 1050 | } |
| | 1051 | |
| 752 | fn addErrorCasesFromDirInner( | 1052 | fn addErrorCasesFromDirInner( |
| 753 | ctx: *TestContext, | 1053 | ctx: *TestContext, |
| 754 | name: []const u8, | 1054 | name: []const u8, |
Jakub Konka <kubkon@jakubkonka.com> 2022-04-26 15:17:41+02:00