authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-13 16:44:45-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2023-04-15 10:33:08-07:00
log3c93c1664a3c3546076b39c2d40405f5db537dae
tree762b7b1e28d4472df59d300455a4fc56f0599d41
parentadc9b77d5fc4d1520905f1b1f4b3c80d6d424da6

tests: avoid skipping native tests

Make the test targets use options that match the actual options of CompileStep. This makes the code more straightforward, and ends up making fewer tests incorrectly skipped. For example, now the CI runner on Windows will no longer skip self-hosted x86_64 backend tests.

2 files changed, 96 insertions(+), 126 deletions(-)

build.zig+4-17
......@@ -85,9 +85,7 @@ pub fn build(b: *std.Build) !void {
8585 const skip_cross_glibc = b.option(bool, "skip-cross-glibc", "Main test suite skips builds that require cross glibc") orelse false;
8686 const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
8787 const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;
88 const skip_stage1 = b.option(bool, "skip-stage1", "Main test suite skips stage1 compile error tests") orelse false;
8988 const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
90 const skip_stage2_tests = b.option(bool, "skip-stage2-tests", "Main test suite skips self-hosted compiler tests") orelse false;
9189
9290 const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
9391
......@@ -187,9 +185,7 @@ pub fn build(b: *std.Build) !void {
187185 const compile_step = b.step("compile", "Build the self-hosted compiler");
188186 compile_step.dependOn(&exe.step);
189187
190 if (!skip_stage2_tests) {
191 test_step.dependOn(&exe.step);
192 }
188 test_step.dependOn(&exe.step);
193189
194190 exe.single_threaded = single_threaded;
195191
......@@ -360,7 +356,6 @@ pub fn build(b: *std.Build) !void {
360356 test_cases_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
361357 test_cases_options.addOption(bool, "skip_non_native", skip_non_native);
362358 test_cases_options.addOption(bool, "skip_cross_glibc", skip_cross_glibc);
363 test_cases_options.addOption(bool, "skip_stage1", skip_stage1);
364359 test_cases_options.addOption(bool, "have_llvm", enable_llvm);
365360 test_cases_options.addOption(bool, "llvm_has_m68k", llvm_has_m68k);
366361 test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
......@@ -416,7 +411,7 @@ pub fn build(b: *std.Build) !void {
416411
417412 const test_cases_step = b.step("test-cases", "Run the main compiler test cases");
418413 try tests.addCases(b, test_cases_step, test_filter, check_case_exe);
419 if (!skip_stage2_tests) test_step.dependOn(test_cases_step);
414 test_step.dependOn(test_cases_step);
420415
421416 test_step.dependOn(tests.addModuleTests(b, .{
422417 .test_filter = test_filter,
......@@ -428,8 +423,6 @@ pub fn build(b: *std.Build) !void {
428423 .skip_non_native = skip_non_native,
429424 .skip_cross_glibc = skip_cross_glibc,
430425 .skip_libc = skip_libc,
431 .skip_stage1 = skip_stage1,
432 .skip_stage2 = skip_stage2_tests,
433426 .max_rss = 1 * 1024 * 1024 * 1024,
434427 }));
435428
......@@ -443,8 +436,6 @@ pub fn build(b: *std.Build) !void {
443436 .skip_non_native = skip_non_native,
444437 .skip_cross_glibc = skip_cross_glibc,
445438 .skip_libc = true,
446 .skip_stage1 = skip_stage1,
447 .skip_stage2 = true, // TODO get all these passing
448439 }));
449440
450441 test_step.dependOn(tests.addModuleTests(b, .{
......@@ -457,8 +448,6 @@ pub fn build(b: *std.Build) !void {
457448 .skip_non_native = skip_non_native,
458449 .skip_cross_glibc = skip_cross_glibc,
459450 .skip_libc = true,
460 .skip_stage1 = skip_stage1,
461 .skip_stage2 = true, // TODO get all these passing
462451 }));
463452
464453 test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));
......@@ -466,11 +455,11 @@ pub fn build(b: *std.Build) !void {
466455 b,
467456 optimization_modes,
468457 enable_macos_sdk,
469 skip_stage2_tests,
458 false,
470459 enable_symlinks_windows,
471460 ));
472461 test_step.dependOn(tests.addCAbiTests(b, skip_non_native, skip_release));
473 test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, skip_stage2_tests, enable_symlinks_windows));
462 test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, false, enable_symlinks_windows));
474463 test_step.dependOn(tests.addStackTraceTests(b, test_filter, optimization_modes));
475464 test_step.dependOn(tests.addCliTests(b));
476465 test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, optimization_modes));
......@@ -489,8 +478,6 @@ pub fn build(b: *std.Build) !void {
489478 .skip_non_native = skip_non_native,
490479 .skip_cross_glibc = skip_cross_glibc,
491480 .skip_libc = skip_libc,
492 .skip_stage1 = skip_stage1,
493 .skip_stage2 = true, // TODO get all these passing
494481 // I observed a value of 3398275072 on my M1, and multiplied by 1.1 to
495482 // get this amount:
496483 .max_rss = 3738102579,
test/tests.zig+92-109
......@@ -22,12 +22,12 @@ pub const CompareOutputContext = @import("src/CompareOutput.zig");
2222pub const StackTracesContext = @import("src/StackTrace.zig");
2323
2424const TestTarget = struct {
25 target: CrossTarget = @as(CrossTarget, .{}),
25 target: CrossTarget = .{},
2626 optimize_mode: std.builtin.OptimizeMode = .Debug,
27 link_libc: bool = false,
28 single_threaded: bool = false,
29 disable_native: bool = false,
30 backend: ?std.builtin.CompilerBackend = null,
27 link_libc: ?bool = null,
28 single_threaded: ?bool = null,
29 use_llvm: ?bool = null,
30 use_lld: ?bool = null,
3131};
3232
3333const test_targets = blk: {
......@@ -43,13 +43,47 @@ const test_targets = blk: {
4343 .{
4444 .single_threaded = true,
4545 },
46 .{
47 .optimize_mode = .ReleaseFast,
48 },
49 .{
50 .link_libc = true,
51 .optimize_mode = .ReleaseFast,
52 },
53 .{
54 .optimize_mode = .ReleaseFast,
55 .single_threaded = true,
56 },
57
58 .{
59 .optimize_mode = .ReleaseSafe,
60 },
61 .{
62 .link_libc = true,
63 .optimize_mode = .ReleaseSafe,
64 },
65 .{
66 .optimize_mode = .ReleaseSafe,
67 .single_threaded = true,
68 },
69
70 .{
71 .optimize_mode = .ReleaseSmall,
72 },
73 .{
74 .link_libc = true,
75 .optimize_mode = .ReleaseSmall,
76 },
77 .{
78 .optimize_mode = .ReleaseSmall,
79 .single_threaded = true,
80 },
4681
4782 .{
4883 .target = .{
4984 .ofmt = .c,
5085 },
5186 .link_libc = true,
52 .backend = .stage2_c,
5387 },
5488 .{
5589 .target = .{
......@@ -57,22 +91,24 @@ const test_targets = blk: {
5791 .os_tag = .linux,
5892 .abi = .none,
5993 },
60 .backend = .stage2_x86_64,
94 .use_llvm = false,
95 .use_lld = false,
6196 },
6297 .{
6398 .target = .{
6499 .cpu_arch = .aarch64,
65100 .os_tag = .linux,
66101 },
67 .backend = .stage2_aarch64,
102 .use_llvm = false,
103 .use_lld = false,
68104 },
69105 .{
70106 .target = .{
71107 .cpu_arch = .wasm32,
72108 .os_tag = .wasi,
73109 },
74 .single_threaded = true,
75 .backend = .stage2_wasm,
110 .use_llvm = false,
111 .use_lld = false,
76112 },
77113 // https://github.com/ziglang/zig/issues/13623
78114 //.{
......@@ -80,7 +116,8 @@ const test_targets = blk: {
80116 // .cpu_arch = .arm,
81117 // .os_tag = .linux,
82118 // },
83 // .backend = .stage2_arm,
119 // .use_llvm = false,
120 // .use_lld = false,
84121 //},
85122 // https://github.com/ziglang/zig/issues/13623
86123 //.{
......@@ -88,7 +125,8 @@ const test_targets = blk: {
88125 // .arch_os_abi = "arm-linux-none",
89126 // .cpu_features = "generic+v8a",
90127 // }) catch unreachable,
91 // .backend = .stage2_arm,
128 // .use_llvm = false,
129 // .use_lld = false,
92130 //},
93131 .{
94132 .target = .{
......@@ -96,7 +134,8 @@ const test_targets = blk: {
96134 .os_tag = .macos,
97135 .abi = .none,
98136 },
99 .backend = .stage2_aarch64,
137 .use_llvm = false,
138 .use_lld = false,
100139 },
101140 .{
102141 .target = .{
......@@ -104,7 +143,8 @@ const test_targets = blk: {
104143 .os_tag = .macos,
105144 .abi = .none,
106145 },
107 .backend = .stage2_x86_64,
146 .use_llvm = false,
147 .use_lld = false,
108148 },
109149 .{
110150 .target = .{
......@@ -112,7 +152,8 @@ const test_targets = blk: {
112152 .os_tag = .windows,
113153 .abi = .gnu,
114154 },
115 .backend = .stage2_x86_64,
155 .use_llvm = false,
156 .use_lld = false,
116157 },
117158
118159 .{
......@@ -121,7 +162,6 @@ const test_targets = blk: {
121162 .os_tag = .wasi,
122163 },
123164 .link_libc = false,
124 .single_threaded = true,
125165 },
126166 .{
127167 .target = .{
......@@ -129,7 +169,6 @@ const test_targets = blk: {
129169 .os_tag = .wasi,
130170 },
131171 .link_libc = true,
132 .single_threaded = true,
133172 },
134173
135174 .{
......@@ -413,43 +452,6 @@ const test_targets = blk: {
413452 },
414453 .link_libc = true,
415454 },
416
417 // Do the release tests last because they take a long time
418 .{
419 .optimize_mode = .ReleaseFast,
420 },
421 .{
422 .link_libc = true,
423 .optimize_mode = .ReleaseFast,
424 },
425 .{
426 .optimize_mode = .ReleaseFast,
427 .single_threaded = true,
428 },
429
430 .{
431 .optimize_mode = .ReleaseSafe,
432 },
433 .{
434 .link_libc = true,
435 .optimize_mode = .ReleaseSafe,
436 },
437 .{
438 .optimize_mode = .ReleaseSafe,
439 .single_threaded = true,
440 },
441
442 .{
443 .optimize_mode = .ReleaseSmall,
444 },
445 .{
446 .link_libc = true,
447 .optimize_mode = .ReleaseSmall,
448 },
449 .{
450 .optimize_mode = .ReleaseSmall,
451 .single_threaded = true,
452 },
453455 };
454456};
455457
......@@ -913,8 +915,6 @@ const ModuleTestOptions = struct {
913915 skip_non_native: bool,
914916 skip_cross_glibc: bool,
915917 skip_libc: bool,
916 skip_stage1: bool,
917 skip_stage2: bool,
918918 max_rss: usize = 0,
919919};
920920
......@@ -922,49 +922,41 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
922922 const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc);
923923
924924 for (test_targets) |test_target| {
925 if (options.skip_non_native and !test_target.target.isNative())
925 const is_native = test_target.target.isNative() or
926 (test_target.target.getOsTag() == builtin.os.tag and
927 test_target.target.getCpuArch() == builtin.cpu.arch);
928
929 if (options.skip_non_native and !is_native)
926930 continue;
927931
928 if (options.skip_cross_glibc and test_target.target.isGnuLibC() and test_target.link_libc)
932 if (options.skip_cross_glibc and test_target.target.isGnuLibC() and test_target.link_libc == true)
929933 continue;
930934
931 if (options.skip_libc and test_target.link_libc)
935 if (options.skip_libc and test_target.link_libc == true)
932936 continue;
933937
934 if (test_target.link_libc and test_target.target.getOs().requiresLibC()) {
935 // This would be a redundant test.
938 if (options.skip_single_threaded and test_target.single_threaded == true)
936939 continue;
937 }
938940
939 if (options.skip_single_threaded and test_target.single_threaded)
941 // TODO get compiler-rt tests passing for self-hosted backends.
942 if (test_target.use_llvm == false and mem.eql(u8, options.name, "compiler-rt"))
940943 continue;
941944
942 if (test_target.disable_native and
943 test_target.target.getOsTag() == builtin.os.tag and
944 test_target.target.getCpuArch() == builtin.cpu.arch)
945 {
945 // TODO get universal-libc tests passing for self-hosted backends.
946 if (test_target.use_llvm == false and mem.eql(u8, options.name, "universal-libc"))
946947 continue;
947 }
948948
949 if (test_target.backend) |backend| switch (backend) {
950 .stage1 => if (options.skip_stage1) continue,
951 .stage2_llvm => {},
952 else => if (options.skip_stage2) continue,
953 };
949 // TODO get std lib tests passing for self-hosted backends.
950 if (test_target.use_llvm == false and mem.eql(u8, options.name, "std"))
951 continue;
954952
955953 const want_this_mode = for (options.optimize_modes) |m| {
956954 if (m == test_target.optimize_mode) break true;
957955 } else false;
958956 if (!want_this_mode) continue;
959957
960 const libc_prefix = if (test_target.target.getOs().requiresLibC())
961 ""
962 else if (test_target.link_libc)
963 "c"
964 else
965 "bare";
966
967 const triple_prefix = test_target.target.zigTriple(b.allocator) catch @panic("OOM");
958 const libc_suffix = if (test_target.link_libc == true) "-libc" else "";
959 const triple_txt = test_target.target.zigTriple(b.allocator) catch @panic("OOM");
968960
969961 // wasm32-wasi builds need more RAM, idk why
970962 const max_rss = if (test_target.target.getOs().tag == .wasi)
......@@ -978,42 +970,33 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
978970 .target = test_target.target,
979971 .max_rss = max_rss,
980972 .filter = options.test_filter,
973 .link_libc = test_target.link_libc,
974 .single_threaded = test_target.single_threaded,
975 .use_llvm = test_target.use_llvm,
976 .use_lld = test_target.use_lld,
981977 });
982 const single_threaded_txt = if (test_target.single_threaded) "single" else "multi";
983 const backend_txt = if (test_target.backend) |backend| @tagName(backend) else "default";
984 these_tests.single_threaded = test_target.single_threaded;
985 if (test_target.link_libc) {
986 these_tests.linkSystemLibrary("c");
987 }
978 const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else "";
979 const backend_suffix = if (test_target.use_llvm == true)
980 "-llvm"
981 else if (test_target.target.ofmt == std.Target.ObjectFormat.c)
982 "-cbe"
983 else if (test_target.use_llvm == false)
984 "-selfhosted"
985 else
986 "";
987
988988 these_tests.overrideZigLibDir("lib");
989989 these_tests.addIncludePath("test");
990 if (test_target.backend) |backend| switch (backend) {
991 .stage1 => {
992 @panic("stage1 testing requested");
993 },
994 .stage2_llvm => {
995 these_tests.use_llvm = true;
996 },
997 .stage2_c => {
998 these_tests.use_llvm = false;
999 },
1000 else => {
1001 these_tests.use_llvm = false;
1002 // TODO: force self-hosted linkers to avoid LLD creeping in
1003 // until the auto-select mechanism deems them worthy
1004 these_tests.use_lld = false;
1005 },
1006 };
1007990
1008991 const run = b.addRunArtifact(these_tests);
1009992 run.skip_foreign_checks = true;
1010 run.setName(b.fmt("run test {s}-{s}-{s}-{s}-{s}-{s}", .{
993 run.setName(b.fmt("run test {s}-{s}-{s}{s}{s}{s}", .{
1011994 options.name,
1012 triple_prefix,
995 triple_txt,
1013996 @tagName(test_target.optimize_mode),
1014 libc_prefix,
1015 single_threaded_txt,
1016 backend_txt,
997 libc_suffix,
998 single_threaded_suffix,
999 backend_suffix,
10171000 }));
10181001
10191002 step.dependOn(&run.step);