| author |  february cozzocrea <february.cozzocrea@gmail.com> 2024-03-04 17:26:41-08:00 |
| committer |  Andrew Kelley <andrew@ziglang.org> 2024-03-08 02:59:45-08:00 |
| log | b2427ea7d839a9e17568206c64da56865cd000f1 |
| tree | ec6fa58802142b8046b8d642600f986f85a251c4 |
| parent | 83e578a181e33eedd57666376dab371b7ae58d5b |
test manifest key checking and other fixes
This commit adds several fixes and improvements for the Zig compiler
test harness.
1. -Dskip-translate-c option added for skipping the translate-c tests.
2. translate-c/run-translated-c tests in test/cases/* have been added to
the steps test-translate-c and test-run-translated-c. Closes #18224.
3. Custom name added to the CheckFile step for the translate-c step to
better communicate which test failed.
4. Test manifest key validation added to return an error if a manifest
contains an invalid key.5 files changed, 120 insertions(+), 74 deletions(-)
build.zig+5-6
| ... | @@ -101,6 +101,7 @@ pub fn build(b: *std.Build) !void { | ... | @@ -101,6 +101,7 @@ pub fn build(b: *std.Build) !void { |
| 101 | const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false; | 101 | const skip_non_native = b.option(bool, "skip-non-native", "Main test suite skips non-native builds") orelse false; |
| 102 | const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false; | 102 | const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false; |
| 103 | const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false; | 103 | const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false; |
| 104 | const skip_translate_c = b.option(bool, "skip-translate-c", "Main test suite skips translate-c tests") orelse false; | ||
| 104 | const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false; | 105 | const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false; |
| 105 | 106 | ||
| 106 | const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false; | 107 | const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false; |
| ... | @@ -453,7 +454,10 @@ pub fn build(b: *std.Build) !void { | ... | @@ -453,7 +454,10 @@ pub fn build(b: *std.Build) !void { |
| 453 | }).step); | 454 | }).step); |
| 454 | 455 | ||
| 455 | const test_cases_step = b.step("test-cases", "Run the main compiler test cases"); | 456 | const test_cases_step = b.step("test-cases", "Run the main compiler test cases"); |
| 456 | try tests.addCases(b, test_cases_step, test_filters, check_case_exe, .{ | 457 | try tests.addCases(b, test_cases_step, test_filters, check_case_exe, target, .{ |
| 458 | .skip_translate_c = skip_translate_c, | ||
| 459 | .skip_run_translated_c = skip_run_translated_c, | ||
| 460 | }, .{ | ||
| 457 | .enable_llvm = enable_llvm, | 461 | .enable_llvm = enable_llvm, |
| 458 | .llvm_has_m68k = llvm_has_m68k, | 462 | .llvm_has_m68k = llvm_has_m68k, |
| 459 | .llvm_has_csky = llvm_has_csky, | 463 | .llvm_has_csky = llvm_has_csky, |
| ... | @@ -525,11 +529,6 @@ pub fn build(b: *std.Build) !void { | ... | @@ -525,11 +529,6 @@ pub fn build(b: *std.Build) !void { |
| 525 | test_step.dependOn(tests.addStackTraceTests(b, test_filters, optimization_modes)); | 529 | test_step.dependOn(tests.addStackTraceTests(b, test_filters, optimization_modes)); |
| 526 | test_step.dependOn(tests.addCliTests(b)); | 530 | test_step.dependOn(tests.addCliTests(b)); |
| 527 | test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filters, optimization_modes)); | 531 | test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filters, optimization_modes)); |
| 528 | test_step.dependOn(tests.addTranslateCTests(b, test_filters)); | ||
| 529 | if (!skip_run_translated_c) { | ||
| 530 | test_step.dependOn(tests.addRunTranslatedCTests(b, test_filters, target)); | ||
| 531 | } | ||
| 532 | |||
| 533 | test_step.dependOn(tests.addModuleTests(b, .{ | 532 | test_step.dependOn(tests.addModuleTests(b, .{ |
| 534 | .test_filters = test_filters, | 533 | .test_filters = test_filters, |
| 535 | .root_src = "lib/std/std.zig", | 534 | .root_src = "lib/std/std.zig", |
test/cases/run_translated_c/compound_assignments_with_implicit_casts.c+1-1
| ... | @@ -17,4 +17,4 @@ int main() { | ... | @@ -17,4 +17,4 @@ int main() { |
| 17 | } | 17 | } |
| 18 | 18 | ||
| 19 | // run-translated-c | 19 | // run-translated-c |
| 20 | // c_frontends=aro,clang | 20 | // c_frontend=clang |
test/cases/run_translated_c/explicit_cast_bool_from_float.c+1-1
| ... | @@ -7,4 +7,4 @@ int main() { | ... | @@ -7,4 +7,4 @@ int main() { |
| 7 | } | 7 | } |
| 8 | 8 | ||
| 9 | // run-translated-c | 9 | // run-translated-c |
| 10 | // c_frontends=aro,clang | 10 | // c_frontend=clang |
test/src/Cases.zig+102-60
| ... | @@ -533,6 +533,98 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases { | ... | @@ -533,6 +533,98 @@ pub fn init(gpa: Allocator, arena: Allocator) Cases { |
| 533 | }; | 533 | }; |
| 534 | } | 534 | } |
| 535 | 535 | ||
| 536 | pub const TranslateCOptions = struct { | ||
| 537 | skip_translate_c: bool = false, | ||
| 538 | skip_run_translated_c: bool = false, | ||
| 539 | }; | ||
| 540 | pub fn lowerToTranslateCSteps( | ||
| 541 | self: *Cases, | ||
| 542 | b: *std.Build, | ||
| 543 | parent_step: *std.Build.Step, | ||
| 544 | test_filters: []const []const u8, | ||
| 545 | target: std.Build.ResolvedTarget, | ||
| 546 | translate_c_options: TranslateCOptions, | ||
| 547 | ) void { | ||
| 548 | const host = std.zig.system.resolveTargetQuery(.{}) catch |err| | ||
| 549 | std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)}); | ||
| 550 | |||
| 551 | const tests = @import("../tests.zig"); | ||
| 552 | const test_translate_c_step = b.step("test-translate-c", "Run the C translation tests"); | ||
| 553 | if (!translate_c_options.skip_translate_c) { | ||
| 554 | tests.addTranslateCTests(b, test_translate_c_step, test_filters); | ||
| 555 | parent_step.dependOn(test_translate_c_step); | ||
| 556 | } | ||
| 557 | |||
| 558 | const test_run_translated_c_step = b.step("test-run-translated-c", "Run the Run-Translated-C tests"); | ||
| 559 | if (!translate_c_options.skip_run_translated_c) { | ||
| 560 | tests.addRunTranslatedCTests(b, test_run_translated_c_step, test_filters, target); | ||
| 561 | parent_step.dependOn(test_run_translated_c_step); | ||
| 562 | } | ||
| 563 | |||
| 564 | for (self.translate.items) |case| switch (case.kind) { | ||
| 565 | .run => |output| { | ||
| 566 | if (translate_c_options.skip_run_translated_c) continue; | ||
| 567 | const annotated_case_name = b.fmt("run-translated-c {s}", .{case.name}); | ||
| 568 | for (test_filters) |test_filter| { | ||
| 569 | if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break; | ||
| 570 | } else if (test_filters.len > 0) continue; | ||
| 571 | if (!std.process.can_spawn) { | ||
| 572 | std.debug.print("Unable to spawn child processes on {s}, skipping test.\n", .{@tagName(builtin.os.tag)}); | ||
| 573 | continue; // Pass test. | ||
| 574 | } | ||
| 575 | |||
| 576 | if (getExternalExecutor(host, &case.target.result, .{ .link_libc = true }) != .native) { | ||
| 577 | // We wouldn't be able to run the compiled C code. | ||
| 578 | continue; // Pass test. | ||
| 579 | } | ||
| 580 | |||
| 581 | const write_src = b.addWriteFiles(); | ||
| 582 | const file_source = write_src.add("tmp.c", case.input); | ||
| 583 | |||
| 584 | const translate_c = b.addTranslateC(.{ | ||
| 585 | .root_source_file = file_source, | ||
| 586 | .optimize = .Debug, | ||
| 587 | .target = case.target, | ||
| 588 | .link_libc = case.link_libc, | ||
| 589 | .use_clang = case.c_frontend == .clang, | ||
| 590 | }); | ||
| 591 | translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name}); | ||
| 592 | |||
| 593 | const run_exe = translate_c.addExecutable(.{}); | ||
| 594 | run_exe.step.name = b.fmt("{s} build-exe", .{annotated_case_name}); | ||
| 595 | run_exe.linkLibC(); | ||
| 596 | const run = b.addRunArtifact(run_exe); | ||
| 597 | run.step.name = b.fmt("{s} run", .{annotated_case_name}); | ||
| 598 | run.expectStdOutEqual(output); | ||
| 599 | |||
| 600 | test_run_translated_c_step.dependOn(&run.step); | ||
| 601 | }, | ||
| 602 | .translate => |output| { | ||
| 603 | if (translate_c_options.skip_translate_c) continue; | ||
| 604 | const annotated_case_name = b.fmt("zig translate-c {s}", .{case.name}); | ||
| 605 | for (test_filters) |test_filter| { | ||
| 606 | if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break; | ||
| 607 | } else if (test_filters.len > 0) continue; | ||
| 608 | |||
| 609 | const write_src = b.addWriteFiles(); | ||
| 610 | const file_source = write_src.add("tmp.c", case.input); | ||
| 611 | |||
| 612 | const translate_c = b.addTranslateC(.{ | ||
| 613 | .root_source_file = file_source, | ||
| 614 | .optimize = .Debug, | ||
| 615 | .target = case.target, | ||
| 616 | .link_libc = case.link_libc, | ||
| 617 | .use_clang = case.c_frontend == .clang, | ||
| 618 | }); | ||
| 619 | translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name}); | ||
| 620 | |||
| 621 | const check_file = translate_c.addCheckFile(output); | ||
| 622 | check_file.step.name = b.fmt("{s} CheckFile", .{annotated_case_name}); | ||
| 623 | test_translate_c_step.dependOn(&check_file.step); | ||
| 624 | }, | ||
| 625 | }; | ||
| 626 | } | ||
| 627 | |||
| 536 | pub fn lowerToBuildSteps( | 628 | pub fn lowerToBuildSteps( |
| 537 | self: *Cases, | 629 | self: *Cases, |
| 538 | b: *std.Build, | 630 | b: *std.Build, |
| ... | @@ -681,66 +773,6 @@ pub fn lowerToBuildSteps( | ... | @@ -681,66 +773,6 @@ pub fn lowerToBuildSteps( |
| 681 | .Header => @panic("TODO"), | 773 | .Header => @panic("TODO"), |
| 682 | } | 774 | } |
| 683 | } | 775 | } |
| 684 | |||
| 685 | for (self.translate.items) |case| switch (case.kind) { | ||
| 686 | .run => |output| { | ||
| 687 | const annotated_case_name = b.fmt("run-translated-c {s}", .{case.name}); | ||
| 688 | for (test_filters) |test_filter| { | ||
| 689 | if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break; | ||
| 690 | } else if (test_filters.len > 0) continue; | ||
| 691 | if (!std.process.can_spawn) { | ||
| 692 | std.debug.print("Unable to spawn child processes on {s}, skipping test.\n", .{@tagName(builtin.os.tag)}); | ||
| 693 | continue; // Pass test. | ||
| 694 | } | ||
| 695 | |||
| 696 | if (getExternalExecutor(host, &case.target.result, .{ .link_libc = true }) != .native) { | ||
| 697 | // We wouldn't be able to run the compiled C code. | ||
| 698 | continue; // Pass test. | ||
| 699 | } | ||
| 700 | |||
| 701 | const write_src = b.addWriteFiles(); | ||
| 702 | const file_source = write_src.add("tmp.c", case.input); | ||
| 703 | |||
| 704 | const translate_c = b.addTranslateC(.{ | ||
| 705 | .root_source_file = file_source, | ||
| 706 | .optimize = .Debug, | ||
| 707 | .target = case.target, | ||
| 708 | .link_libc = case.link_libc, | ||
| 709 | .use_clang = case.c_frontend == .clang, | ||
| 710 | }); | ||
| 711 | translate_c.step.name = b.fmt("{s} translate-c", .{annotated_case_name}); | ||
| 712 | |||
| 713 | const run_exe = translate_c.addExecutable(.{}); | ||
| 714 | run_exe.step.name = b.fmt("{s} build-exe", .{annotated_case_name}); | ||
| 715 | run_exe.linkLibC(); | ||
| 716 | const run = b.addRunArtifact(run_exe); | ||
| 717 | run.step.name = b.fmt("{s} run", .{annotated_case_name}); | ||
| 718 | run.expectStdOutEqual(output); | ||
| 719 | |||
| 720 | parent_step.dependOn(&run.step); | ||
| 721 | }, | ||
| 722 | .translate => |output| { | ||
| 723 | const annotated_case_name = b.fmt("zig translate-c {s}", .{case.name}); | ||
| 724 | for (test_filters) |test_filter| { | ||
| 725 | if (std.mem.indexOf(u8, annotated_case_name, test_filter)) |_| break; | ||
| 726 | } else if (test_filters.len > 0) continue; | ||
| 727 | |||
| 728 | const write_src = b.addWriteFiles(); | ||
| 729 | const file_source = write_src.add("tmp.c", case.input); | ||
| 730 | |||
| 731 | const translate_c = b.addTranslateC(.{ | ||
| 732 | .root_source_file = file_source, | ||
| 733 | .optimize = .Debug, | ||
| 734 | .target = case.target, | ||
| 735 | .link_libc = case.link_libc, | ||
| 736 | .use_clang = case.c_frontend == .clang, | ||
| 737 | }); | ||
| 738 | translate_c.step.name = annotated_case_name; | ||
| 739 | |||
| 740 | const check_file = translate_c.addCheckFile(output); | ||
| 741 | parent_step.dependOn(&check_file.step); | ||
| 742 | }, | ||
| 743 | }; | ||
| 744 | } | 776 | } |
| 745 | 777 | ||
| 746 | /// Sort test filenames in-place, so that incremental test cases ("foo.0.zig", | 778 | /// Sort test filenames in-place, so that incremental test cases ("foo.0.zig", |
| ... | @@ -961,6 +993,15 @@ const TestManifest = struct { | ... | @@ -961,6 +993,15 @@ const TestManifest = struct { |
| 961 | config_map: std.StringHashMap([]const u8), | 993 | config_map: std.StringHashMap([]const u8), |
| 962 | trailing_bytes: []const u8 = "", | 994 | trailing_bytes: []const u8 = "", |
| 963 | 995 | ||
| 996 | const valid_keys = std.ComptimeStringMap(void, .{ | ||
| 997 | .{ "is_test", {} }, | ||
| 998 | .{ "output_mode", {} }, | ||
| 999 | .{ "target", {} }, | ||
| 1000 | .{ "c_frontend", {} }, | ||
| 1001 | .{ "link_libc", {} }, | ||
| 1002 | .{ "backend", {} }, | ||
| 1003 | }); | ||
| 1004 | |||
| 964 | const Type = enum { | 1005 | const Type = enum { |
| 965 | @"error", | 1006 | @"error", |
| 966 | run, | 1007 | run, |
| ... | @@ -1059,6 +1100,7 @@ const TestManifest = struct { | ... | @@ -1059,6 +1100,7 @@ const TestManifest = struct { |
| 1059 | // Parse key=value(s) | 1100 | // Parse key=value(s) |
| 1060 | var kv_it = std.mem.splitScalar(u8, trimmed, '='); | 1101 | var kv_it = std.mem.splitScalar(u8, trimmed, '='); |
| 1061 | const key = kv_it.first(); | 1102 | const key = kv_it.first(); |
| 1103 | if (!valid_keys.has(key)) return error.InvalidKey; | ||
| 1062 | try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig); | 1104 | try manifest.config_map.putNoClobber(key, kv_it.next() orelse return error.MissingValuesForConfig); |
| 1063 | } | 1105 | } |
| 1064 | 1106 |
test/tests.zig+11-6
| ... | @@ -998,29 +998,30 @@ pub fn addAssembleAndLinkTests(b: *std.Build, test_filters: []const []const u8, | ... | @@ -998,29 +998,30 @@ pub fn addAssembleAndLinkTests(b: *std.Build, test_filters: []const []const u8, |
| 998 | return cases.step; | 998 | return cases.step; |
| 999 | } | 999 | } |
| 1000 | 1000 | ||
| 1001 | pub fn addTranslateCTests(b: *std.Build, test_filters: []const []const u8) *Step { | 1001 | pub fn addTranslateCTests(b: *std.Build, parent_step: *std.Build.Step, test_filters: []const []const u8) void { |
| 1002 | const cases = b.allocator.create(TranslateCContext) catch @panic("OOM"); | 1002 | const cases = b.allocator.create(TranslateCContext) catch @panic("OOM"); |
| 1003 | cases.* = TranslateCContext{ | 1003 | cases.* = TranslateCContext{ |
| 1004 | .b = b, | 1004 | .b = b, |
| 1005 | .step = b.step("test-translate-c", "Run the C translation tests"), | 1005 | .step = parent_step, |
| 1006 | .test_index = 0, | 1006 | .test_index = 0, |
| 1007 | .test_filters = test_filters, | 1007 | .test_filters = test_filters, |
| 1008 | }; | 1008 | }; |
| 1009 | 1009 | ||
| 1010 | translate_c.addCases(cases); | 1010 | translate_c.addCases(cases); |
| 1011 | 1011 | ||
| 1012 | return cases.step; | 1012 | return; |
| 1013 | } | 1013 | } |
| 1014 | 1014 | ||
| 1015 | pub fn addRunTranslatedCTests( | 1015 | pub fn addRunTranslatedCTests( |
| 1016 | b: *std.Build, | 1016 | b: *std.Build, |
| 1017 | parent_step: *std.Build.Step, | ||
| 1017 | test_filters: []const []const u8, | 1018 | test_filters: []const []const u8, |
| 1018 | target: std.Build.ResolvedTarget, | 1019 | target: std.Build.ResolvedTarget, |
| 1019 | ) *Step { | 1020 | ) void { |
| 1020 | const cases = b.allocator.create(RunTranslatedCContext) catch @panic("OOM"); | 1021 | const cases = b.allocator.create(RunTranslatedCContext) catch @panic("OOM"); |
| 1021 | cases.* = .{ | 1022 | cases.* = .{ |
| 1022 | .b = b, | 1023 | .b = b, |
| 1023 | .step = b.step("test-run-translated-c", "Run the Run-Translated-C tests"), | 1024 | .step = parent_step, |
| 1024 | .test_index = 0, | 1025 | .test_index = 0, |
| 1025 | .test_filters = test_filters, | 1026 | .test_filters = test_filters, |
| 1026 | .target = target, | 1027 | .target = target, |
| ... | @@ -1028,7 +1029,7 @@ pub fn addRunTranslatedCTests( | ... | @@ -1028,7 +1029,7 @@ pub fn addRunTranslatedCTests( |
| 1028 | 1029 | ||
| 1029 | run_translated_c.addCases(cases); | 1030 | run_translated_c.addCases(cases); |
| 1030 | 1031 | ||
| 1031 | return cases.step; | 1032 | return; |
| 1032 | } | 1033 | } |
| 1033 | 1034 | ||
| 1034 | const ModuleTestOptions = struct { | 1035 | const ModuleTestOptions = struct { |
| ... | @@ -1288,6 +1289,8 @@ pub fn addCases( | ... | @@ -1288,6 +1289,8 @@ pub fn addCases( |
| 1288 | parent_step: *Step, | 1289 | parent_step: *Step, |
| 1289 | test_filters: []const []const u8, | 1290 | test_filters: []const []const u8, |
| 1290 | check_case_exe: *std.Build.Step.Compile, | 1291 | check_case_exe: *std.Build.Step.Compile, |
| 1292 | target: std.Build.ResolvedTarget, | ||
| 1293 | translate_c_options: @import("src/Cases.zig").TranslateCOptions, | ||
| 1291 | build_options: @import("cases.zig").BuildOptions, | 1294 | build_options: @import("cases.zig").BuildOptions, |
| 1292 | ) !void { | 1295 | ) !void { |
| 1293 | const arena = b.allocator; | 1296 | const arena = b.allocator; |
| ... | @@ -1301,6 +1304,8 @@ pub fn addCases( | ... | @@ -1301,6 +1304,8 @@ pub fn addCases( |
| 1301 | cases.addFromDir(dir, b); | 1304 | cases.addFromDir(dir, b); |
| 1302 | try @import("cases.zig").addCases(&cases, build_options, b); | 1305 | try @import("cases.zig").addCases(&cases, build_options, b); |
| 1303 | 1306 | ||
| 1307 | cases.lowerToTranslateCSteps(b, parent_step, test_filters, target, translate_c_options); | ||
| 1308 | |||
| 1304 | const cases_dir_path = try b.build_root.join(b.allocator, &.{ "test", "cases" }); | 1309 | const cases_dir_path = try b.build_root.join(b.allocator, &.{ "test", "cases" }); |
| 1305 | cases.lowerToBuildSteps( | 1310 | cases.lowerToBuildSteps( |
| 1306 | b, | 1311 | b, |