| author |  Andrew Kelley <andrew@ziglang.org> 2024-07-24 17:41:44-07:00 |
| committer | Andrew Kelley <andrew@ziglang.org> 2024-07-25 18:52:21-07:00 |
| log | a3c74aca99c7eaf719c745ec6e9ee7366cad1910 |
| tree | b0396505cb9623d1a906501ed97905f7cc8bb6c6 |
| parent | 90dfd86ebee1639e5455ace4e34157ff9b68ac0f |
add --debug-rt CLI arg to the compiler + bonus edits
The flag makes compiler_rt and libfuzzer be in debug mode.
Also:
* fuzzer: override debug logs and disable debug logs for frequently
called functions
* std.Build.Fuzz: fix bug of rerunning the old unit test binary
* report errors from rebuilding the unit tests better
* link.Elf: additionally add tsan lib and fuzzer lib to the hash9 files changed, 116 insertions(+), 29 deletions(-)
lib
compiler
- fuzzer.zig
std
lib/compiler/build_runner.zig+6-4
| ... | ... | @@ -208,6 +208,8 @@ pub fn main() !void { |
| 208 | 208 | try debug_log_scopes.append(next_arg); |
| 209 | 209 | } else if (mem.eql(u8, arg, "--debug-pkg-config")) { |
| 210 | 210 | builder.debug_pkg_config = true; |
| 211 | } else if (mem.eql(u8, arg, "--debug-rt")) { | |
| 212 | graph.debug_compiler_runtime_libs = true; | |
| 211 | 213 | } else if (mem.eql(u8, arg, "--debug-compile-errors")) { |
| 212 | 214 | builder.debug_compile_errors = true; |
| 213 | 215 | } else if (mem.eql(u8, arg, "--system")) { |
| ... | ... | @@ -1072,7 +1074,8 @@ fn workerMakeOneStep( |
| 1072 | 1074 | std.debug.lockStdErr(); |
| 1073 | 1075 | defer std.debug.unlockStdErr(); |
| 1074 | 1076 | |
| 1075 | printErrorMessages(b, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {}; | |
| 1077 | const gpa = b.allocator; | |
| 1078 | printErrorMessages(gpa, s, run.ttyconf, run.stderr, run.prominent_compile_errors) catch {}; | |
| 1076 | 1079 | } |
| 1077 | 1080 | |
| 1078 | 1081 | handle_result: { |
| ... | ... | @@ -1126,14 +1129,12 @@ fn workerMakeOneStep( |
| 1126 | 1129 | } |
| 1127 | 1130 | |
| 1128 | 1131 | pub fn printErrorMessages( |
| 1129 | b: *std.Build, | |
| 1132 | gpa: Allocator, | |
| 1130 | 1133 | failing_step: *Step, |
| 1131 | 1134 | ttyconf: std.io.tty.Config, |
| 1132 | 1135 | stderr: File, |
| 1133 | 1136 | prominent_compile_errors: bool, |
| 1134 | 1137 | ) !void { |
| 1135 | const gpa = b.allocator; | |
| 1136 | ||
| 1137 | 1138 | // Provide context for where these error messages are coming from by |
| 1138 | 1139 | // printing the corresponding Step subtree. |
| 1139 | 1140 | |
| ... | ... | @@ -1313,6 +1314,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { |
| 1313 | 1314 | \\ --seed [integer] For shuffling dependency traversal order (default: random) |
| 1314 | 1315 | \\ --debug-log [scope] Enable debugging the compiler |
| 1315 | 1316 | \\ --debug-pkg-config Fail if unknown pkg-config flags encountered |
| 1317 | \\ --debug-rt Debug compiler runtime libraries | |
| 1316 | 1318 | \\ --verbose-link Enable compiler debug output for linking |
| 1317 | 1319 | \\ --verbose-air Enable compiler debug output for Zig AIR |
| 1318 | 1320 | \\ --verbose-llvm-ir[=file] Enable compiler debug output for LLVM IR |
lib/fuzzer.zig+62-10
| ... | ... | @@ -1,14 +1,40 @@ |
| 1 | 1 | const std = @import("std"); |
| 2 | 2 | const Allocator = std.mem.Allocator; |
| 3 | 3 | |
| 4 | pub const std_options = .{ | |
| 5 | .logFn = logOverride, | |
| 6 | }; | |
| 7 | ||
| 8 | var log_file: ?std.fs.File = null; | |
| 9 | ||
| 10 | fn logOverride( | |
| 11 | comptime level: std.log.Level, | |
| 12 | comptime scope: @TypeOf(.EnumLiteral), | |
| 13 | comptime format: []const u8, | |
| 14 | args: anytype, | |
| 15 | ) void { | |
| 16 | const f = if (log_file) |f| f else f: { | |
| 17 | const f = std.fs.cwd().createFile("libfuzzer.log", .{}) catch @panic("failed to open fuzzer log file"); | |
| 18 | log_file = f; | |
| 19 | break :f f; | |
| 20 | }; | |
| 21 | const prefix1 = comptime level.asText(); | |
| 22 | const prefix2 = if (scope == .default) ": " else "(" ++ @tagName(scope) ++ "): "; | |
| 23 | f.writer().print(prefix1 ++ prefix2 ++ format ++ "\n", args) catch @panic("failed to write to fuzzer log"); | |
| 24 | } | |
| 25 | ||
| 4 | 26 | export threadlocal var __sancov_lowest_stack: usize = 0; |
| 5 | 27 | |
| 6 | 28 | export fn __sanitizer_cov_8bit_counters_init(start: [*]u8, stop: [*]u8) void { |
| 7 | 29 | std.log.debug("__sanitizer_cov_8bit_counters_init start={*}, stop={*}", .{ start, stop }); |
| 8 | 30 | } |
| 9 | 31 | |
| 10 | export fn __sanitizer_cov_pcs_init(pcs_beg: [*]const usize, pcs_end: [*]const usize) void { | |
| 11 | std.log.debug("__sanitizer_cov_pcs_init pcs_beg={*}, pcs_end={*}", .{ pcs_beg, pcs_end }); | |
| 32 | export fn __sanitizer_cov_pcs_init(pc_start: [*]const usize, pc_end: [*]const usize) void { | |
| 33 | std.log.debug("__sanitizer_cov_pcs_init pc_start={*}, pc_end={*}", .{ pc_start, pc_end }); | |
| 34 | fuzzer.pc_range = .{ | |
| 35 | .start = @intFromPtr(pc_start), | |
| 36 | .end = @intFromPtr(pc_start), | |
| 37 | }; | |
| 12 | 38 | } |
| 13 | 39 | |
| 14 | 40 | export fn __sanitizer_cov_trace_const_cmp1(arg1: u8, arg2: u8) void { |
| ... | ... | @@ -48,34 +74,45 @@ export fn __sanitizer_cov_trace_switch(val: u64, cases_ptr: [*]u64) void { |
| 48 | 74 | const len = cases_ptr[0]; |
| 49 | 75 | const val_size_in_bits = cases_ptr[1]; |
| 50 | 76 | const cases = cases_ptr[2..][0..len]; |
| 51 | std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{ | |
| 52 | pc, val, val_size_in_bits, cases.len, | |
| 53 | }); | |
| 77 | _ = val; | |
| 78 | _ = pc; | |
| 79 | _ = val_size_in_bits; | |
| 80 | _ = cases; | |
| 81 | //std.log.debug("0x{x}: switch on value {d} ({d} bits) with {d} cases", .{ | |
| 82 | // pc, val, val_size_in_bits, cases.len, | |
| 83 | //}); | |
| 54 | 84 | } |
| 55 | 85 | |
| 56 | 86 | export fn __sanitizer_cov_trace_pc_indir(callee: usize) void { |
| 57 | 87 | const pc = @returnAddress(); |
| 58 | std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee }); | |
| 88 | _ = callee; | |
| 89 | _ = pc; | |
| 90 | //std.log.debug("0x{x}: indirect call to 0x{x}", .{ pc, callee }); | |
| 59 | 91 | } |
| 60 | 92 | |
| 61 | 93 | fn handleCmp(pc: usize, arg1: u64, arg2: u64) void { |
| 62 | std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); | |
| 94 | _ = pc; | |
| 95 | _ = arg1; | |
| 96 | _ = arg2; | |
| 97 | //std.log.debug("0x{x}: comparison of {d} and {d}", .{ pc, arg1, arg2 }); | |
| 63 | 98 | } |
| 64 | 99 | |
| 65 | 100 | const Fuzzer = struct { |
| 66 | 101 | gpa: Allocator, |
| 67 | 102 | rng: std.Random.DefaultPrng, |
| 68 | 103 | input: std.ArrayListUnmanaged(u8), |
| 104 | pc_range: PcRange, | |
| 105 | count: usize, | |
| 69 | 106 | |
| 70 | 107 | const Slice = extern struct { |
| 71 | 108 | ptr: [*]const u8, |
| 72 | 109 | len: usize, |
| 73 | 110 | |
| 74 | fn toSlice(s: Slice) []const u8 { | |
| 111 | fn toZig(s: Slice) []const u8 { | |
| 75 | 112 | return s.ptr[0..s.len]; |
| 76 | 113 | } |
| 77 | 114 | |
| 78 | fn fromSlice(s: []const u8) Slice { | |
| 115 | fn fromZig(s: []const u8) Slice { | |
| 79 | 116 | return .{ |
| 80 | 117 | .ptr = s.ptr, |
| 81 | 118 | .len = s.len, |
| ... | ... | @@ -83,14 +120,27 @@ const Fuzzer = struct { |
| 83 | 120 | } |
| 84 | 121 | }; |
| 85 | 122 | |
| 123 | const PcRange = struct { | |
| 124 | start: usize, | |
| 125 | end: usize, | |
| 126 | }; | |
| 127 | ||
| 86 | 128 | fn next(f: *Fuzzer) ![]const u8 { |
| 87 | 129 | const gpa = f.gpa; |
| 130 | ||
| 131 | // Prepare next input. | |
| 88 | 132 | const rng = fuzzer.rng.random(); |
| 89 | 133 | const len = rng.uintLessThan(usize, 64); |
| 90 | 134 | try f.input.resize(gpa, len); |
| 91 | 135 | rng.bytes(f.input.items); |
| 136 | f.resetCoverage(); | |
| 137 | f.count += 1; | |
| 92 | 138 | return f.input.items; |
| 93 | 139 | } |
| 140 | ||
| 141 | fn resetCoverage(f: *Fuzzer) void { | |
| 142 | _ = f; | |
| 143 | } | |
| 94 | 144 | }; |
| 95 | 145 | |
| 96 | 146 | var general_purpose_allocator: std.heap.GeneralPurposeAllocator(.{}) = .{}; |
| ... | ... | @@ -99,10 +149,12 @@ var fuzzer: Fuzzer = .{ |
| 99 | 149 | .gpa = general_purpose_allocator.allocator(), |
| 100 | 150 | .rng = std.Random.DefaultPrng.init(0), |
| 101 | 151 | .input = .{}, |
| 152 | .pc_range = .{ .start = 0, .end = 0 }, | |
| 153 | .count = 0, | |
| 102 | 154 | }; |
| 103 | 155 | |
| 104 | 156 | export fn fuzzer_next() Fuzzer.Slice { |
| 105 | return Fuzzer.Slice.fromSlice(fuzzer.next() catch |err| switch (err) { | |
| 157 | return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) { | |
| 106 | 158 | error.OutOfMemory => @panic("out of memory"), |
| 107 | 159 | }); |
| 108 | 160 | } |
lib/std/Build.zig+1
| ... | ... | @@ -113,6 +113,7 @@ pub const Graph = struct { |
| 113 | 113 | arena: Allocator, |
| 114 | 114 | system_library_options: std.StringArrayHashMapUnmanaged(SystemLibraryMode) = .{}, |
| 115 | 115 | system_package_mode: bool = false, |
| 116 | debug_compiler_runtime_libs: bool = false, | |
| 116 | 117 | cache: Cache, |
| 117 | 118 | zig_exe: [:0]const u8, |
| 118 | 119 | env_map: EnvMap, |
lib/std/Build/Fuzz.zig+23-13
| ... | ... | @@ -55,22 +55,32 @@ pub fn start( |
| 55 | 55 | } |
| 56 | 56 | |
| 57 | 57 | fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void { |
| 58 | const compile_step = run.producer.?; | |
| 59 | const prog_node = parent_prog_node.start(compile_step.step.name, 0); | |
| 58 | const gpa = run.step.owner.allocator; | |
| 59 | const stderr = std.io.getStdErr(); | |
| 60 | ||
| 61 | const compile = run.producer.?; | |
| 62 | const prog_node = parent_prog_node.start(compile.step.name, 0); | |
| 60 | 63 | defer prog_node.end(); |
| 61 | if (compile_step.rebuildInFuzzMode(prog_node)) |rebuilt_bin_path| { | |
| 64 | ||
| 65 | const result = compile.rebuildInFuzzMode(prog_node); | |
| 66 | ||
| 67 | const show_compile_errors = compile.step.result_error_bundle.errorMessageCount() > 0; | |
| 68 | const show_error_msgs = compile.step.result_error_msgs.items.len > 0; | |
| 69 | const show_stderr = compile.step.result_stderr.len > 0; | |
| 70 | ||
| 71 | if (show_error_msgs or show_compile_errors or show_stderr) { | |
| 72 | std.debug.lockStdErr(); | |
| 73 | defer std.debug.unlockStdErr(); | |
| 74 | build_runner.printErrorMessages(gpa, &compile.step, ttyconf, stderr, false) catch {}; | |
| 75 | } | |
| 76 | ||
| 77 | if (result) |rebuilt_bin_path| { | |
| 62 | 78 | run.rebuilt_executable = rebuilt_bin_path; |
| 63 | 79 | } else |err| switch (err) { |
| 64 | error.MakeFailed => { | |
| 65 | const b = run.step.owner; | |
| 66 | const stderr = std.io.getStdErr(); | |
| 67 | std.debug.lockStdErr(); | |
| 68 | defer std.debug.unlockStdErr(); | |
| 69 | build_runner.printErrorMessages(b, &compile_step.step, ttyconf, stderr, false) catch {}; | |
| 70 | }, | |
| 80 | error.MakeFailed => {}, | |
| 71 | 81 | else => { |
| 72 | 82 | std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{ |
| 73 | compile_step.step.name, @errorName(err), | |
| 83 | compile.step.name, @errorName(err), | |
| 74 | 84 | }); |
| 75 | 85 | }, |
| 76 | 86 | } |
| ... | ... | @@ -82,6 +92,7 @@ fn fuzzWorkerRun( |
| 82 | 92 | ttyconf: std.io.tty.Config, |
| 83 | 93 | parent_prog_node: std.Progress.Node, |
| 84 | 94 | ) void { |
| 95 | const gpa = run.step.owner.allocator; | |
| 85 | 96 | const test_name = run.cached_test_metadata.?.testName(unit_test_index); |
| 86 | 97 | |
| 87 | 98 | const prog_node = parent_prog_node.start(test_name, 0); |
| ... | ... | @@ -89,11 +100,10 @@ fn fuzzWorkerRun( |
| 89 | 100 | |
| 90 | 101 | run.rerunInFuzzMode(unit_test_index, prog_node) catch |err| switch (err) { |
| 91 | 102 | error.MakeFailed => { |
| 92 | const b = run.step.owner; | |
| 93 | 103 | const stderr = std.io.getStdErr(); |
| 94 | 104 | std.debug.lockStdErr(); |
| 95 | 105 | defer std.debug.unlockStdErr(); |
| 96 | build_runner.printErrorMessages(b, &run.step, ttyconf, stderr, false) catch {}; | |
| 106 | build_runner.printErrorMessages(gpa, &run.step, ttyconf, stderr, false) catch {}; | |
| 97 | 107 | }, |
| 98 | 108 | else => { |
| 99 | 109 | std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{ |
lib/std/Build/Step/Compile.zig+10
| ... | ... | @@ -1483,6 +1483,8 @@ fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { |
| 1483 | 1483 | try zig_args.append("--global-cache-dir"); |
| 1484 | 1484 | try zig_args.append(b.graph.global_cache_root.path orelse "."); |
| 1485 | 1485 | |
| 1486 | if (b.graph.debug_compiler_runtime_libs) try zig_args.append("--debug-rt"); | |
| 1487 | ||
| 1486 | 1488 | try zig_args.append("--name"); |
| 1487 | 1489 | try zig_args.append(compile.name); |
| 1488 | 1490 | |
| ... | ... | @@ -1840,6 +1842,14 @@ fn make(step: *Step, options: Step.MakeOptions) !void { |
| 1840 | 1842 | } |
| 1841 | 1843 | |
| 1842 | 1844 | pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 { |
| 1845 | const gpa = c.step.owner.allocator; | |
| 1846 | ||
| 1847 | c.step.result_error_msgs.clearRetainingCapacity(); | |
| 1848 | c.step.result_stderr = ""; | |
| 1849 | ||
| 1850 | c.step.result_error_bundle.deinit(gpa); | |
| 1851 | c.step.result_error_bundle = std.zig.ErrorBundle.empty; | |
| 1852 | ||
| 1843 | 1853 | const zig_args = try getZigArgs(c, true); |
| 1844 | 1854 | const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false); |
| 1845 | 1855 | return maybe_output_bin_path.?; |
lib/std/Build/Step/Run.zig+4-1
| ... | ... | @@ -865,7 +865,10 @@ pub fn rerunInFuzzMode(run: *Run, unit_test_index: u32, prog_node: std.Progress. |
| 865 | 865 | }, |
| 866 | 866 | .artifact => |pa| { |
| 867 | 867 | const artifact = pa.artifact; |
| 868 | const file_path = artifact.installed_path orelse artifact.generated_bin.?.path.?; | |
| 868 | const file_path = if (artifact == run.producer.?) | |
| 869 | run.rebuilt_executable.? | |
| 870 | else | |
| 871 | (artifact.installed_path orelse artifact.generated_bin.?.path.?); | |
| 869 | 872 | try argv_list.append(arena, b.fmt("{s}{s}", .{ pa.prefix, file_path })); |
| 870 | 873 | }, |
| 871 | 874 | .output_file, .output_directory => unreachable, |
src/Compilation.zig+3-1
| ... | ... | @@ -2180,7 +2180,9 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { |
| 2180 | 2180 | comp.bin_file = try link.File.createEmpty(arena, comp, emit, whole.lf_open_opts); |
| 2181 | 2181 | } |
| 2182 | 2182 | }, |
| 2183 | .incremental => {}, | |
| 2183 | .incremental => { | |
| 2184 | log.debug("Compilation.update for {s}, CacheMode.incremental", .{comp.root_name}); | |
| 2185 | }, | |
| 2184 | 2186 | } |
| 2185 | 2187 | |
| 2186 | 2188 | // From this point we add a preliminary set of file system inputs that |
src/link/Elf.zig+2
| ... | ... | @@ -2286,6 +2286,8 @@ fn linkWithLLD(self: *Elf, arena: Allocator, tid: Zcu.PerThread.Id, prog_node: s |
| 2286 | 2286 | } |
| 2287 | 2287 | try man.addOptionalFile(module_obj_path); |
| 2288 | 2288 | try man.addOptionalFile(compiler_rt_path); |
| 2289 | try man.addOptionalFile(if (comp.tsan_lib) |l| l.full_object_path else null); | |
| 2290 | try man.addOptionalFile(if (comp.fuzzer_lib) |l| l.full_object_path else null); | |
| 2289 | 2291 | |
| 2290 | 2292 | // We can skip hashing libc and libc++ components that we are in charge of building from Zig |
| 2291 | 2293 | // installation sources because they are always a product of the compiler version + target information. |
src/main.zig+5
| ... | ... | @@ -655,6 +655,7 @@ const usage_build_generic = |
| 655 | 655 | \\ --debug-log [scope] Enable printing debug/info log messages for scope |
| 656 | 656 | \\ --debug-compile-errors Crash with helpful diagnostics at the first compile error |
| 657 | 657 | \\ --debug-link-snapshot Enable dumping of the linker's state in JSON format |
| 658 | \\ --debug-rt Debug compiler runtime libraries | |
| 658 | 659 | \\ |
| 659 | 660 | ; |
| 660 | 661 | |
| ... | ... | @@ -912,6 +913,7 @@ fn buildOutputType( |
| 912 | 913 | var minor_subsystem_version: ?u16 = null; |
| 913 | 914 | var mingw_unicode_entry_point: bool = false; |
| 914 | 915 | var enable_link_snapshots: bool = false; |
| 916 | var debug_compiler_runtime_libs = false; | |
| 915 | 917 | var opt_incremental: ?bool = null; |
| 916 | 918 | var install_name: ?[]const u8 = null; |
| 917 | 919 | var hash_style: link.File.Elf.HashStyle = .both; |
| ... | ... | @@ -1367,6 +1369,8 @@ fn buildOutputType( |
| 1367 | 1369 | } else { |
| 1368 | 1370 | enable_link_snapshots = true; |
| 1369 | 1371 | } |
| 1372 | } else if (mem.eql(u8, arg, "--debug-rt")) { | |
| 1373 | debug_compiler_runtime_libs = true; | |
| 1370 | 1374 | } else if (mem.eql(u8, arg, "-fincremental")) { |
| 1371 | 1375 | dev.check(.incremental); |
| 1372 | 1376 | opt_incremental = true; |
| ... | ... | @@ -3408,6 +3412,7 @@ fn buildOutputType( |
| 3408 | 3412 | // noise when --search-prefix and --mod are combined. |
| 3409 | 3413 | .global_cc_argv = try cc_argv.toOwnedSlice(arena), |
| 3410 | 3414 | .file_system_inputs = &file_system_inputs, |
| 3415 | .debug_compiler_runtime_libs = debug_compiler_runtime_libs, | |
| 3411 | 3416 | }) catch |err| switch (err) { |
| 3412 | 3417 | error.LibCUnavailable => { |
| 3413 | 3418 | const triple_name = try target.zigTriple(arena); |