authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-04 00:16:28-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
log517cfb0dd1e2b5b8efc8e90ce4e5593a38fa158c
treeb630ec6fa767f2aaf6932472a8acb85ac5089cf5
parent5f92a036f9a9a137e4276d0f605e4cb940eca3a7

fuzzing: progress towards web UI

* libfuzzer: close file after mmap * fuzzer/main.js: connect with EventSource and debug dump the messages. currently this prints how many fuzzer runs have been attempted to console.log. * extract some `std.debug.Info` logic into `std.debug.Coverage`. Prepares for consolidation across multiple different executables which share source files, and makes it possible to send all the PC/SourceLocation mapping data with 4 memcpy'd arrays. * std.Build.Fuzz: - spawn a thread to watch the message queue and signal event subscribers. - track coverage map data - respond to /events URL with EventSource messages on a timer

8 files changed, 478 insertions(+), 165 deletions(-)

lib/fuzzer.zig+1
...@@ -218,6 +218,7 @@ const Fuzzer = struct {...@@ -218,6 +218,7 @@ const Fuzzer = struct {
218 .read = true,218 .read = true,
219 .truncate = false,219 .truncate = false,
220 });220 });
221 defer coverage_file.close();
221 const n_bitset_elems = (flagged_pcs.len + 7) / 8;222 const n_bitset_elems = (flagged_pcs.len + 7) / 8;
222 const bytes_len = @sizeOf(SeenPcsHeader) + flagged_pcs.len * @sizeOf(usize) + n_bitset_elems;223 const bytes_len = @sizeOf(SeenPcsHeader) + flagged_pcs.len * @sizeOf(usize) + n_bitset_elems;
223 const existing_len = coverage_file.getEndPos() catch |err| {224 const existing_len = coverage_file.getEndPos() catch |err| {
lib/fuzzer/main.js+10-3
...@@ -12,6 +12,9 @@...@@ -12,6 +12,9 @@
12 const text_decoder = new TextDecoder();12 const text_decoder = new TextDecoder();
13 const text_encoder = new TextEncoder();13 const text_encoder = new TextEncoder();
1414
15 const eventSource = new EventSource("events");
16 eventSource.addEventListener('message', onMessage, false);
17
15 WebAssembly.instantiateStreaming(wasm_promise, {18 WebAssembly.instantiateStreaming(wasm_promise, {
16 js: {19 js: {
17 log: function(ptr, len) {20 log: function(ptr, len) {
...@@ -38,11 +41,15 @@...@@ -38,11 +41,15 @@
38 });41 });
39 });42 });
4043
44 function onMessage(e) {
45 console.log("Message", e.data);
46 }
47
41 function render() {48 function render() {
42 domSectSource.classList.add("hidden");49 domSectSource.classList.add("hidden");
4350
44 // TODO this is temporary debugging data51 // TODO this is temporary debugging data
45 renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");52 renderSource("/home/andy/dev/zig/lib/std/zig/tokenizer.zig");
46 }53 }
4754
48 function renderSource(path) {55 function renderSource(path) {
lib/std/Build/Fuzz.zig+197-12
...@@ -6,6 +6,7 @@ const assert = std.debug.assert;...@@ -6,6 +6,7 @@ const assert = std.debug.assert;
6const fatal = std.process.fatal;6const fatal = std.process.fatal;
7const Allocator = std.mem.Allocator;7const Allocator = std.mem.Allocator;
8const log = std.log;8const log = std.log;
9const Coverage = std.debug.Coverage;
910
10const Fuzz = @This();11const Fuzz = @This();
11const build_runner = @import("root");12const build_runner = @import("root");
...@@ -53,17 +54,30 @@ pub fn start(...@@ -53,17 +54,30 @@ pub fn start(
53 .global_cache_directory = global_cache_directory,54 .global_cache_directory = global_cache_directory,
54 .zig_lib_directory = zig_lib_directory,55 .zig_lib_directory = zig_lib_directory,
55 .zig_exe_path = zig_exe_path,56 .zig_exe_path = zig_exe_path,
56 .msg_queue = .{},
57 .mutex = .{},
58 .listen_address = listen_address,57 .listen_address = listen_address,
59 .fuzz_run_steps = fuzz_run_steps,58 .fuzz_run_steps = fuzz_run_steps,
59
60 .msg_queue = .{},
61 .mutex = .{},
62 .condition = .{},
63
64 .coverage_files = .{},
65 .coverage_mutex = .{},
66 .coverage_condition = .{},
60 };67 };
6168
69 // For accepting HTTP connections.
62 const web_server_thread = std.Thread.spawn(.{}, WebServer.run, .{&web_server}) catch |err| {70 const web_server_thread = std.Thread.spawn(.{}, WebServer.run, .{&web_server}) catch |err| {
63 fatal("unable to spawn web server thread: {s}", .{@errorName(err)});71 fatal("unable to spawn web server thread: {s}", .{@errorName(err)});
64 };72 };
65 defer web_server_thread.join();73 defer web_server_thread.join();
6674
75 // For polling messages and sending updates to subscribers.
76 const coverage_thread = std.Thread.spawn(.{}, WebServer.coverageRun, .{&web_server}) catch |err| {
77 fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
78 };
79 defer coverage_thread.join();
80
67 {81 {
68 const fuzz_node = prog_node.start("Fuzzing", fuzz_run_steps.len);82 const fuzz_node = prog_node.start("Fuzzing", fuzz_run_steps.len);
69 defer fuzz_node.end();83 defer fuzz_node.end();
...@@ -88,14 +102,38 @@ pub const WebServer = struct {...@@ -88,14 +102,38 @@ pub const WebServer = struct {
88 global_cache_directory: Build.Cache.Directory,102 global_cache_directory: Build.Cache.Directory,
89 zig_lib_directory: Build.Cache.Directory,103 zig_lib_directory: Build.Cache.Directory,
90 zig_exe_path: []const u8,104 zig_exe_path: []const u8,
105 listen_address: std.net.Address,
106 fuzz_run_steps: []const *Step.Run,
107
91 /// Messages from fuzz workers. Protected by mutex.108 /// Messages from fuzz workers. Protected by mutex.
92 msg_queue: std.ArrayListUnmanaged(Msg),109 msg_queue: std.ArrayListUnmanaged(Msg),
110 /// Protects `msg_queue` only.
93 mutex: std.Thread.Mutex,111 mutex: std.Thread.Mutex,
94 listen_address: std.net.Address,112 /// Signaled when there is a message in `msg_queue`.
95 fuzz_run_steps: []const *Step.Run,113 condition: std.Thread.Condition,
114
115 coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
116 /// Protects `coverage_files` only.
117 coverage_mutex: std.Thread.Mutex,
118 /// Signaled when `coverage_files` changes.
119 coverage_condition: std.Thread.Condition,
120
121 const CoverageMap = struct {
122 mapped_memory: []align(std.mem.page_size) const u8,
123 coverage: Coverage,
124
125 fn deinit(cm: *CoverageMap, gpa: Allocator) void {
126 std.posix.munmap(cm.mapped_memory);
127 cm.coverage.deinit(gpa);
128 cm.* = undefined;
129 }
130 };
96131
97 const Msg = union(enum) {132 const Msg = union(enum) {
98 coverage_id: u64,133 coverage: struct {
134 id: u64,
135 run: *Step.Run,
136 },
99 };137 };
100138
101 fn run(ws: *WebServer) void {139 fn run(ws: *WebServer) void {
...@@ -162,6 +200,10 @@ pub const WebServer = struct {...@@ -162,6 +200,10 @@ pub const WebServer = struct {
162 std.mem.eql(u8, request.head.target, "/debug/sources.tar"))200 std.mem.eql(u8, request.head.target, "/debug/sources.tar"))
163 {201 {
164 try serveSourcesTar(ws, request);202 try serveSourcesTar(ws, request);
203 } else if (std.mem.eql(u8, request.head.target, "/events") or
204 std.mem.eql(u8, request.head.target, "/debug/events"))
205 {
206 try serveEvents(ws, request);
165 } else {207 } else {
166 try request.respond("not found", .{208 try request.respond("not found", .{
167 .status = .not_found,209 .status = .not_found,
...@@ -384,6 +426,58 @@ pub const WebServer = struct {...@@ -384,6 +426,58 @@ pub const WebServer = struct {
384 try file.writeAll(std.mem.asBytes(&header));426 try file.writeAll(std.mem.asBytes(&header));
385 }427 }
386428
429 fn serveEvents(ws: *WebServer, request: *std.http.Server.Request) !void {
430 var send_buffer: [0x4000]u8 = undefined;
431 var response = request.respondStreaming(.{
432 .send_buffer = &send_buffer,
433 .respond_options = .{
434 .extra_headers = &.{
435 .{ .name = "content-type", .value = "text/event-stream" },
436 },
437 .transfer_encoding = .none,
438 },
439 });
440
441 ws.coverage_mutex.lock();
442 defer ws.coverage_mutex.unlock();
443
444 if (getStats(ws)) |stats| {
445 try response.writer().print("data: {d}\n\n", .{stats.n_runs});
446 } else {
447 try response.writeAll("data: loading debug information\n\n");
448 }
449 try response.flush();
450
451 while (true) {
452 ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};
453 if (getStats(ws)) |stats| {
454 try response.writer().print("data: {d}\n\n", .{stats.n_runs});
455 try response.flush();
456 }
457 }
458 }
459
460 const Stats = struct {
461 n_runs: u64,
462 };
463
464 fn getStats(ws: *WebServer) ?Stats {
465 const coverage_maps = ws.coverage_files.values();
466 if (coverage_maps.len == 0) return null;
467 // TODO: make each events URL correspond to one coverage map
468 const ptr = coverage_maps[0].mapped_memory;
469 const SeenPcsHeader = extern struct {
470 n_runs: usize,
471 deduplicated_runs: usize,
472 pcs_len: usize,
473 lowest_stack: usize,
474 };
475 const header: *const SeenPcsHeader = @ptrCast(ptr[0..@sizeOf(SeenPcsHeader)]);
476 return .{
477 .n_runs = @atomicLoad(usize, &header.n_runs, .monotonic),
478 };
479 }
480
387 fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {481 fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {
388 const gpa = ws.gpa;482 const gpa = ws.gpa;
389483
...@@ -471,6 +565,95 @@ pub const WebServer = struct {...@@ -471,6 +565,95 @@ pub const WebServer = struct {
471 .name = "cache-control",565 .name = "cache-control",
472 .value = "max-age=0, must-revalidate",566 .value = "max-age=0, must-revalidate",
473 };567 };
568
569 fn coverageRun(ws: *WebServer) void {
570 ws.mutex.lock();
571 defer ws.mutex.unlock();
572
573 while (true) {
574 ws.condition.wait(&ws.mutex);
575 for (ws.msg_queue.items) |msg| switch (msg) {
576 .coverage => |coverage| prepareTables(ws, coverage.run, coverage.id) catch |err| switch (err) {
577 error.AlreadyReported => continue,
578 else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
579 },
580 };
581 ws.msg_queue.clearRetainingCapacity();
582 }
583 }
584
585 fn prepareTables(
586 ws: *WebServer,
587 run_step: *Step.Run,
588 coverage_id: u64,
589 ) error{ OutOfMemory, AlreadyReported }!void {
590 const gpa = ws.gpa;
591
592 ws.coverage_mutex.lock();
593 defer ws.coverage_mutex.unlock();
594
595 const gop = try ws.coverage_files.getOrPut(gpa, coverage_id);
596 if (gop.found_existing) {
597 // We are fuzzing the same executable with multiple threads.
598 // Perhaps the same unit test; perhaps a different one. In any
599 // case, since the coverage file is the same, we only have to
600 // notice changes to that one file in order to learn coverage for
601 // this particular executable.
602 return;
603 }
604 errdefer _ = ws.coverage_files.pop();
605
606 gop.value_ptr.* = .{
607 .coverage = std.debug.Coverage.init,
608 .mapped_memory = undefined, // populated below
609 };
610 errdefer gop.value_ptr.coverage.deinit(gpa);
611
612 const rebuilt_exe_path: Build.Cache.Path = .{
613 .root_dir = Build.Cache.Directory.cwd(),
614 .sub_path = run_step.rebuilt_executable.?,
615 };
616 var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
617 log.err("step '{s}': failed to load debug information for '{}': {s}", .{
618 run_step.step.name, rebuilt_exe_path, @errorName(err),
619 });
620 return error.AlreadyReported;
621 };
622 defer debug_info.deinit(gpa);
623
624 const coverage_file_path: Build.Cache.Path = .{
625 .root_dir = run_step.step.owner.cache_root,
626 .sub_path = "v/" ++ std.fmt.hex(coverage_id),
627 };
628 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
629 log.err("step '{s}': failed to load coverage file '{}': {s}", .{
630 run_step.step.name, coverage_file_path, @errorName(err),
631 });
632 return error.AlreadyReported;
633 };
634 defer coverage_file.close();
635
636 const file_size = coverage_file.getEndPos() catch |err| {
637 log.err("unable to check len of coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
638 return error.AlreadyReported;
639 };
640
641 const mapped_memory = std.posix.mmap(
642 null,
643 file_size,
644 std.posix.PROT.READ,
645 .{ .TYPE = .SHARED },
646 coverage_file.handle,
647 0,
648 ) catch |err| {
649 log.err("failed to map coverage file '{}': {s}", .{ coverage_file_path, @errorName(err) });
650 return error.AlreadyReported;
651 };
652
653 gop.value_ptr.mapped_memory = mapped_memory;
654
655 ws.coverage_condition.broadcast();
656 }
474};657};
475658
476fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {659fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog_node: std.Progress.Node) void {
...@@ -493,16 +676,16 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog...@@ -493,16 +676,16 @@ fn rebuildTestsWorkerRun(run: *Step.Run, ttyconf: std.io.tty.Config, parent_prog
493 build_runner.printErrorMessages(gpa, &compile.step, ttyconf, stderr, false) catch {};676 build_runner.printErrorMessages(gpa, &compile.step, ttyconf, stderr, false) catch {};
494 }677 }
495678
496 if (result) |rebuilt_bin_path| {679 const rebuilt_bin_path = result catch |err| switch (err) {
497 run.rebuilt_executable = rebuilt_bin_path;680 error.MakeFailed => return,
498 } else |err| switch (err) {
499 error.MakeFailed => {},
500 else => {681 else => {
501 std.debug.print("step '{s}': failed to rebuild in fuzz mode: {s}\n", .{682 log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
502 compile.step.name, @errorName(err),683 compile.step.name, @errorName(err),
503 });684 });
685 return;
504 },686 },
505 }687 };
688 run.rebuilt_executable = rebuilt_bin_path;
506}689}
507690
508fn fuzzWorkerRun(691fn fuzzWorkerRun(
...@@ -524,11 +707,13 @@ fn fuzzWorkerRun(...@@ -524,11 +707,13 @@ fn fuzzWorkerRun(
524 std.debug.lockStdErr();707 std.debug.lockStdErr();
525 defer std.debug.unlockStdErr();708 defer std.debug.unlockStdErr();
526 build_runner.printErrorMessages(gpa, &run.step, ttyconf, stderr, false) catch {};709 build_runner.printErrorMessages(gpa, &run.step, ttyconf, stderr, false) catch {};
710 return;
527 },711 },
528 else => {712 else => {
529 std.debug.print("step '{s}': failed to rebuild '{s}' in fuzz mode: {s}\n", .{713 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {s}", .{
530 run.step.name, test_name, @errorName(err),714 run.step.name, test_name, @errorName(err),
531 });715 });
716 return;
532 },717 },
533 };718 };
534}719}
lib/std/Build/Step/Run.zig+5-1
...@@ -1521,7 +1521,11 @@ fn evalZigTest(...@@ -1521,7 +1521,11 @@ fn evalZigTest(
1521 {1521 {
1522 web_server.mutex.lock();1522 web_server.mutex.lock();
1523 defer web_server.mutex.unlock();1523 defer web_server.mutex.unlock();
1524 try web_server.msg_queue.append(web_server.gpa, .{ .coverage_id = coverage_id });1524 try web_server.msg_queue.append(web_server.gpa, .{ .coverage = .{
1525 .id = coverage_id,
1526 .run = run,
1527 } });
1528 web_server.condition.signal();
1525 }1529 }
1526 },1530 },
1527 else => {}, // ignore other messages1531 else => {}, // ignore other messages
lib/std/debug.zig+1
...@@ -19,6 +19,7 @@ pub const Dwarf = @import("debug/Dwarf.zig");...@@ -19,6 +19,7 @@ pub const Dwarf = @import("debug/Dwarf.zig");
19pub const Pdb = @import("debug/Pdb.zig");19pub const Pdb = @import("debug/Pdb.zig");
20pub const SelfInfo = @import("debug/SelfInfo.zig");20pub const SelfInfo = @import("debug/SelfInfo.zig");
21pub const Info = @import("debug/Info.zig");21pub const Info = @import("debug/Info.zig");
22pub const Coverage = @import("debug/Coverage.zig");
2223
23/// Unresolved source locations can be represented with a single `usize` that24/// Unresolved source locations can be represented with a single `usize` that
24/// corresponds to a virtual memory address of the program counter. Combined25/// corresponds to a virtual memory address of the program counter. Combined
lib/std/debug/Coverage.zig created+244
...@@ -0,0 +1,244 @@
1const std = @import("../std.zig");
2const Allocator = std.mem.Allocator;
3const Hash = std.hash.Wyhash;
4const Dwarf = std.debug.Dwarf;
5const assert = std.debug.assert;
6
7const Coverage = @This();
8
9/// Provides a globally-scoped integer index for directories.
10///
11/// As opposed to, for example, a directory index that is compilation-unit
12/// scoped inside a single ELF module.
13///
14/// String memory references the memory-mapped debug information.
15///
16/// Protected by `mutex`.
17directories: std.ArrayHashMapUnmanaged(String, void, String.MapContext, false),
18/// Provides a globally-scoped integer index for files.
19///
20/// String memory references the memory-mapped debug information.
21///
22/// Protected by `mutex`.
23files: std.ArrayHashMapUnmanaged(File, void, File.MapContext, false),
24string_bytes: std.ArrayListUnmanaged(u8),
25/// Protects the other fields.
26mutex: std.Thread.Mutex,
27
28pub const init: Coverage = .{
29 .directories = .{},
30 .files = .{},
31 .mutex = .{},
32 .string_bytes = .{},
33};
34
35pub const String = enum(u32) {
36 _,
37
38 pub const MapContext = struct {
39 string_bytes: []const u8,
40
41 pub fn eql(self: @This(), a: String, b: String, b_index: usize) bool {
42 _ = b_index;
43 const a_slice = span(self.string_bytes[@intFromEnum(a)..]);
44 const b_slice = span(self.string_bytes[@intFromEnum(b)..]);
45 return std.mem.eql(u8, a_slice, b_slice);
46 }
47
48 pub fn hash(self: @This(), a: String) u32 {
49 return @truncate(Hash.hash(0, span(self.string_bytes[@intFromEnum(a)..])));
50 }
51 };
52
53 pub const SliceAdapter = struct {
54 string_bytes: []const u8,
55
56 pub fn eql(self: @This(), a_slice: []const u8, b: String, b_index: usize) bool {
57 _ = b_index;
58 const b_slice = span(self.string_bytes[@intFromEnum(b)..]);
59 return std.mem.eql(u8, a_slice, b_slice);
60 }
61 pub fn hash(self: @This(), a: []const u8) u32 {
62 _ = self;
63 return @truncate(Hash.hash(0, a));
64 }
65 };
66};
67
68pub const SourceLocation = struct {
69 file: File.Index,
70 line: u32,
71 column: u32,
72
73 pub const invalid: SourceLocation = .{
74 .file = .invalid,
75 .line = 0,
76 .column = 0,
77 };
78};
79
80pub const File = struct {
81 directory_index: u32,
82 basename: String,
83
84 pub const Index = enum(u32) {
85 invalid = std.math.maxInt(u32),
86 _,
87 };
88
89 pub const MapContext = struct {
90 string_bytes: []const u8,
91
92 pub fn hash(self: MapContext, a: File) u32 {
93 const a_basename = span(self.string_bytes[@intFromEnum(a.basename)..]);
94 return @truncate(Hash.hash(a.directory_index, a_basename));
95 }
96
97 pub fn eql(self: MapContext, a: File, b: File, b_index: usize) bool {
98 _ = b_index;
99 if (a.directory_index != b.directory_index) return false;
100 const a_basename = span(self.string_bytes[@intFromEnum(a.basename)..]);
101 const b_basename = span(self.string_bytes[@intFromEnum(b.basename)..]);
102 return std.mem.eql(u8, a_basename, b_basename);
103 }
104 };
105
106 pub const SliceAdapter = struct {
107 string_bytes: []const u8,
108
109 pub const Entry = struct {
110 directory_index: u32,
111 basename: []const u8,
112 };
113
114 pub fn hash(self: @This(), a: Entry) u32 {
115 _ = self;
116 return @truncate(Hash.hash(a.directory_index, a.basename));
117 }
118
119 pub fn eql(self: @This(), a: Entry, b: File, b_index: usize) bool {
120 _ = b_index;
121 if (a.directory_index != b.directory_index) return false;
122 const b_basename = span(self.string_bytes[@intFromEnum(b.basename)..]);
123 return std.mem.eql(u8, a.basename, b_basename);
124 }
125 };
126};
127
128pub fn deinit(cov: *Coverage, gpa: Allocator) void {
129 cov.directories.deinit(gpa);
130 cov.files.deinit(gpa);
131 cov.string_bytes.deinit(gpa);
132 cov.* = undefined;
133}
134
135pub fn fileAt(cov: *Coverage, index: File.Index) *File {
136 return &cov.files.keys()[@intFromEnum(index)];
137}
138
139pub fn stringAt(cov: *Coverage, index: String) [:0]const u8 {
140 return span(cov.string_bytes.items[@intFromEnum(index)..]);
141}
142
143pub const ResolveAddressesDwarfError = Dwarf.ScanError;
144
145pub fn resolveAddressesDwarf(
146 cov: *Coverage,
147 gpa: Allocator,
148 sorted_pc_addrs: []const u64,
149 /// Asserts its length equals length of `sorted_pc_addrs`.
150 output: []SourceLocation,
151 d: *Dwarf,
152) ResolveAddressesDwarfError!void {
153 assert(sorted_pc_addrs.len == output.len);
154 assert(d.compile_units_sorted);
155
156 var cu_i: usize = 0;
157 var line_table_i: usize = 0;
158 var cu: *Dwarf.CompileUnit = &d.compile_unit_list.items[0];
159 var range = cu.pc_range.?;
160 // Protects directories and files tables from other threads.
161 cov.mutex.lock();
162 defer cov.mutex.unlock();
163 next_pc: for (sorted_pc_addrs, output) |pc, *out| {
164 while (pc >= range.end) {
165 cu_i += 1;
166 if (cu_i >= d.compile_unit_list.items.len) {
167 out.* = SourceLocation.invalid;
168 continue :next_pc;
169 }
170 cu = &d.compile_unit_list.items[cu_i];
171 line_table_i = 0;
172 range = cu.pc_range orelse {
173 out.* = SourceLocation.invalid;
174 continue :next_pc;
175 };
176 }
177 if (pc < range.start) {
178 out.* = SourceLocation.invalid;
179 continue :next_pc;
180 }
181 if (line_table_i == 0) {
182 line_table_i = 1;
183 cov.mutex.unlock();
184 defer cov.mutex.lock();
185 d.populateSrcLocCache(gpa, cu) catch |err| switch (err) {
186 error.MissingDebugInfo, error.InvalidDebugInfo => {
187 out.* = SourceLocation.invalid;
188 cu_i += 1;
189 if (cu_i < d.compile_unit_list.items.len) {
190 cu = &d.compile_unit_list.items[cu_i];
191 line_table_i = 0;
192 if (cu.pc_range) |r| range = r;
193 }
194 continue :next_pc;
195 },
196 else => |e| return e,
197 };
198 }
199 const slc = &cu.src_loc_cache.?;
200 const table_addrs = slc.line_table.keys();
201 while (line_table_i < table_addrs.len and table_addrs[line_table_i] < pc) line_table_i += 1;
202
203 const entry = slc.line_table.values()[line_table_i - 1];
204 const corrected_file_index = entry.file - @intFromBool(slc.version < 5);
205 const file_entry = slc.files[corrected_file_index];
206 const dir_path = slc.directories[file_entry.dir_index].path;
207 try cov.string_bytes.ensureUnusedCapacity(gpa, dir_path.len + file_entry.path.len + 2);
208 const dir_gop = try cov.directories.getOrPutContextAdapted(gpa, dir_path, String.SliceAdapter{
209 .string_bytes = cov.string_bytes.items,
210 }, String.MapContext{
211 .string_bytes = cov.string_bytes.items,
212 });
213 if (!dir_gop.found_existing)
214 dir_gop.key_ptr.* = addStringAssumeCapacity(cov, dir_path);
215 const file_gop = try cov.files.getOrPutContextAdapted(gpa, File.SliceAdapter.Entry{
216 .directory_index = @intCast(dir_gop.index),
217 .basename = file_entry.path,
218 }, File.SliceAdapter{
219 .string_bytes = cov.string_bytes.items,
220 }, File.MapContext{
221 .string_bytes = cov.string_bytes.items,
222 });
223 if (!file_gop.found_existing) file_gop.key_ptr.* = .{
224 .directory_index = @intCast(dir_gop.index),
225 .basename = addStringAssumeCapacity(cov, file_entry.path),
226 };
227 out.* = .{
228 .file = @enumFromInt(file_gop.index),
229 .line = entry.line,
230 .column = entry.column,
231 };
232 }
233}
234
235pub fn addStringAssumeCapacity(cov: *Coverage, s: []const u8) String {
236 const result: String = @enumFromInt(cov.string_bytes.items.len);
237 cov.string_bytes.appendSliceAssumeCapacity(s);
238 cov.string_bytes.appendAssumeCapacity(0);
239 return result;
240}
241
242fn span(s: []const u8) [:0]const u8 {
243 return std.mem.sliceTo(@as([:0]const u8, @ptrCast(s)), 0);
244}
lib/std/debug/Info.zig+10-143
...@@ -12,85 +12,31 @@ const Path = std.Build.Cache.Path;...@@ -12,85 +12,31 @@ const Path = std.Build.Cache.Path;
12const Dwarf = std.debug.Dwarf;12const Dwarf = std.debug.Dwarf;
13const page_size = std.mem.page_size;13const page_size = std.mem.page_size;
14const assert = std.debug.assert;14const assert = std.debug.assert;
15const Hash = std.hash.Wyhash;15const Coverage = std.debug.Coverage;
16const SourceLocation = std.debug.Coverage.SourceLocation;
1617
17const Info = @This();18const Info = @This();
1819
19/// Sorted by key, ascending.20/// Sorted by key, ascending.
20address_map: std.AutoArrayHashMapUnmanaged(u64, Dwarf.ElfModule),21address_map: std.AutoArrayHashMapUnmanaged(u64, Dwarf.ElfModule),
2122/// Externally managed, outlives this `Info` instance.
22/// Provides a globally-scoped integer index for directories.23coverage: *Coverage,
23///
24/// As opposed to, for example, a directory index that is compilation-unit
25/// scoped inside a single ELF module.
26///
27/// String memory references the memory-mapped debug information.
28///
29/// Protected by `mutex`.
30directories: std.StringArrayHashMapUnmanaged(void),
31/// Provides a globally-scoped integer index for files.
32///
33/// String memory references the memory-mapped debug information.
34///
35/// Protected by `mutex`.
36files: std.ArrayHashMapUnmanaged(File, void, File.MapContext, false),
37/// Protects `directories` and `files`.
38mutex: std.Thread.Mutex,
39
40pub const SourceLocation = struct {
41 file: File.Index,
42 line: u32,
43 column: u32,
44
45 pub const invalid: SourceLocation = .{
46 .file = .invalid,
47 .line = 0,
48 .column = 0,
49 };
50};
51
52pub const File = struct {
53 directory_index: u32,
54 basename: []const u8,
55
56 pub const Index = enum(u32) {
57 invalid = std.math.maxInt(u32),
58 _,
59 };
60
61 pub const MapContext = struct {
62 pub fn hash(ctx: MapContext, a: File) u32 {
63 _ = ctx;
64 return @truncate(Hash.hash(a.directory_index, a.basename));
65 }
66
67 pub fn eql(ctx: MapContext, a: File, b: File, b_index: usize) bool {
68 _ = ctx;
69 _ = b_index;
70 return a.directory_index == b.directory_index and std.mem.eql(u8, a.basename, b.basename);
71 }
72 };
73};
7424
75pub const LoadError = Dwarf.ElfModule.LoadError;25pub const LoadError = Dwarf.ElfModule.LoadError;
7626
77pub fn load(gpa: Allocator, path: Path) LoadError!Info {27pub fn load(gpa: Allocator, path: Path, coverage: *Coverage) LoadError!Info {
78 var sections: Dwarf.SectionArray = Dwarf.null_section_array;28 var sections: Dwarf.SectionArray = Dwarf.null_section_array;
79 var elf_module = try Dwarf.ElfModule.loadPath(gpa, path, null, null, &sections, null);29 var elf_module = try Dwarf.ElfModule.loadPath(gpa, path, null, null, &sections, null);
80 try elf_module.dwarf.sortCompileUnits();30 try elf_module.dwarf.sortCompileUnits();
81 var info: Info = .{31 var info: Info = .{
82 .address_map = .{},32 .address_map = .{},
83 .directories = .{},33 .coverage = coverage,
84 .files = .{},
85 .mutex = .{},
86 };34 };
87 try info.address_map.put(gpa, elf_module.base_address, elf_module);35 try info.address_map.put(gpa, elf_module.base_address, elf_module);
88 return info;36 return info;
89}37}
9038
91pub fn deinit(info: *Info, gpa: Allocator) void {39pub fn deinit(info: *Info, gpa: Allocator) void {
92 info.directories.deinit(gpa);
93 info.files.deinit(gpa);
94 for (info.address_map.values()) |*elf_module| {40 for (info.address_map.values()) |*elf_module| {
95 elf_module.dwarf.deinit(gpa);41 elf_module.dwarf.deinit(gpa);
96 }42 }
...@@ -98,98 +44,19 @@ pub fn deinit(info: *Info, gpa: Allocator) void {...@@ -98,98 +44,19 @@ pub fn deinit(info: *Info, gpa: Allocator) void {
98 info.* = undefined;44 info.* = undefined;
99}45}
10046
101pub fn fileAt(info: *Info, index: File.Index) *File {47pub const ResolveAddressesError = Coverage.ResolveAddressesDwarfError;
102 return &info.files.keys()[@intFromEnum(index)];
103}
104
105pub const ResolveSourceLocationsError = Dwarf.ScanError;
10648
107/// Given an array of virtual memory addresses, sorted ascending, outputs a49/// Given an array of virtual memory addresses, sorted ascending, outputs a
108/// corresponding array of source locations.50/// corresponding array of source locations.
109pub fn resolveSourceLocations(51pub fn resolveAddresses(
110 info: *Info,52 info: *Info,
111 gpa: Allocator,53 gpa: Allocator,
112 sorted_pc_addrs: []const u64,54 sorted_pc_addrs: []const u64,
113 /// Asserts its length equals length of `sorted_pc_addrs`.55 /// Asserts its length equals length of `sorted_pc_addrs`.
114 output: []SourceLocation,56 output: []SourceLocation,
115) ResolveSourceLocationsError!void {57) ResolveAddressesError!void {
116 assert(sorted_pc_addrs.len == output.len);58 assert(sorted_pc_addrs.len == output.len);
117 if (info.address_map.entries.len != 1) @panic("TODO");59 if (info.address_map.entries.len != 1) @panic("TODO");
118 const elf_module = &info.address_map.values()[0];60 const elf_module = &info.address_map.values()[0];
119 return resolveSourceLocationsDwarf(info, gpa, sorted_pc_addrs, output, &elf_module.dwarf);61 return info.coverage.resolveAddressesDwarf(gpa, sorted_pc_addrs, output, &elf_module.dwarf);
120}
121
122pub fn resolveSourceLocationsDwarf(
123 info: *Info,
124 gpa: Allocator,
125 sorted_pc_addrs: []const u64,
126 /// Asserts its length equals length of `sorted_pc_addrs`.
127 output: []SourceLocation,
128 d: *Dwarf,
129) ResolveSourceLocationsError!void {
130 assert(sorted_pc_addrs.len == output.len);
131 assert(d.compile_units_sorted);
132
133 var cu_i: usize = 0;
134 var line_table_i: usize = 0;
135 var cu: *Dwarf.CompileUnit = &d.compile_unit_list.items[0];
136 var range = cu.pc_range.?;
137 // Protects directories and files tables from other threads.
138 info.mutex.lock();
139 defer info.mutex.unlock();
140 next_pc: for (sorted_pc_addrs, output) |pc, *out| {
141 while (pc >= range.end) {
142 cu_i += 1;
143 if (cu_i >= d.compile_unit_list.items.len) {
144 out.* = SourceLocation.invalid;
145 continue :next_pc;
146 }
147 cu = &d.compile_unit_list.items[cu_i];
148 line_table_i = 0;
149 range = cu.pc_range orelse {
150 out.* = SourceLocation.invalid;
151 continue :next_pc;
152 };
153 }
154 if (pc < range.start) {
155 out.* = SourceLocation.invalid;
156 continue :next_pc;
157 }
158 if (line_table_i == 0) {
159 line_table_i = 1;
160 info.mutex.unlock();
161 defer info.mutex.lock();
162 d.populateSrcLocCache(gpa, cu) catch |err| switch (err) {
163 error.MissingDebugInfo, error.InvalidDebugInfo => {
164 out.* = SourceLocation.invalid;
165 cu_i += 1;
166 if (cu_i < d.compile_unit_list.items.len) {
167 cu = &d.compile_unit_list.items[cu_i];
168 line_table_i = 0;
169 if (cu.pc_range) |r| range = r;
170 }
171 continue :next_pc;
172 },
173 else => |e| return e,
174 };
175 }
176 const slc = &cu.src_loc_cache.?;
177 const table_addrs = slc.line_table.keys();
178 while (line_table_i < table_addrs.len and table_addrs[line_table_i] < pc) line_table_i += 1;
179
180 const entry = slc.line_table.values()[line_table_i - 1];
181 const corrected_file_index = entry.file - @intFromBool(slc.version < 5);
182 const file_entry = slc.files[corrected_file_index];
183 const dir_path = slc.directories[file_entry.dir_index].path;
184 const dir_gop = try info.directories.getOrPut(gpa, dir_path);
185 const file_gop = try info.files.getOrPut(gpa, .{
186 .directory_index = @intCast(dir_gop.index),
187 .basename = file_entry.path,
188 });
189 out.* = .{
190 .file = @enumFromInt(file_gop.index),
191 .line = entry.line,
192 .column = entry.column,
193 };
194 }
195}62}
tools/dump-cov.zig+10-6
...@@ -28,7 +28,10 @@ pub fn main() !void {...@@ -28,7 +28,10 @@ pub fn main() !void {
28 .sub_path = cov_file_name,28 .sub_path = cov_file_name,
29 };29 };
3030
31 var debug_info = std.debug.Info.load(gpa, exe_path) catch |err| {31 var coverage = std.debug.Coverage.init;
32 defer coverage.deinit(gpa);
33
34 var debug_info = std.debug.Info.load(gpa, exe_path, &coverage) catch |err| {
32 fatal("failed to load debug info for {}: {s}", .{ exe_path, @errorName(err) });35 fatal("failed to load debug info for {}: {s}", .{ exe_path, @errorName(err) });
33 };36 };
34 defer debug_info.deinit(gpa);37 defer debug_info.deinit(gpa);
...@@ -50,14 +53,15 @@ pub fn main() !void {...@@ -50,14 +53,15 @@ pub fn main() !void {
50 }53 }
51 assert(std.sort.isSorted(usize, pcs, {}, std.sort.asc(usize)));54 assert(std.sort.isSorted(usize, pcs, {}, std.sort.asc(usize)));
5255
53 const source_locations = try arena.alloc(std.debug.Info.SourceLocation, pcs.len);56 const source_locations = try arena.alloc(std.debug.Coverage.SourceLocation, pcs.len);
54 try debug_info.resolveSourceLocations(gpa, pcs, source_locations);57 try debug_info.resolveAddresses(gpa, pcs, source_locations);
5558
56 for (pcs, source_locations) |pc, sl| {59 for (pcs, source_locations) |pc, sl| {
57 const file = debug_info.fileAt(sl.file);60 const file = debug_info.coverage.fileAt(sl.file);
58 const dir_name = debug_info.directories.keys()[file.directory_index];61 const dir_name = debug_info.coverage.directories.keys()[file.directory_index];
62 const dir_name_slice = debug_info.coverage.stringAt(dir_name);
59 try stdout.print("{x}: {s}/{s}:{d}:{d}\n", .{63 try stdout.print("{x}: {s}/{s}:{d}:{d}\n", .{
60 pc, dir_name, file.basename, sl.line, sl.column,64 pc, dir_name_slice, debug_info.coverage.stringAt(file.basename), sl.line, sl.column,
61 });65 });
62 }66 }
6367