authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-04 17:48:08-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-08-07 00:48:32-07:00
loge64a00950e7ec505fdcdb53da265828d1263bc22
tree171577a48868f967738ee8b9e53d3fc04796decd
parent6e6164f8a6c74c8ea5a87ac9961b491b9f89b780

fuzzer web ui: introduce entry points

so you can have somewhere to start browsing

8 files changed, 163 insertions(+), 11 deletions(-)

lib/compiler/test_runner.zig+13-1
...@@ -1,8 +1,10 @@...@@ -1,8 +1,10 @@
1//! Default test runner for unit tests.1//! Default test runner for unit tests.
2const builtin = @import("builtin");2const builtin = @import("builtin");
3
3const std = @import("std");4const std = @import("std");
4const io = std.io;5const io = std.io;
5const testing = std.testing;6const testing = std.testing;
7const assert = std.debug.assert;
68
7pub const std_options = .{9pub const std_options = .{
8 .logFn = log,10 .logFn = log,
...@@ -141,7 +143,9 @@ fn mainServer() !void {...@@ -141,7 +143,9 @@ fn mainServer() !void {
141 });143 });
142 },144 },
143 .start_fuzzing => {145 .start_fuzzing => {
146 if (!builtin.fuzz) unreachable;
144 const index = try server.receiveBody_u32();147 const index = try server.receiveBody_u32();
148 var first = true;
145 const test_fn = builtin.test_functions[index];149 const test_fn = builtin.test_functions[index];
146 while (true) {150 while (true) {
147 testing.allocator_instance = .{};151 testing.allocator_instance = .{};
...@@ -160,6 +164,10 @@ fn mainServer() !void {...@@ -160,6 +164,10 @@ fn mainServer() !void {
160 };164 };
161 if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput");165 if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput");
162 if (log_err_count != 0) @panic("error logs detected");166 if (log_err_count != 0) @panic("error logs detected");
167 if (first) {
168 first = false;
169 try server.serveU64Message(.fuzz_start_addr, entry_addr);
170 }
163 }171 }
164 },172 },
165173
...@@ -339,6 +347,7 @@ const FuzzerSlice = extern struct {...@@ -339,6 +347,7 @@ const FuzzerSlice = extern struct {
339};347};
340348
341var is_fuzz_test: bool = undefined;349var is_fuzz_test: bool = undefined;
350var entry_addr: usize = 0;
342351
343extern fn fuzzer_next() FuzzerSlice;352extern fn fuzzer_next() FuzzerSlice;
344extern fn fuzzer_init(cache_dir: FuzzerSlice) void;353extern fn fuzzer_init(cache_dir: FuzzerSlice) void;
...@@ -348,7 +357,10 @@ pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {...@@ -348,7 +357,10 @@ pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 {
348 @disableInstrumentation();357 @disableInstrumentation();
349 if (crippled) return "";358 if (crippled) return "";
350 is_fuzz_test = true;359 is_fuzz_test = true;
351 if (builtin.fuzz) return fuzzer_next().toSlice();360 if (builtin.fuzz) {
361 if (entry_addr == 0) entry_addr = @returnAddress();
362 return fuzzer_next().toSlice();
363 }
352 if (options.corpus.len == 0) return "";364 if (options.corpus.len == 0) return "";
353 var prng = std.Random.DefaultPrng.init(testing.random_seed);365 var prng = std.Random.DefaultPrng.init(testing.random_seed);
354 const random = prng.random();366 const random = prng.random();
lib/fuzzer/index.html+1
...@@ -131,6 +131,7 @@...@@ -131,6 +131,7 @@
131 <li>Unique Runs: <span id="statUniqueRuns"></span></li>131 <li>Unique Runs: <span id="statUniqueRuns"></span></li>
132 <li>Coverage: <span id="statCoverage"></span></li>132 <li>Coverage: <span id="statCoverage"></span></li>
133 <li>Lowest Stack: <span id="statLowestStack"></span></li>133 <li>Lowest Stack: <span id="statLowestStack"></span></li>
134 <li>Entry Points: <ul id="entryPointsList"></ul></li>
134 </ul>135 </ul>
135 </div>136 </div>
136 <div id="sectSource" class="hidden">137 <div id="sectSource" class="hidden">
lib/fuzzer/main.js+27-5
...@@ -7,6 +7,7 @@...@@ -7,6 +7,7 @@
7 const domStatUniqueRuns = document.getElementById("statUniqueRuns");7 const domStatUniqueRuns = document.getElementById("statUniqueRuns");
8 const domStatCoverage = document.getElementById("statCoverage");8 const domStatCoverage = document.getElementById("statCoverage");
9 const domStatLowestStack = document.getElementById("statLowestStack");9 const domStatLowestStack = document.getElementById("statLowestStack");
10 const domEntryPointsList = document.getElementById("entryPointsList");
1011
11 let wasm_promise = fetch("main.wasm");12 let wasm_promise = fetch("main.wasm");
12 let sources_promise = fetch("sources.tar").then(function(response) {13 let sources_promise = fetch("sources.tar").then(function(response) {
...@@ -30,7 +31,8 @@...@@ -30,7 +31,8 @@
30 throw new Error("panic: " + msg);31 throw new Error("panic: " + msg);
31 },32 },
32 emitSourceIndexChange: onSourceIndexChange,33 emitSourceIndexChange: onSourceIndexChange,
33 emitCoverageUpdate: onCoverageUpdate,34 emitCoverageUpdate: renderStats,
35 emitEntryPointsUpdate: renderStats,
34 },36 },
35 }).then(function(obj) {37 }).then(function(obj) {
36 wasm_exports = obj.instance.exports;38 wasm_exports = obj.instance.exports;
...@@ -98,10 +100,6 @@...@@ -98,10 +100,6 @@
98 render();100 render();
99 }101 }
100102
101 function onCoverageUpdate() {
102 renderStats();
103 }
104
105 function render() {103 function render() {
106 domStatus.classList.add("hidden");104 domStatus.classList.add("hidden");
107 domSectSource.classList.add("hidden");105 domSectSource.classList.add("hidden");
...@@ -120,9 +118,26 @@...@@ -120,9 +118,26 @@
120 domStatCoverage.innerText = coveredSourceLocations + " / " + totalSourceLocations + " (" + percent(coveredSourceLocations, totalSourceLocations) + "%)";118 domStatCoverage.innerText = coveredSourceLocations + " / " + totalSourceLocations + " (" + percent(coveredSourceLocations, totalSourceLocations) + "%)";
121 domStatLowestStack.innerText = unwrapString(wasm_exports.lowestStack());119 domStatLowestStack.innerText = unwrapString(wasm_exports.lowestStack());
122120
121 const entryPoints = unwrapInt32Array(wasm_exports.entryPoints());
122 resizeDomList(domEntryPointsList, entryPoints.length, "<li></li>");
123 for (let i = 0; i < entryPoints.length; i += 1) {
124 const liDom = domEntryPointsList.children[i];
125 liDom.innerText = unwrapString(wasm_exports.sourceLocationLinkHtml(entryPoints[i]));
126 }
127
128
123 domSectStats.classList.remove("hidden");129 domSectStats.classList.remove("hidden");
124 }130 }
125131
132 function resizeDomList(listDom, desiredLen, templateHtml) {
133 for (let i = listDom.childElementCount; i < desiredLen; i += 1) {
134 listDom.insertAdjacentHTML('beforeend', templateHtml);
135 }
136 while (desiredLen < listDom.childElementCount) {
137 listDom.removeChild(listDom.lastChild);
138 }
139 }
140
126 function percent(a, b) {141 function percent(a, b) {
127 return ((Number(a) / Number(b)) * 100).toFixed(1);142 return ((Number(a) / Number(b)) * 100).toFixed(1);
128 }143 }
...@@ -150,6 +165,13 @@...@@ -150,6 +165,13 @@
150 return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));165 return text_decoder.decode(new Uint8Array(wasm_exports.memory.buffer, ptr, len));
151 }166 }
152167
168 function unwrapInt32Array(bigint) {
169 const ptr = Number(bigint & 0xffffffffn);
170 const len = Number(bigint >> 32n);
171 if (len === 0) return new Uint32Array();
172 return new Uint32Array(wasm_exports.memory.buffer, ptr, len);
173 }
174
153 function setInputString(s) {175 function setInputString(s) {
154 const jsArray = text_encoder.encode(s);176 const jsArray = text_encoder.encode(s);
155 const len = jsArray.length;177 const len = jsArray.length;
lib/fuzzer/wasm/main.zig+26
...@@ -14,6 +14,7 @@ const js = struct {...@@ -14,6 +14,7 @@ const js = struct {
14 extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;14 extern "js" fn panic(ptr: [*]const u8, len: usize) noreturn;
15 extern "js" fn emitSourceIndexChange() void;15 extern "js" fn emitSourceIndexChange() void;
16 extern "js" fn emitCoverageUpdate() void;16 extern "js" fn emitCoverageUpdate() void;
17 extern "js" fn emitEntryPointsUpdate() void;
17};18};
1819
19pub const std_options: std.Options = .{20pub const std_options: std.Options = .{
...@@ -64,6 +65,7 @@ export fn message_end() void {...@@ -64,6 +65,7 @@ export fn message_end() void {
64 switch (tag) {65 switch (tag) {
65 .source_index => return sourceIndexMessage(msg_bytes) catch @panic("OOM"),66 .source_index => return sourceIndexMessage(msg_bytes) catch @panic("OOM"),
66 .coverage_update => return coverageUpdateMessage(msg_bytes) catch @panic("OOM"),67 .coverage_update => return coverageUpdateMessage(msg_bytes) catch @panic("OOM"),
68 .entry_points => return entryPointsMessage(msg_bytes) catch @panic("OOM"),
67 _ => unreachable,69 _ => unreachable,
68 }70 }
69}71}
...@@ -219,6 +221,19 @@ fn coverageUpdateMessage(msg_bytes: []u8) error{OutOfMemory}!void {...@@ -219,6 +221,19 @@ fn coverageUpdateMessage(msg_bytes: []u8) error{OutOfMemory}!void {
219 js.emitCoverageUpdate();221 js.emitCoverageUpdate();
220}222}
221223
224var entry_points: std.ArrayListUnmanaged(u32) = .{};
225
226fn entryPointsMessage(msg_bytes: []u8) error{OutOfMemory}!void {
227 const header: abi.EntryPointHeader = @bitCast(msg_bytes[0..@sizeOf(abi.EntryPointHeader)].*);
228 entry_points.resize(gpa, header.flags.locs_len) catch @panic("OOM");
229 @memcpy(entry_points.items, std.mem.bytesAsSlice(u32, msg_bytes[@sizeOf(abi.EntryPointHeader)..]));
230 js.emitEntryPointsUpdate();
231}
232
233export fn entryPoints() Slice(u32) {
234 return Slice(u32).init(entry_points.items);
235}
236
222var coverage = Coverage.init;237var coverage = Coverage.init;
223var coverage_source_locations: std.ArrayListUnmanaged(Coverage.SourceLocation) = .{};238var coverage_source_locations: std.ArrayListUnmanaged(Coverage.SourceLocation) = .{};
224/// Contains the most recent coverage update message, unmodified.239/// Contains the most recent coverage update message, unmodified.
...@@ -246,3 +261,14 @@ fn updateCoverage(...@@ -246,3 +261,14 @@ fn updateCoverage(
246 @memcpy(coverage.directories.entries.items(.key), directories);261 @memcpy(coverage.directories.entries.items(.key), directories);
247 try coverage.directories.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items });262 try coverage.directories.reIndexContext(gpa, .{ .string_bytes = coverage.string_bytes.items });
248}263}
264
265export fn sourceLocationLinkHtml(index: u32) String {
266 const sl = coverage_source_locations.items[index];
267 const file_name = coverage.stringAt(coverage.fileAt(sl.file).basename);
268
269 string_result.clearRetainingCapacity();
270 string_result.writer(gpa).print("{s}:{d}:{d}", .{
271 file_name, sl.line, sl.column,
272 }) catch @panic("OOM");
273 return String.init(string_result.items);
274}
lib/std/Build/Fuzz/WebServer.zig+61-3
...@@ -34,6 +34,8 @@ const CoverageMap = struct {...@@ -34,6 +34,8 @@ const CoverageMap = struct {
34 mapped_memory: []align(std.mem.page_size) const u8,34 mapped_memory: []align(std.mem.page_size) const u8,
35 coverage: Coverage,35 coverage: Coverage,
36 source_locations: []Coverage.SourceLocation,36 source_locations: []Coverage.SourceLocation,
37 /// Elements are indexes into `source_locations` pointing to the unit tests that are being fuzz tested.
38 entry_points: std.ArrayListUnmanaged(u32),
3739
38 fn deinit(cm: *CoverageMap, gpa: Allocator) void {40 fn deinit(cm: *CoverageMap, gpa: Allocator) void {
39 std.posix.munmap(cm.mapped_memory);41 std.posix.munmap(cm.mapped_memory);
...@@ -47,6 +49,10 @@ const Msg = union(enum) {...@@ -47,6 +49,10 @@ const Msg = union(enum) {
47 id: u64,49 id: u64,
48 run: *Step.Run,50 run: *Step.Run,
49 },51 },
52 entry_point: struct {
53 coverage_id: u64,
54 addr: u64,
55 },
50};56};
5157
52pub fn run(ws: *WebServer) void {58pub fn run(ws: *WebServer) void {
...@@ -356,14 +362,20 @@ fn serveWebSocket(ws: *WebServer, web_socket: *std.http.WebSocket) !void {...@@ -356,14 +362,20 @@ fn serveWebSocket(ws: *WebServer, web_socket: *std.http.WebSocket) !void {
356 // On first connection, the client needs all the coverage information362 // On first connection, the client needs all the coverage information
357 // so that subsequent updates can contain only the updated bits.363 // so that subsequent updates can contain only the updated bits.
358 var prev_unique_runs: usize = 0;364 var prev_unique_runs: usize = 0;
359 try sendCoverageContext(ws, web_socket, &prev_unique_runs);365 var prev_entry_points: usize = 0;
366 try sendCoverageContext(ws, web_socket, &prev_unique_runs, &prev_entry_points);
360 while (true) {367 while (true) {
361 ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};368 ws.coverage_condition.timedWait(&ws.coverage_mutex, std.time.ns_per_ms * 500) catch {};
362 try sendCoverageContext(ws, web_socket, &prev_unique_runs);369 try sendCoverageContext(ws, web_socket, &prev_unique_runs, &prev_entry_points);
363 }370 }
364}371}
365372
366fn sendCoverageContext(ws: *WebServer, web_socket: *std.http.WebSocket, prev_unique_runs: *usize) !void {373fn sendCoverageContext(
374 ws: *WebServer,
375 web_socket: *std.http.WebSocket,
376 prev_unique_runs: *usize,
377 prev_entry_points: *usize,
378) !void {
367 const coverage_maps = ws.coverage_files.values();379 const coverage_maps = ws.coverage_files.values();
368 if (coverage_maps.len == 0) return;380 if (coverage_maps.len == 0) return;
369 // TODO: make each events URL correspond to one coverage map381 // TODO: make each events URL correspond to one coverage map
...@@ -407,6 +419,21 @@ fn sendCoverageContext(ws: *WebServer, web_socket: *std.http.WebSocket, prev_uni...@@ -407,6 +419,21 @@ fn sendCoverageContext(ws: *WebServer, web_socket: *std.http.WebSocket, prev_uni
407419
408 prev_unique_runs.* = unique_runs;420 prev_unique_runs.* = unique_runs;
409 }421 }
422
423 if (prev_entry_points.* != coverage_map.entry_points.items.len) {
424 const header: abi.EntryPointHeader = .{
425 .flags = .{
426 .locs_len = @intCast(coverage_map.entry_points.items.len),
427 },
428 };
429 const iovecs: [2]std.posix.iovec_const = .{
430 makeIov(std.mem.asBytes(&header)),
431 makeIov(std.mem.sliceAsBytes(coverage_map.entry_points.items)),
432 };
433 try web_socket.writeMessagev(&iovecs, .binary);
434
435 prev_entry_points.* = coverage_map.entry_points.items.len;
436 }
410}437}
411438
412fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {439fn serveSourcesTar(ws: *WebServer, request: *std.http.Server.Request) !void {
...@@ -508,6 +535,10 @@ pub fn coverageRun(ws: *WebServer) void {...@@ -508,6 +535,10 @@ pub fn coverageRun(ws: *WebServer) void {
508 error.AlreadyReported => continue,535 error.AlreadyReported => continue,
509 else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),536 else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
510 },537 },
538 .entry_point => |entry_point| addEntryPoint(ws, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) {
539 error.AlreadyReported => continue,
540 else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
541 },
511 };542 };
512 ws.msg_queue.clearRetainingCapacity();543 ws.msg_queue.clearRetainingCapacity();
513 }544 }
...@@ -538,6 +569,7 @@ fn prepareTables(...@@ -538,6 +569,7 @@ fn prepareTables(
538 .coverage = std.debug.Coverage.init,569 .coverage = std.debug.Coverage.init,
539 .mapped_memory = undefined, // populated below570 .mapped_memory = undefined, // populated below
540 .source_locations = undefined, // populated below571 .source_locations = undefined, // populated below
572 .entry_points = .{},
541 };573 };
542 errdefer gop.value_ptr.coverage.deinit(gpa);574 errdefer gop.value_ptr.coverage.deinit(gpa);
543575
...@@ -597,6 +629,32 @@ fn prepareTables(...@@ -597,6 +629,32 @@ fn prepareTables(
597 ws.coverage_condition.broadcast();629 ws.coverage_condition.broadcast();
598}630}
599631
632fn addEntryPoint(ws: *WebServer, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {
633 ws.coverage_mutex.lock();
634 defer ws.coverage_mutex.unlock();
635
636 const coverage_map = ws.coverage_files.getPtr(coverage_id).?;
637 const ptr = coverage_map.mapped_memory;
638 const pcs_bytes = ptr[@sizeOf(abi.SeenPcsHeader)..][0 .. coverage_map.source_locations.len * @sizeOf(usize)];
639 const pcs: []const usize = @alignCast(std.mem.bytesAsSlice(usize, pcs_bytes));
640 const index = std.sort.upperBound(usize, addr, pcs, {}, std.sort.asc(usize));
641 if (index >= pcs.len) {
642 log.err("unable to find unit test entry address 0x{x} in source locations (range: 0x{x} to 0x{x})", .{
643 addr, pcs[0], pcs[pcs.len - 1],
644 });
645 return error.AlreadyReported;
646 }
647 if (false) {
648 const sl = coverage_map.source_locations[index];
649 const file_name = coverage_map.coverage.stringAt(coverage_map.coverage.fileAt(sl.file).basename);
650 log.debug("server found entry point {s}:{d}:{d}", .{
651 file_name, sl.line, sl.column,
652 });
653 }
654 const gpa = ws.gpa;
655 try coverage_map.entry_points.append(gpa, @intCast(index));
656}
657
600fn makeIov(s: []const u8) std.posix.iovec_const {658fn makeIov(s: []const u8) std.posix.iovec_const {
601 return .{659 return .{
602 .base = s.ptr,660 .base = s.ptr,
lib/std/Build/Fuzz/abi.zig+14
...@@ -19,6 +19,7 @@ pub const SeenPcsHeader = extern struct {...@@ -19,6 +19,7 @@ pub const SeenPcsHeader = extern struct {
19pub const ToClientTag = enum(u8) {19pub const ToClientTag = enum(u8) {
20 source_index,20 source_index,
21 coverage_update,21 coverage_update,
22 entry_points,
22 _,23 _,
23};24};
2425
...@@ -53,3 +54,16 @@ pub const CoverageUpdateHeader = extern struct {...@@ -53,3 +54,16 @@ pub const CoverageUpdateHeader = extern struct {
53 unique_runs: u64 align(1),54 unique_runs: u64 align(1),
54 lowest_stack: u64 align(1),55 lowest_stack: u64 align(1),
55};56};
57
58/// Sent to the fuzzer web client when the set of entry points is updated.
59///
60/// Trailing:
61/// * one u32 index of source_locations per locs_len
62pub const EntryPointHeader = extern struct {
63 flags: Flags,
64
65 pub const Flags = packed struct(u32) {
66 tag: ToClientTag = .entry_points,
67 locs_len: u24,
68 };
69};
lib/std/Build/Step/Run.zig+17-2
...@@ -1427,6 +1427,7 @@ fn evalZigTest(...@@ -1427,6 +1427,7 @@ fn evalZigTest(
1427 var log_err_count: u32 = 0;1427 var log_err_count: u32 = 0;
14281428
1429 var metadata: ?TestMetadata = null;1429 var metadata: ?TestMetadata = null;
1430 var coverage_id: ?u64 = null;
14301431
1431 var sub_prog_node: ?std.Progress.Node = null;1432 var sub_prog_node: ?std.Progress.Node = null;
1432 defer if (sub_prog_node) |n| n.end();1433 defer if (sub_prog_node) |n| n.end();
...@@ -1517,17 +1518,31 @@ fn evalZigTest(...@@ -1517,17 +1518,31 @@ fn evalZigTest(
1517 .coverage_id => {1518 .coverage_id => {
1518 const web_server = fuzz_context.?.web_server;1519 const web_server = fuzz_context.?.web_server;
1519 const msg_ptr: *align(1) const u64 = @ptrCast(body);1520 const msg_ptr: *align(1) const u64 = @ptrCast(body);
1520 const coverage_id = msg_ptr.*;1521 coverage_id = msg_ptr.*;
1521 {1522 {
1522 web_server.mutex.lock();1523 web_server.mutex.lock();
1523 defer web_server.mutex.unlock();1524 defer web_server.mutex.unlock();
1524 try web_server.msg_queue.append(web_server.gpa, .{ .coverage = .{1525 try web_server.msg_queue.append(web_server.gpa, .{ .coverage = .{
1525 .id = coverage_id,1526 .id = coverage_id.?,
1526 .run = run,1527 .run = run,
1527 } });1528 } });
1528 web_server.condition.signal();1529 web_server.condition.signal();
1529 }1530 }
1530 },1531 },
1532 .fuzz_start_addr => {
1533 const web_server = fuzz_context.?.web_server;
1534 const msg_ptr: *align(1) const u64 = @ptrCast(body);
1535 const addr = msg_ptr.*;
1536 {
1537 web_server.mutex.lock();
1538 defer web_server.mutex.unlock();
1539 try web_server.msg_queue.append(web_server.gpa, .{ .entry_point = .{
1540 .addr = addr,
1541 .coverage_id = coverage_id.?,
1542 } });
1543 web_server.condition.signal();
1544 }
1545 },
1531 else => {}, // ignore other messages1546 else => {}, // ignore other messages
1532 }1547 }
15331548
lib/std/zig/Server.zig+4
...@@ -32,6 +32,10 @@ pub const Message = struct {...@@ -32,6 +32,10 @@ pub const Message = struct {
32 /// to store coverage information. The integer is a hash of the PCs32 /// to store coverage information. The integer is a hash of the PCs
33 /// stored within that file.33 /// stored within that file.
34 coverage_id,34 coverage_id,
35 /// Body is a u64le that indicates the function pointer virtual memory
36 /// address of the fuzz unit test. This is used to provide a starting
37 /// point to view coverage.
38 fuzz_start_addr,
3539
36 _,40 _,
37 };41 };