authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2025-11-25 02:49:37-08:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-11-25 02:49:37-08:00
log5816646aa057383d8f87214824a08ae7a9ecd18e
tree94b381603926e3c7a2130556871b41a7fff8263f
parent53e615b920a5c8de19df515c40edc70c82aee392
parent84353183c724298d2341b857bbd69975d315ec8a
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #26037 from ziglang/build-runner-std.Io

build runner: update from std.Thread.Pool to std.Io

14 files changed, 269 insertions(+), 232 deletions(-)

lib/compiler/build_runner.zig+48-64
......@@ -107,7 +107,6 @@ pub fn main() !void {
107107
108108 var targets = std.array_list.Managed([]const u8).init(arena);
109109 var debug_log_scopes = std.array_list.Managed([]const u8).init(arena);
110 var thread_pool_options: std.Thread.Pool.Options = .{ .allocator = arena };
111110
112111 var install_prefix: ?[]const u8 = null;
113112 var dir_list = std.Build.DirList{};
......@@ -413,19 +412,11 @@ pub fn main() !void {
413412 };
414413 } else if (mem.eql(u8, arg, "-fno-reference-trace")) {
415414 builder.reference_trace = null;
416 } else if (mem.startsWith(u8, arg, "-j")) {
417 const num = arg["-j".len..];
418 const n_jobs = std.fmt.parseUnsigned(u32, num, 10) catch |err| {
419 std.debug.print("unable to parse jobs count '{s}': {s}", .{
420 num, @errorName(err),
421 });
422 process.exit(1);
423 };
424 if (n_jobs < 1) {
425 std.debug.print("number of jobs must be at least 1\n", .{});
426 process.exit(1);
427 }
428 thread_pool_options.n_jobs = n_jobs;
415 } else if (mem.cutPrefix(u8, arg, "-j")) |text| {
416 const n = std.fmt.parseUnsigned(u32, text, 10) catch |err|
417 fatal("unable to parse jobs count '{s}': {t}", .{ text, err });
418 if (n < 1) fatal("number of jobs must be at least 1", .{});
419 threaded.setAsyncLimit(.limited(n));
429420 } else if (mem.eql(u8, arg, "--")) {
430421 builder.args = argsRest(args, arg_idx);
431422 break;
......@@ -503,7 +494,7 @@ pub fn main() !void {
503494
504495 .max_rss = max_rss,
505496 .max_rss_is_default = false,
506 .max_rss_mutex = .{},
497 .max_rss_mutex = .init,
507498 .skip_oom_steps = skip_oom_steps,
508499 .unit_test_timeout_ns = test_timeout_ns,
509500
......@@ -516,7 +507,6 @@ pub fn main() !void {
516507 .error_style = error_style,
517508 .multiline_errors = multiline_errors,
518509 .summary = summary orelse if (watch or webui_listen != null) .line else .failures,
519 .thread_pool = undefined,
520510
521511 .ttyconf = ttyconf,
522512 };
......@@ -547,16 +537,12 @@ pub fn main() !void {
547537 break :w try .init();
548538 };
549539
550 try run.thread_pool.init(thread_pool_options);
551 defer run.thread_pool.deinit();
552
553540 const now = Io.Clock.Timestamp.now(io, .awake) catch |err| fatal("failed to collect timestamp: {t}", .{err});
554541
555542 run.web_server = if (webui_listen) |listen_address| ws: {
556543 if (builtin.single_threaded) unreachable; // `fatal` above
557544 break :ws .init(.{
558545 .gpa = gpa,
559 .thread_pool = &run.thread_pool,
560546 .ttyconf = ttyconf,
561547 .graph = &graph,
562548 .all_steps = run.step_stack.keys(),
......@@ -597,7 +583,7 @@ pub fn main() !void {
597583
598584 if (run.web_server) |*ws| {
599585 assert(!watch); // fatal error after CLI parsing
600 while (true) switch (ws.wait()) {
586 while (true) switch (try ws.wait()) {
601587 .rebuild => {
602588 for (run.step_stack.keys()) |step| {
603589 step.state = .precheck_done;
......@@ -666,7 +652,7 @@ const Run = struct {
666652 gpa: Allocator,
667653 max_rss: u64,
668654 max_rss_is_default: bool,
669 max_rss_mutex: std.Thread.Mutex,
655 max_rss_mutex: Io.Mutex,
670656 skip_oom_steps: bool,
671657 unit_test_timeout_ns: ?u64,
672658 watch: bool,
......@@ -675,7 +661,6 @@ const Run = struct {
675661 memory_blocked_steps: std.ArrayList(*Step),
676662 /// Allocated into `gpa`.
677663 step_stack: std.AutoArrayHashMapUnmanaged(*Step, void),
678 thread_pool: std.Thread.Pool,
679664 /// Similar to the `tty.Config` returned by `std.debug.lockStderrWriter`,
680665 /// but also respects the '--color' flag.
681666 ttyconf: tty.Config,
......@@ -754,14 +739,13 @@ fn runStepNames(
754739 const gpa = run.gpa;
755740 const io = b.graph.io;
756741 const step_stack = &run.step_stack;
757 const thread_pool = &run.thread_pool;
758742
759743 {
760744 const step_prog = parent_prog_node.start("steps", step_stack.count());
761745 defer step_prog.end();
762746
763 var wait_group: std.Thread.WaitGroup = .{};
764 defer wait_group.wait();
747 var group: Io.Group = .init;
748 defer group.wait(io);
765749
766750 // Here we spawn the initial set of tasks with a nice heuristic -
767751 // dependency order. Each worker when it finishes a step will then
......@@ -771,9 +755,7 @@ fn runStepNames(
771755 const step = steps_slice[steps_slice.len - i - 1];
772756 if (step.state == .skipped_oom) continue;
773757
774 thread_pool.spawnWg(&wait_group, workerMakeOneStep, .{
775 &wait_group, b, step, step_prog, run,
776 });
758 group.async(io, workerMakeOneStep, .{ &group, b, step, step_prog, run });
777759 }
778760 }
779761
......@@ -855,7 +837,6 @@ fn runStepNames(
855837 var f = std.Build.Fuzz.init(
856838 gpa,
857839 io,
858 thread_pool,
859840 run.ttyconf,
860841 step_stack.keys(),
861842 parent_prog_node,
......@@ -1318,13 +1299,14 @@ fn constructGraphAndCheckForDependencyLoop(
13181299}
13191300
13201301fn workerMakeOneStep(
1321 wg: *std.Thread.WaitGroup,
1302 group: *Io.Group,
13221303 b: *std.Build,
13231304 s: *Step,
13241305 prog_node: std.Progress.Node,
13251306 run: *Run,
13261307) void {
1327 const thread_pool = &run.thread_pool;
1308 const io = b.graph.io;
1309 const gpa = run.gpa;
13281310
13291311 // First, check the conditions for running this step. If they are not met,
13301312 // then we return without doing the step, relying on another worker to
......@@ -1347,8 +1329,8 @@ fn workerMakeOneStep(
13471329 }
13481330
13491331 if (s.max_rss != 0) {
1350 run.max_rss_mutex.lock();
1351 defer run.max_rss_mutex.unlock();
1332 run.max_rss_mutex.lockUncancelable(io);
1333 defer run.max_rss_mutex.unlock(io);
13521334
13531335 // Avoid running steps twice.
13541336 if (s.state != .precheck_done) {
......@@ -1360,7 +1342,7 @@ fn workerMakeOneStep(
13601342 if (new_claimed_rss > run.max_rss) {
13611343 // Running this step right now could possibly exceed the allotted RSS.
13621344 // Add this step to the queue of memory-blocked steps.
1363 run.memory_blocked_steps.append(run.gpa, s) catch @panic("OOM");
1345 run.memory_blocked_steps.append(gpa, s) catch @panic("OOM");
13641346 return;
13651347 }
13661348
......@@ -1381,12 +1363,11 @@ fn workerMakeOneStep(
13811363
13821364 const make_result = s.make(.{
13831365 .progress_node = sub_prog_node,
1384 .thread_pool = thread_pool,
13851366 .watch = run.watch,
13861367 .web_server = if (run.web_server) |*ws| ws else null,
13871368 .ttyconf = run.ttyconf,
13881369 .unit_test_timeout_ns = run.unit_test_timeout_ns,
1389 .gpa = run.gpa,
1370 .gpa = gpa,
13901371 });
13911372
13921373 // No matter the result, we want to display error/warning messages.
......@@ -1397,7 +1378,7 @@ fn workerMakeOneStep(
13971378 const bw, _ = std.debug.lockStderrWriter(&stdio_buffer_allocation);
13981379 defer std.debug.unlockStderrWriter();
13991380 const ttyconf = run.ttyconf;
1400 printErrorMessages(run.gpa, s, .{}, bw, ttyconf, run.error_style, run.multiline_errors) catch {};
1381 printErrorMessages(gpa, s, .{}, bw, ttyconf, run.error_style, run.multiline_errors) catch {};
14011382 }
14021383
14031384 handle_result: {
......@@ -1419,40 +1400,43 @@ fn workerMakeOneStep(
14191400
14201401 // Successful completion of a step, so we queue up its dependants as well.
14211402 for (s.dependants.items) |dep| {
1422 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1423 wg, b, dep, prog_node, run,
1424 });
1403 group.async(io, workerMakeOneStep, .{ group, b, dep, prog_node, run });
14251404 }
14261405 }
14271406
14281407 // If this is a step that claims resources, we must now queue up other
14291408 // steps that are waiting for resources.
14301409 if (s.max_rss != 0) {
1431 run.max_rss_mutex.lock();
1432 defer run.max_rss_mutex.unlock();
1433
1434 // Give the memory back to the scheduler.
1435 run.claimed_rss -= s.max_rss;
1436 // Avoid kicking off too many tasks that we already know will not have
1437 // enough resources.
1438 var remaining = run.max_rss - run.claimed_rss;
1439 var i: usize = 0;
1440 var j: usize = 0;
1441 while (j < run.memory_blocked_steps.items.len) : (j += 1) {
1442 const dep = run.memory_blocked_steps.items[j];
1443 assert(dep.max_rss != 0);
1444 if (dep.max_rss <= remaining) {
1445 remaining -= dep.max_rss;
1446
1447 thread_pool.spawnWg(wg, workerMakeOneStep, .{
1448 wg, b, dep, prog_node, run,
1449 });
1450 } else {
1451 run.memory_blocked_steps.items[i] = dep;
1452 i += 1;
1410 var dispatch_deps: std.ArrayList(*Step) = .empty;
1411 defer dispatch_deps.deinit(gpa);
1412 dispatch_deps.ensureUnusedCapacity(gpa, run.memory_blocked_steps.items.len) catch @panic("OOM");
1413
1414 {
1415 run.max_rss_mutex.lockUncancelable(io);
1416 defer run.max_rss_mutex.unlock(io);
1417
1418 // Give the memory back to the scheduler.
1419 run.claimed_rss -= s.max_rss;
1420 // Avoid kicking off too many tasks that we already know will not have
1421 // enough resources.
1422 var remaining = run.max_rss - run.claimed_rss;
1423 var i: usize = 0;
1424 for (run.memory_blocked_steps.items) |dep| {
1425 assert(dep.max_rss != 0);
1426 if (dep.max_rss <= remaining) {
1427 remaining -= dep.max_rss;
1428 dispatch_deps.appendAssumeCapacity(dep);
1429 } else {
1430 run.memory_blocked_steps.items[i] = dep;
1431 i += 1;
1432 }
14531433 }
1434 run.memory_blocked_steps.shrinkRetainingCapacity(i);
1435 }
1436 for (dispatch_deps.items) |dep| {
1437 // Must be called without max_rss_mutex held in case it executes recursively.
1438 group.async(io, workerMakeOneStep, .{ group, b, dep, prog_node, run });
14541439 }
1455 run.memory_blocked_steps.shrinkRetainingCapacity(i);
14561440 }
14571441}
14581442
lib/std/Build/Cache.zig+22-9
......@@ -22,7 +22,7 @@ manifest_dir: fs.Dir,
2222hash: HashHelper = .{},
2323/// This value is accessed from multiple threads, protected by mutex.
2424recent_problematic_timestamp: Io.Timestamp = .zero,
25mutex: std.Thread.Mutex = .{},
25mutex: Io.Mutex = .init,
2626
2727/// A set of strings such as the zig library directory or project source root, which
2828/// are stripped from the file paths before putting into the cache. They
......@@ -472,6 +472,7 @@ pub const Manifest = struct {
472472 /// A cache manifest file exists however it could not be parsed.
473473 InvalidFormat,
474474 OutOfMemory,
475 Canceled,
475476 };
476477
477478 /// Check the cache to see if the input exists in it. If it exists, returns `true`.
......@@ -557,12 +558,14 @@ pub const Manifest = struct {
557558 self.diagnostic = .{ .manifest_create = error.FileNotFound };
558559 return error.CacheCheckFailed;
559560 },
561 error.Canceled => return error.Canceled,
560562 else => |e| {
561563 self.diagnostic = .{ .manifest_create = e };
562564 return error.CacheCheckFailed;
563565 },
564566 }
565567 },
568 error.Canceled => return error.Canceled,
566569 else => |e| {
567570 self.diagnostic = .{ .manifest_create = e };
568571 return error.CacheCheckFailed;
......@@ -760,6 +763,7 @@ pub const Manifest = struct {
760763 // Every digest before this one has been populated successfully.
761764 return .{ .miss = .{ .file_digests_populated = idx } };
762765 },
766 error.Canceled => return error.Canceled,
763767 else => |e| {
764768 self.diagnostic = .{ .file_open = .{
765769 .file_index = idx,
......@@ -788,7 +792,7 @@ pub const Manifest = struct {
788792 .inode = actual_stat.inode,
789793 };
790794
791 if (self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
795 if (try self.isProblematicTimestamp(cache_hash_file.stat.mtime)) {
792796 // The actual file has an unreliable timestamp, force it to be hashed
793797 cache_hash_file.stat.mtime = .zero;
794798 cache_hash_file.stat.inode = 0;
......@@ -846,7 +850,9 @@ pub const Manifest = struct {
846850 }
847851 }
848852
849 fn isProblematicTimestamp(man: *Manifest, timestamp: Io.Timestamp) bool {
853 fn isProblematicTimestamp(man: *Manifest, timestamp: Io.Timestamp) error{Canceled}!bool {
854 const io = man.cache.io;
855
850856 // If the file_time is prior to the most recent problematic timestamp
851857 // then we don't need to access the filesystem.
852858 if (timestamp.nanoseconds < man.recent_problematic_timestamp.nanoseconds)
......@@ -854,8 +860,8 @@ pub const Manifest = struct {
854860
855861 // Next we will check the globally shared Cache timestamp, which is accessed
856862 // from multiple threads.
857 man.cache.mutex.lock();
858 defer man.cache.mutex.unlock();
863 try man.cache.mutex.lock(io);
864 defer man.cache.mutex.unlock(io);
859865
860866 // Save the global one to our local one to avoid locking next time.
861867 man.recent_problematic_timestamp = man.cache.recent_problematic_timestamp;
......@@ -869,11 +875,18 @@ pub const Manifest = struct {
869875 var file = man.cache.manifest_dir.createFile("timestamp", .{
870876 .read = true,
871877 .truncate = true,
872 }) catch return true;
878 }) catch |err| switch (err) {
879 error.Canceled => return error.Canceled,
880 else => return true,
881 };
873882 defer file.close();
874883
875884 // Save locally and also save globally (we still hold the global lock).
876 man.recent_problematic_timestamp = (file.stat() catch return true).mtime;
885 const stat = file.stat() catch |err| switch (err) {
886 error.Canceled => return error.Canceled,
887 else => return true,
888 };
889 man.recent_problematic_timestamp = stat.mtime;
877890 man.cache.recent_problematic_timestamp = man.recent_problematic_timestamp;
878891 }
879892
......@@ -900,7 +913,7 @@ pub const Manifest = struct {
900913 .inode = actual_stat.inode,
901914 };
902915
903 if (self.isProblematicTimestamp(ch_file.stat.mtime)) {
916 if (try self.isProblematicTimestamp(ch_file.stat.mtime)) {
904917 // The actual file has an unreliable timestamp, force it to be hashed
905918 ch_file.stat.mtime = .zero;
906919 ch_file.stat.inode = 0;
......@@ -1036,7 +1049,7 @@ pub const Manifest = struct {
10361049 .contents = null,
10371050 };
10381051
1039 if (self.isProblematicTimestamp(new_file.stat.mtime)) {
1052 if (try self.isProblematicTimestamp(new_file.stat.mtime)) {
10401053 // The actual file has an unreliable timestamp, force it to be hashed
10411054 new_file.stat.mtime = .zero;
10421055 new_file.stat.inode = 0;
lib/std/Build/Fuzz.zig+63-58
......@@ -22,17 +22,16 @@ mode: Mode,
2222/// Allocated into `gpa`.
2323run_steps: []const *Step.Run,
2424
25wait_group: std.Thread.WaitGroup,
25group: Io.Group,
2626root_prog_node: std.Progress.Node,
2727prog_node: std.Progress.Node,
28thread_pool: *std.Thread.Pool,
2928
3029/// Protects `coverage_files`.
31coverage_mutex: std.Thread.Mutex,
30coverage_mutex: Io.Mutex,
3231coverage_files: std.AutoArrayHashMapUnmanaged(u64, CoverageMap),
3332
34queue_mutex: std.Thread.Mutex,
35queue_cond: std.Thread.Condition,
33queue_mutex: Io.Mutex,
34queue_cond: Io.Condition,
3635msg_queue: std.ArrayList(Msg),
3736
3837pub const Mode = union(enum) {
......@@ -78,7 +77,6 @@ const CoverageMap = struct {
7877pub fn init(
7978 gpa: Allocator,
8079 io: Io,
81 thread_pool: *std.Thread.Pool,
8280 ttyconf: tty.Config,
8381 all_steps: []const *Build.Step,
8482 root_prog_node: std.Progress.Node,
......@@ -89,20 +87,22 @@ pub fn init(
8987 defer steps.deinit(gpa);
9088 const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
9189 defer rebuild_node.end();
92 var rebuild_wg: std.Thread.WaitGroup = .{};
93 defer rebuild_wg.wait();
90 var rebuild_group: Io.Group = .init;
91 defer rebuild_group.cancel(io);
9492
9593 for (all_steps) |step| {
9694 const run = step.cast(Step.Run) orelse continue;
9795 if (run.producer == null) continue;
9896 if (run.fuzz_tests.items.len == 0) continue;
9997 try steps.append(gpa, run);
100 thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
98 rebuild_group.async(io, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
10199 }
102100
103101 if (steps.items.len == 0) fatal("no fuzz tests found", .{});
104102 rebuild_node.setEstimatedTotalItems(steps.items.len);
105 break :steps try gpa.dupe(*Step.Run, steps.items);
103 const run_steps = try gpa.dupe(*Step.Run, steps.items);
104 rebuild_group.wait(io);
105 break :steps run_steps;
106106 };
107107 errdefer gpa.free(run_steps);
108108
......@@ -118,42 +118,38 @@ pub fn init(
118118 .ttyconf = ttyconf,
119119 .mode = mode,
120120 .run_steps = run_steps,
121 .wait_group = .{},
122 .thread_pool = thread_pool,
121 .group = .init,
123122 .root_prog_node = root_prog_node,
124123 .prog_node = .none,
125124 .coverage_files = .empty,
126 .coverage_mutex = .{},
127 .queue_mutex = .{},
125 .coverage_mutex = .init,
126 .queue_mutex = .init,
128127 .queue_cond = .{},
129128 .msg_queue = .empty,
130129 };
131130}
132131
133132pub fn start(fuzz: *Fuzz) void {
133 const io = fuzz.io;
134134 fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
135135
136136 if (fuzz.mode == .forever) {
137137 // For polling messages and sending updates to subscribers.
138 fuzz.wait_group.start();
139 _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
140 fuzz.wait_group.finish();
141 fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
142 };
138 fuzz.group.concurrent(io, coverageRun, .{fuzz}) catch |err|
139 fatal("unable to spawn coverage task: {t}", .{err});
143140 }
144141
145142 for (fuzz.run_steps) |run| {
146143 for (run.fuzz_tests.items) |unit_test_index| {
147144 assert(run.rebuilt_executable != null);
148 fuzz.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
149 fuzz, run, unit_test_index,
150 });
145 fuzz.group.async(io, fuzzWorkerRun, .{ fuzz, run, unit_test_index });
151146 }
152147 }
153148}
154149
155150pub fn deinit(fuzz: *Fuzz) void {
156 if (!fuzz.wait_group.isDone()) @panic("TODO: terminate the fuzzer processes");
151 const io = fuzz.io;
152 fuzz.group.cancel(io);
157153 fuzz.prog_node.end();
158154 fuzz.gpa.free(fuzz.run_steps);
159155}
......@@ -161,9 +157,7 @@ pub fn deinit(fuzz: *Fuzz) void {
161157fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: tty.Config, parent_prog_node: std.Progress.Node) void {
162158 rebuildTestsWorkerRunFallible(run, gpa, ttyconf, parent_prog_node) catch |err| {
163159 const compile = run.producer.?;
164 log.err("step '{s}': failed to rebuild in fuzz mode: {s}", .{
165 compile.step.name, @errorName(err),
166 });
160 log.err("step '{s}': failed to rebuild in fuzz mode: {t}", .{ compile.step.name, err });
167161 };
168162}
169163
......@@ -212,9 +206,7 @@ fn fuzzWorkerRun(
212206 return;
213207 },
214208 else => {
215 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {s}", .{
216 run.step.name, test_name, @errorName(err),
217 });
209 log.err("step '{s}': failed to rerun '{s}' in fuzz mode: {t}", .{ run.step.name, test_name, err });
218210 return;
219211 },
220212 };
......@@ -273,8 +265,10 @@ pub fn sendUpdate(
273265 socket: *std.http.Server.WebSocket,
274266 prev: *Previous,
275267) !void {
276 fuzz.coverage_mutex.lock();
277 defer fuzz.coverage_mutex.unlock();
268 const io = fuzz.io;
269
270 try fuzz.coverage_mutex.lock(io);
271 defer fuzz.coverage_mutex.unlock(io);
278272
279273 const coverage_maps = fuzz.coverage_files.values();
280274 if (coverage_maps.len == 0) return;
......@@ -335,32 +329,41 @@ pub fn sendUpdate(
335329}
336330
337331fn coverageRun(fuzz: *Fuzz) void {
338 defer fuzz.wait_group.finish();
332 coverageRunCancelable(fuzz) catch |err| switch (err) {
333 error.Canceled => return,
334 };
335}
336
337fn coverageRunCancelable(fuzz: *Fuzz) Io.Cancelable!void {
338 const io = fuzz.io;
339339
340 fuzz.queue_mutex.lock();
341 defer fuzz.queue_mutex.unlock();
340 try fuzz.queue_mutex.lock(io);
341 defer fuzz.queue_mutex.unlock(io);
342342
343343 while (true) {
344 fuzz.queue_cond.wait(&fuzz.queue_mutex);
344 try fuzz.queue_cond.wait(io, &fuzz.queue_mutex);
345345 for (fuzz.msg_queue.items) |msg| switch (msg) {
346346 .coverage => |coverage| prepareTables(fuzz, coverage.run, coverage.id) catch |err| switch (err) {
347347 error.AlreadyReported => continue,
348 else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
348 error.Canceled => return,
349 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
349350 },
350351 .entry_point => |entry_point| addEntryPoint(fuzz, entry_point.coverage_id, entry_point.addr) catch |err| switch (err) {
351352 error.AlreadyReported => continue,
352 else => |e| log.err("failed to prepare code coverage tables: {s}", .{@errorName(e)}),
353 error.Canceled => return,
354 else => |e| log.err("failed to prepare code coverage tables: {t}", .{e}),
353355 },
354356 };
355357 fuzz.msg_queue.clearRetainingCapacity();
356358 }
357359}
358fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void {
360fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported, Canceled }!void {
359361 assert(fuzz.mode == .forever);
360362 const ws = fuzz.mode.forever.ws;
363 const io = fuzz.io;
361364
362 fuzz.coverage_mutex.lock();
363 defer fuzz.coverage_mutex.unlock();
365 try fuzz.coverage_mutex.lock(io);
366 defer fuzz.coverage_mutex.unlock(io);
364367
365368 const gop = try fuzz.coverage_files.getOrPut(fuzz.gpa, coverage_id);
366369 if (gop.found_existing) {
......@@ -391,8 +394,8 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
391394 target.ofmt,
392395 target.cpu.arch,
393396 ) catch |err| {
394 log.err("step '{s}': failed to load debug information for '{f}': {s}", .{
395 run_step.step.name, rebuilt_exe_path, @errorName(err),
397 log.err("step '{s}': failed to load debug information for '{f}': {t}", .{
398 run_step.step.name, rebuilt_exe_path, err,
396399 });
397400 return error.AlreadyReported;
398401 };
......@@ -403,15 +406,15 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
403406 .sub_path = "v/" ++ std.fmt.hex(coverage_id),
404407 };
405408 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
406 log.err("step '{s}': failed to load coverage file '{f}': {s}", .{
407 run_step.step.name, coverage_file_path, @errorName(err),
409 log.err("step '{s}': failed to load coverage file '{f}': {t}", .{
410 run_step.step.name, coverage_file_path, err,
408411 });
409412 return error.AlreadyReported;
410413 };
411414 defer coverage_file.close();
412415
413416 const file_size = coverage_file.getEndPos() catch |err| {
414 log.err("unable to check len of coverage file '{f}': {s}", .{ coverage_file_path, @errorName(err) });
417 log.err("unable to check len of coverage file '{f}': {t}", .{ coverage_file_path, err });
415418 return error.AlreadyReported;
416419 };
417420
......@@ -423,7 +426,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
423426 coverage_file.handle,
424427 0,
425428 ) catch |err| {
426 log.err("failed to map coverage file '{f}': {s}", .{ coverage_file_path, @errorName(err) });
429 log.err("failed to map coverage file '{f}': {t}", .{ coverage_file_path, err });
427430 return error.AlreadyReported;
428431 };
429432 gop.value_ptr.mapped_memory = mapped_memory;
......@@ -449,7 +452,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
449452 }{ .addrs = sorted_pcs.items(.pc) });
450453
451454 debug_info.resolveAddresses(fuzz.gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
452 log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
455 log.err("failed to resolve addresses to source locations: {t}", .{err});
453456 return error.AlreadyReported;
454457 };
455458
......@@ -459,9 +462,11 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
459462 ws.notifyUpdate();
460463}
461464
462fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {
463 fuzz.coverage_mutex.lock();
464 defer fuzz.coverage_mutex.unlock();
465fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory, Canceled }!void {
466 const io = fuzz.io;
467
468 try fuzz.coverage_mutex.lock(io);
469 defer fuzz.coverage_mutex.unlock(io);
465470
466471 const coverage_map = fuzz.coverage_files.getPtr(coverage_id).?;
467472 const header: *const abi.SeenPcsHeader = @ptrCast(coverage_map.mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
......@@ -511,8 +516,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) void {
511516 assert(fuzz.mode == .limit);
512517 const io = fuzz.io;
513518
514 fuzz.wait_group.wait();
515 fuzz.wait_group.reset();
519 fuzz.group.wait(io);
520 fuzz.group = .init;
516521
517522 std.debug.print("======= FUZZING REPORT =======\n", .{});
518523 for (fuzz.msg_queue.items) |msg| {
......@@ -524,8 +529,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) void {
524529 .sub_path = "v/" ++ std.fmt.hex(cov.id),
525530 };
526531 var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
527 fatal("step '{s}': failed to load coverage file '{f}': {s}", .{
528 cov.run.step.name, coverage_file_path, @errorName(err),
532 fatal("step '{s}': failed to load coverage file '{f}': {t}", .{
533 cov.run.step.name, coverage_file_path, err,
529534 });
530535 };
531536 defer coverage_file.close();
......@@ -536,8 +541,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) void {
536541
537542 var header: fuzz_abi.SeenPcsHeader = undefined;
538543 r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
539 fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
540 cov.run.step.name, coverage_file_path, @errorName(err),
544 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
545 cov.run.step.name, coverage_file_path, err,
541546 });
542547 };
543548
......@@ -551,8 +556,8 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) void {
551556 const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
552557 for (0..chunk_count) |_| {
553558 const seen = r.interface.takeInt(usize, .little) catch |err| {
554 fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
555 cov.run.step.name, coverage_file_path, @errorName(err),
559 fatal("step '{s}': failed to read from coverage file '{f}': {t}", .{
560 cov.run.step.name, coverage_file_path, err,
556561 });
557562 };
558563 seen_count += @popCount(seen);
lib/std/Build/Step.zig+19-19
......@@ -110,7 +110,6 @@ pub const TestResults = struct {
110110
111111pub const MakeOptions = struct {
112112 progress_node: std.Progress.Node,
113 thread_pool: *std.Thread.Pool,
114113 watch: bool,
115114 web_server: switch (builtin.target.cpu.arch) {
116115 else => ?*Build.WebServer,
......@@ -363,7 +362,7 @@ pub fn captureChildProcess(
363362 .allocator = arena,
364363 .argv = argv,
365364 .progress_node = progress_node,
366 }) catch |err| return s.fail("failed to run {s}: {s}", .{ argv[0], @errorName(err) });
365 }) catch |err| return s.fail("failed to run {s}: {t}", .{ argv[0], err });
367366
368367 if (result.stderr.len > 0) {
369368 try s.result_error_msgs.append(arena, result.stderr);
......@@ -413,7 +412,7 @@ pub fn evalZigProcess(
413412 error.BrokenPipe => {
414413 // Process restart required.
415414 const term = zp.child.wait() catch |e| {
416 return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) });
415 return s.fail("unable to wait for {s}: {t}", .{ argv[0], e });
417416 };
418417 _ = term;
419418 s.clearZigProcess(gpa);
......@@ -429,7 +428,7 @@ pub fn evalZigProcess(
429428 if (s.result_error_msgs.items.len > 0 and result == null) {
430429 // Crash detected.
431430 const term = zp.child.wait() catch |e| {
432 return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(e) });
431 return s.fail("unable to wait for {s}: {t}", .{ argv[0], e });
433432 };
434433 s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
435434 s.clearZigProcess(gpa);
......@@ -454,9 +453,7 @@ pub fn evalZigProcess(
454453 child.request_resource_usage_statistics = true;
455454 child.progress_node = prog_node;
456455
457 child.spawn() catch |err| return s.fail("failed to spawn zig compiler {s}: {s}", .{
458 argv[0], @errorName(err),
459 });
456 child.spawn() catch |err| return s.fail("failed to spawn zig compiler {s}: {t}", .{ argv[0], err });
460457
461458 const zp = try gpa.create(ZigProcess);
462459 zp.* = .{
......@@ -481,7 +478,7 @@ pub fn evalZigProcess(
481478 zp.child.stdin = null;
482479
483480 const term = zp.child.wait() catch |err| {
484 return s.fail("unable to wait for {s}: {s}", .{ argv[0], @errorName(err) });
481 return s.fail("unable to wait for {s}: {t}", .{ argv[0], err });
485482 };
486483 s.result_peak_rss = zp.child.resource_usage_statistics.getMaxRss() orelse 0;
487484
......@@ -514,8 +511,8 @@ pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u
514511 const src_path = src_lazy_path.getPath3(b, s);
515512 try handleVerbose(b, null, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path });
516513 return Io.Dir.updateFile(src_path.root_dir.handle.adaptToNewApi(), io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| {
517 return s.fail("unable to update file from '{f}' to '{s}': {s}", .{
518 src_path, dest_path, @errorName(err),
514 return s.fail("unable to update file from '{f}' to '{s}': {t}", .{
515 src_path, dest_path, err,
519516 });
520517 };
521518}
......@@ -525,9 +522,7 @@ pub fn installDir(s: *Step, dest_path: []const u8) !std.fs.Dir.MakePathStatus {
525522 const b = s.owner;
526523 try handleVerbose(b, null, &.{ "install", "-d", dest_path });
527524 return std.fs.cwd().makePathStatus(dest_path) catch |err| {
528 return s.fail("unable to create dir '{s}': {s}", .{
529 dest_path, @errorName(err),
530 });
525 return s.fail("unable to create dir '{s}': {t}", .{ dest_path, err });
531526 };
532527}
533528
......@@ -826,22 +821,27 @@ pub fn cacheHitAndWatch(s: *Step, man: *Build.Cache.Manifest) !bool {
826821 return is_hit;
827822}
828823
829fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: Build.Cache.Manifest.HitError) error{ OutOfMemory, MakeFailed } {
824fn failWithCacheError(
825 s: *Step,
826 man: *const Build.Cache.Manifest,
827 err: Build.Cache.Manifest.HitError,
828) error{ OutOfMemory, Canceled, MakeFailed } {
830829 switch (err) {
831830 error.CacheCheckFailed => switch (man.diagnostic) {
832831 .none => unreachable,
833 .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {s} {s}", .{
834 @tagName(man.diagnostic), @errorName(e),
832 .manifest_create, .manifest_read, .manifest_lock => |e| return s.fail("failed to check cache: {t} {t}", .{
833 man.diagnostic, e,
835834 }),
836835 .file_open, .file_stat, .file_read, .file_hash => |op| {
837836 const pp = man.files.keys()[op.file_index].prefixed_path;
838837 const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
839 return s.fail("failed to check cache: '{s}{c}{s}' {s} {s}", .{
840 prefix, std.fs.path.sep, pp.sub_path, @tagName(man.diagnostic), @errorName(op.err),
838 return s.fail("failed to check cache: '{s}{c}{s}' {t} {t}", .{
839 prefix, std.fs.path.sep, pp.sub_path, man.diagnostic, op.err,
841840 });
842841 },
843842 },
844843 error.OutOfMemory => return error.OutOfMemory,
844 error.Canceled => return error.Canceled,
845845 error.InvalidFormat => return s.fail("failed to check cache: invalid manifest file format", .{}),
846846 }
847847}
......@@ -851,7 +851,7 @@ fn failWithCacheError(s: *Step, man: *const Build.Cache.Manifest, err: Build.Cac
851851pub fn writeManifest(s: *Step, man: *Build.Cache.Manifest) !void {
852852 if (s.test_results.isSuccess()) {
853853 man.writeManifest() catch |err| {
854 try s.addError("unable to write cache manifest: {s}", .{@errorName(err)});
854 try s.addError("unable to write cache manifest: {t}", .{err});
855855 };
856856 }
857857}
lib/std/Build/Step/Run.zig+7-7
......@@ -1151,7 +1151,6 @@ pub fn rerunInFuzzMode(
11511151 const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int);
11521152 try runCommand(run, argv_list.items, has_side_effects, tmp_dir_path, .{
11531153 .progress_node = prog_node,
1154 .thread_pool = undefined, // not used by `runCommand`
11551154 .watch = undefined, // not used by `runCommand`
11561155 .web_server = null, // only needed for time reports
11571156 .ttyconf = fuzz.ttyconf,
......@@ -1831,6 +1830,7 @@ fn pollZigTest(
18311830} {
18321831 const gpa = run.step.owner.allocator;
18331832 const arena = run.step.owner.allocator;
1833 const io = run.step.owner.graph.io;
18341834
18351835 var sub_prog_node: ?std.Progress.Node = null;
18361836 defer if (sub_prog_node) |n| n.end();
......@@ -2036,8 +2036,8 @@ fn pollZigTest(
20362036
20372037 {
20382038 const fuzz = fuzz_context.?.fuzz;
2039 fuzz.queue_mutex.lock();
2040 defer fuzz.queue_mutex.unlock();
2039 fuzz.queue_mutex.lockUncancelable(io);
2040 defer fuzz.queue_mutex.unlock(io);
20412041 try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
20422042 .id = coverage_id.?,
20432043 .cumulative = .{
......@@ -2047,20 +2047,20 @@ fn pollZigTest(
20472047 },
20482048 .run = run,
20492049 } });
2050 fuzz.queue_cond.signal();
2050 fuzz.queue_cond.signal(io);
20512051 }
20522052 },
20532053 .fuzz_start_addr => {
20542054 const fuzz = fuzz_context.?.fuzz;
20552055 const addr = body_r.takeInt(u64, .little) catch unreachable;
20562056 {
2057 fuzz.queue_mutex.lock();
2058 defer fuzz.queue_mutex.unlock();
2057 fuzz.queue_mutex.lockUncancelable(io);
2058 defer fuzz.queue_mutex.unlock(io);
20592059 try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
20602060 .addr = addr,
20612061 .coverage_id = coverage_id.?,
20622062 } });
2063 fuzz.queue_cond.signal();
2063 fuzz.queue_cond.signal(io);
20642064 }
20652065 },
20662066 else => {}, // ignore other messages
lib/std/Build/WebServer.zig+44-37
......@@ -1,5 +1,4 @@
11gpa: Allocator,
2thread_pool: *std.Thread.Pool,
32graph: *const Build.Graph,
43all_steps: []const *Build.Step,
54listen_address: net.IpAddress,
......@@ -20,7 +19,7 @@ step_names_trailing: []u8,
2019step_status_bits: []u8,
2120
2221fuzz: ?Fuzz,
23time_report_mutex: std.Thread.Mutex,
22time_report_mutex: Io.Mutex,
2423time_report_msgs: [][]u8,
2524time_report_update_times: []i64,
2625
......@@ -34,9 +33,9 @@ build_status: std.atomic.Value(abi.BuildStatus),
3433/// an unreasonable number of packets.
3534update_id: std.atomic.Value(u32),
3635
37runner_request_mutex: std.Thread.Mutex,
38runner_request_ready_cond: std.Thread.Condition,
39runner_request_empty_cond: std.Thread.Condition,
36runner_request_mutex: Io.Mutex,
37runner_request_ready_cond: Io.Condition,
38runner_request_empty_cond: Io.Condition,
4039runner_request: ?RunnerRequest,
4140
4241/// If a client is not explicitly notified of changes with `notifyUpdate`, it will be sent updates
......@@ -53,7 +52,6 @@ pub fn notifyUpdate(ws: *WebServer) void {
5352
5453pub const Options = struct {
5554 gpa: Allocator,
56 thread_pool: *std.Thread.Pool,
5755 ttyconf: Io.tty.Config,
5856 graph: *const std.Build.Graph,
5957 all_steps: []const *Build.Step,
......@@ -100,7 +98,6 @@ pub fn init(opts: Options) WebServer {
10098
10199 return .{
102100 .gpa = opts.gpa,
103 .thread_pool = opts.thread_pool,
104101 .ttyconf = opts.ttyconf,
105102 .graph = opts.graph,
106103 .all_steps = all_steps,
......@@ -117,14 +114,14 @@ pub fn init(opts: Options) WebServer {
117114 .step_status_bits = step_status_bits,
118115
119116 .fuzz = null,
120 .time_report_mutex = .{},
117 .time_report_mutex = .init,
121118 .time_report_msgs = time_report_msgs,
122119 .time_report_update_times = time_report_update_times,
123120
124121 .build_status = .init(.idle),
125122 .update_id = .init(0),
126123
127 .runner_request_mutex = .{},
124 .runner_request_mutex = .init,
128125 .runner_request_ready_cond = .{},
129126 .runner_request_empty_cond = .{},
130127 .runner_request = null,
......@@ -235,7 +232,6 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
235232 ws.fuzz = Fuzz.init(
236233 ws.gpa,
237234 ws.graph.io,
238 ws.thread_pool,
239235 ws.ttyconf,
240236 ws.all_steps,
241237 ws.root_prog_node,
......@@ -300,6 +296,8 @@ fn accept(ws: *WebServer, stream: net.Stream) void {
300296}
301297
302298fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn {
299 const io = ws.graph.io;
300
303301 var prev_build_status = ws.build_status.load(.monotonic);
304302
305303 const prev_step_status_bits = try ws.gpa.alloc(u8, ws.step_status_bits.len);
......@@ -335,8 +333,8 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn {
335333 }
336334
337335 {
338 ws.time_report_mutex.lock();
339 defer ws.time_report_mutex.unlock();
336 try ws.time_report_mutex.lock(io);
337 defer ws.time_report_mutex.unlock(io);
340338 for (ws.time_report_msgs, ws.time_report_update_times) |msg, update_time| {
341339 if (update_time <= prev_time) continue;
342340 // We want to send `msg`, but shouldn't block `ws.time_report_mutex` while we do, so
......@@ -344,8 +342,8 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn {
344342 const owned_msg = try ws.gpa.dupe(u8, msg);
345343 defer ws.gpa.free(owned_msg);
346344 // Temporarily unlock, then re-lock after the message is sent.
347 ws.time_report_mutex.unlock();
348 defer ws.time_report_mutex.lock();
345 ws.time_report_mutex.unlock(io);
346 defer ws.time_report_mutex.lockUncancelable(io);
349347 try sock.writeMessage(owned_msg, .binary);
350348 }
351349 }
......@@ -386,6 +384,8 @@ fn serveWebSocket(ws: *WebServer, sock: *http.Server.WebSocket) !noreturn {
386384 }
387385}
388386fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void {
387 const io = ws.graph.io;
388
389389 while (true) {
390390 const msg = sock.readSmallMessage() catch return;
391391 if (msg.opcode != .binary) continue;
......@@ -394,14 +394,16 @@ fn recvWebSocketMessages(ws: *WebServer, sock: *http.Server.WebSocket) void {
394394 switch (tag) {
395395 _ => continue,
396396 .rebuild => while (true) {
397 ws.runner_request_mutex.lock();
398 defer ws.runner_request_mutex.unlock();
397 ws.runner_request_mutex.lock(io) catch |err| switch (err) {
398 error.Canceled => return,
399 };
400 defer ws.runner_request_mutex.unlock(io);
399401 if (ws.runner_request == null) {
400402 ws.runner_request = .rebuild;
401 ws.runner_request_ready_cond.signal();
403 ws.runner_request_ready_cond.signal(io);
402404 break;
403405 }
404 ws.runner_request_empty_cond.wait(&ws.runner_request_mutex);
406 ws.runner_request_empty_cond.wait(io, &ws.runner_request_mutex) catch return;
405407 },
406408 }
407409 }
......@@ -695,14 +697,15 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
695697 trailing: []const u8,
696698}) void {
697699 const gpa = ws.gpa;
700 const io = ws.graph.io;
698701
699702 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
700703 if (s == &opts.compile.step) break @intCast(i);
701704 } else unreachable;
702705
703706 const old_buf = old: {
704 ws.time_report_mutex.lock();
705 defer ws.time_report_mutex.unlock();
707 ws.time_report_mutex.lock(io) catch return;
708 defer ws.time_report_mutex.unlock(io);
706709 const old = ws.time_report_msgs[step_idx];
707710 ws.time_report_msgs[step_idx] = &.{};
708711 break :old old;
......@@ -724,8 +727,8 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
724727 @memcpy(buf[@sizeOf(abi.time_report.CompileResult)..], opts.trailing);
725728
726729 {
727 ws.time_report_mutex.lock();
728 defer ws.time_report_mutex.unlock();
730 ws.time_report_mutex.lock(io) catch return;
731 defer ws.time_report_mutex.unlock(io);
729732 assert(ws.time_report_msgs[step_idx].len == 0);
730733 ws.time_report_msgs[step_idx] = buf;
731734 ws.time_report_update_times[step_idx] = ws.now();
......@@ -735,14 +738,15 @@ pub fn updateTimeReportCompile(ws: *WebServer, opts: struct {
735738
736739pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, ns_total: u64) void {
737740 const gpa = ws.gpa;
741 const io = ws.graph.io;
738742
739743 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
740744 if (s == step) break @intCast(i);
741745 } else unreachable;
742746
743747 const old_buf = old: {
744 ws.time_report_mutex.lock();
745 defer ws.time_report_mutex.unlock();
748 ws.time_report_mutex.lock(io) catch return;
749 defer ws.time_report_mutex.unlock(io);
746750 const old = ws.time_report_msgs[step_idx];
747751 ws.time_report_msgs[step_idx] = &.{};
748752 break :old old;
......@@ -754,8 +758,8 @@ pub fn updateTimeReportGeneric(ws: *WebServer, step: *Build.Step, ns_total: u64)
754758 .ns_total = ns_total,
755759 };
756760 {
757 ws.time_report_mutex.lock();
758 defer ws.time_report_mutex.unlock();
761 ws.time_report_mutex.lock(io) catch return;
762 defer ws.time_report_mutex.unlock(io);
759763 assert(ws.time_report_msgs[step_idx].len == 0);
760764 ws.time_report_msgs[step_idx] = buf;
761765 ws.time_report_update_times[step_idx] = ws.now();
......@@ -770,6 +774,7 @@ pub fn updateTimeReportRunTest(
770774 ns_per_test: []const u64,
771775) void {
772776 const gpa = ws.gpa;
777 const io = ws.graph.io;
773778
774779 const step_idx: u32 = for (ws.all_steps, 0..) |s, i| {
775780 if (s == &run.step) break @intCast(i);
......@@ -786,8 +791,8 @@ pub fn updateTimeReportRunTest(
786791 break :len @sizeOf(abi.time_report.RunTestResult) + names_len + 8 * tests_len;
787792 };
788793 const old_buf = old: {
789 ws.time_report_mutex.lock();
790 defer ws.time_report_mutex.unlock();
794 ws.time_report_mutex.lock(io) catch return;
795 defer ws.time_report_mutex.unlock(io);
791796 const old = ws.time_report_msgs[step_idx];
792797 ws.time_report_msgs[step_idx] = &.{};
793798 break :old old;
......@@ -812,8 +817,8 @@ pub fn updateTimeReportRunTest(
812817 assert(offset == buf.len);
813818
814819 {
815 ws.time_report_mutex.lock();
816 defer ws.time_report_mutex.unlock();
820 ws.time_report_mutex.lock(io) catch return;
821 defer ws.time_report_mutex.unlock(io);
817822 assert(ws.time_report_msgs[step_idx].len == 0);
818823 ws.time_report_msgs[step_idx] = buf;
819824 ws.time_report_update_times[step_idx] = ws.now();
......@@ -825,8 +830,9 @@ const RunnerRequest = union(enum) {
825830 rebuild,
826831};
827832pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest {
828 ws.runner_request_mutex.lock();
829 defer ws.runner_request_mutex.unlock();
833 const io = ws.graph.io;
834 ws.runner_request_mutex.lock(io) catch return;
835 defer ws.runner_request_mutex.unlock(io);
830836 if (ws.runner_request) |req| {
831837 ws.runner_request = null;
832838 ws.runner_request_empty_cond.signal();
......@@ -834,16 +840,17 @@ pub fn getRunnerRequest(ws: *WebServer) ?RunnerRequest {
834840 }
835841 return null;
836842}
837pub fn wait(ws: *WebServer) RunnerRequest {
838 ws.runner_request_mutex.lock();
839 defer ws.runner_request_mutex.unlock();
843pub fn wait(ws: *WebServer) Io.Cancelable!RunnerRequest {
844 const io = ws.graph.io;
845 try ws.runner_request_mutex.lock(io);
846 defer ws.runner_request_mutex.unlock(io);
840847 while (true) {
841848 if (ws.runner_request) |req| {
842849 ws.runner_request = null;
843 ws.runner_request_empty_cond.signal();
850 ws.runner_request_empty_cond.signal(io);
844851 return req;
845852 }
846 ws.runner_request_ready_cond.wait(&ws.runner_request_mutex);
853 try ws.runner_request_ready_cond.wait(io, &ws.runner_request_mutex);
847854 }
848855}
849856
lib/std/Io/Threaded.zig+11-3
......@@ -33,6 +33,9 @@ wait_group: std.Thread.WaitGroup = .{},
3333/// immediately.
3434///
3535/// Defaults to a number equal to logical CPU cores.
36///
37/// Protected by `mutex` once the I/O instance is already in use. See
38/// `setAsyncLimit`.
3639async_limit: Io.Limit,
3740/// Maximum thread pool size (excluding main thread) for dispatching concurrent
3841/// tasks. Until this limit, calls to `Io.concurrent` will increase the thread
......@@ -168,6 +171,12 @@ pub const init_single_threaded: Threaded = .{
168171 .have_signal_handler = false,
169172};
170173
174pub fn setAsyncLimit(t: *Threaded, new_limit: Io.Limit) void {
175 t.mutex.lock();
176 defer t.mutex.unlock();
177 t.async_limit = new_limit;
178}
179
171180pub fn deinit(t: *Threaded) void {
172181 t.join();
173182 if (is_windows and t.wsa.status == .initialized) {
......@@ -507,7 +516,7 @@ fn async(
507516 start: *const fn (context: *const anyopaque, result: *anyopaque) void,
508517) ?*Io.AnyFuture {
509518 const t: *Threaded = @ptrCast(@alignCast(userdata));
510 if (builtin.single_threaded or t.async_limit == .nothing) {
519 if (builtin.single_threaded) {
511520 start(context.ptr, result.ptr);
512521 return null;
513522 }
......@@ -684,8 +693,7 @@ fn groupAsync(
684693 start: *const fn (*Io.Group, context: *const anyopaque) void,
685694) void {
686695 const t: *Threaded = @ptrCast(@alignCast(userdata));
687 if (builtin.single_threaded or t.async_limit == .nothing)
688 return start(group, context.ptr);
696 if (builtin.single_threaded) return start(group, context.ptr);
689697
690698 const gpa = t.allocator;
691699 const gc = GroupClosure.init(gpa, t, group, context, context_alignment, start) catch
src/Compilation.zig+15-15
......@@ -2851,6 +2851,7 @@ fn cleanupAfterUpdate(comp: *Compilation, tmp_dir_rand_int: u64) void {
28512851
28522852pub const UpdateError = error{
28532853 OutOfMemory,
2854 Canceled,
28542855 Unexpected,
28552856 CurrentWorkingDirectoryUnlinked,
28562857};
......@@ -2930,6 +2931,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) UpdateE
29302931 },
29312932 },
29322933 error.OutOfMemory => return error.OutOfMemory,
2934 error.Canceled => return error.Canceled,
29332935 error.InvalidFormat => return comp.setMiscFailure(
29342936 .check_whole_cache,
29352937 "failed to check cache: invalid manifest file format",
......@@ -5010,7 +5012,7 @@ fn performAllTheWork(
50105012 }
50115013}
50125014
5013const JobError = Allocator.Error;
5015const JobError = Allocator.Error || Io.Cancelable;
50145016
50155017pub fn queueJob(comp: *Compilation, job: Job) !void {
50165018 try comp.work_queues[Job.stage(job)].pushBack(comp.gpa, job);
......@@ -5117,6 +5119,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
51175119
51185120 pt.ensureFuncBodyUpToDate(func) catch |err| switch (err) {
51195121 error.OutOfMemory => |e| return e,
5122 error.Canceled => |e| return e,
51205123 error.AnalysisFail => return,
51215124 };
51225125 },
......@@ -5137,6 +5140,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
51375140 };
51385141 maybe_err catch |err| switch (err) {
51395142 error.OutOfMemory => |e| return e,
5143 error.Canceled => |e| return e,
51405144 error.AnalysisFail => return,
51415145 };
51425146
......@@ -5166,7 +5170,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
51665170 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
51675171 defer pt.deactivate();
51685172 Type.fromInterned(ty).resolveFully(pt) catch |err| switch (err) {
5169 error.OutOfMemory => return error.OutOfMemory,
5173 error.OutOfMemory, error.Canceled => |e| return e,
51705174 error.AnalysisFail => return,
51715175 };
51725176 },
......@@ -5177,7 +5181,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
51775181 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
51785182 defer pt.deactivate();
51795183 pt.semaMod(mod) catch |err| switch (err) {
5180 error.OutOfMemory => return error.OutOfMemory,
5184 error.OutOfMemory, error.Canceled => |e| return e,
51815185 error.AnalysisFail => return,
51825186 };
51835187 },
......@@ -5190,8 +5194,8 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
51905194 // TODO Surface more error details.
51915195 comp.lockAndSetMiscFailure(
51925196 .windows_import_lib,
5193 "unable to generate DLL import .lib file for {s}: {s}",
5194 .{ link_lib, @errorName(err) },
5197 "unable to generate DLL import .lib file for {s}: {t}",
5198 .{ link_lib, err },
51955199 );
51965200 };
51975201 },
......@@ -6066,14 +6070,10 @@ fn buildLibZigC(comp: *Compilation, prog_node: std.Progress.Node) void {
60666070 };
60676071}
60686072
6069fn reportRetryableCObjectError(
6070 comp: *Compilation,
6071 c_object: *CObject,
6072 err: anyerror,
6073) error{OutOfMemory}!void {
6073fn reportRetryableCObjectError(comp: *Compilation, c_object: *CObject, err: anyerror) error{OutOfMemory}!void {
60746074 c_object.status = .failure_retryable;
60756075
6076 switch (comp.failCObj(c_object, "{s}", .{@errorName(err)})) {
6076 switch (comp.failCObj(c_object, "{t}", .{err})) {
60776077 error.AnalysisFail => return,
60786078 else => |e| return e,
60796079 }
......@@ -7317,7 +7317,7 @@ fn failCObj(
73177317 c_object: *CObject,
73187318 comptime format: []const u8,
73197319 args: anytype,
7320) SemaError {
7320) error{ OutOfMemory, AnalysisFail } {
73217321 @branchHint(.cold);
73227322 const diag_bundle = blk: {
73237323 const diag_bundle = try comp.gpa.create(CObject.Diag.Bundle);
......@@ -7341,7 +7341,7 @@ fn failCObjWithOwnedDiagBundle(
73417341 comp: *Compilation,
73427342 c_object: *CObject,
73437343 diag_bundle: *CObject.Diag.Bundle,
7344) SemaError {
7344) error{ OutOfMemory, AnalysisFail } {
73457345 @branchHint(.cold);
73467346 assert(diag_bundle.diags.len > 0);
73477347 {
......@@ -7357,7 +7357,7 @@ fn failCObjWithOwnedDiagBundle(
73577357 return error.AnalysisFail;
73587358}
73597359
7360fn failWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, comptime format: []const u8, args: anytype) SemaError {
7360fn failWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, comptime format: []const u8, args: anytype) error{ OutOfMemory, AnalysisFail } {
73617361 @branchHint(.cold);
73627362 var bundle: ErrorBundle.Wip = undefined;
73637363 try bundle.init(comp.gpa);
......@@ -7384,7 +7384,7 @@ fn failWin32ResourceWithOwnedBundle(
73847384 comp: *Compilation,
73857385 win32_resource: *Win32Resource,
73867386 err_bundle: ErrorBundle,
7387) SemaError {
7387) error{ OutOfMemory, AnalysisFail } {
73887388 @branchHint(.cold);
73897389 {
73907390 comp.mutex.lock();
src/Sema.zig+8-6
......@@ -6696,7 +6696,7 @@ pub fn analyzeSaveErrRetIndex(sema: *Sema, block: *Block) SemaError!Air.Inst.Ref
66966696 const field_index = sema.structFieldIndex(block, stack_trace_ty, field_name, LazySrcLoc.unneeded) catch |err| switch (err) {
66976697 error.AnalysisFail => @panic("std.builtin.StackTrace is corrupt"),
66986698 error.ComptimeReturn, error.ComptimeBreak => unreachable,
6699 error.OutOfMemory => |e| return e,
6699 error.OutOfMemory, error.Canceled => |e| return e,
67006700 };
67016701
67026702 return try block.addInst(.{
......@@ -13924,6 +13924,7 @@ fn zirEmbedFile(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A
1392413924 return sema.fail(block, operand_src, "unable to resolve '{s}': working directory has been unlinked", .{name});
1392513925 },
1392613926 error.OutOfMemory => |e| return e,
13927 error.Canceled => |e| return e,
1392713928 };
1392813929 try sema.declareDependency(.{ .embed_file = ef_idx });
1392913930
......@@ -34345,7 +34346,7 @@ pub fn resolveStructLayout(sema: *Sema, ty: Type) SemaError!void {
3434534346
3434634347 if (struct_type.layout == .@"packed") {
3434734348 sema.backingIntType(struct_type) catch |err| switch (err) {
34348 error.OutOfMemory, error.AnalysisFail => |e| return e,
34349 error.AnalysisFail, error.OutOfMemory, error.Canceled => |e| return e,
3434934350 error.ComptimeBreak, error.ComptimeReturn => unreachable,
3435034351 };
3435134352 return;
......@@ -34893,7 +34894,7 @@ pub fn resolveStructFieldTypes(
3489334894 defer tracked_unit.end(zcu);
3489434895
3489534896 sema.structFields(struct_type) catch |err| switch (err) {
34896 error.AnalysisFail, error.OutOfMemory => |e| return e,
34897 error.AnalysisFail, error.OutOfMemory, error.Canceled => |e| return e,
3489734898 error.ComptimeBreak, error.ComptimeReturn => unreachable,
3489834899 };
3489934900}
......@@ -34926,7 +34927,7 @@ pub fn resolveStructFieldInits(sema: *Sema, ty: Type) SemaError!void {
3492634927 defer tracked_unit.end(zcu);
3492734928
3492834929 sema.structFieldInits(struct_type) catch |err| switch (err) {
34929 error.AnalysisFail, error.OutOfMemory => |e| return e,
34930 error.AnalysisFail, error.OutOfMemory, error.Canceled => |e| return e,
3493034931 error.ComptimeBreak, error.ComptimeReturn => unreachable,
3493134932 };
3493234933 struct_type.setHaveFieldInits(ip);
......@@ -34960,7 +34961,7 @@ pub fn resolveUnionFieldTypes(sema: *Sema, ty: Type, union_type: InternPool.Load
3496034961 union_type.setStatus(ip, .field_types_wip);
3496134962 errdefer union_type.setStatus(ip, .none);
3496234963 sema.unionFields(ty.toIntern(), union_type) catch |err| switch (err) {
34963 error.AnalysisFail, error.OutOfMemory => |e| return e,
34964 error.AnalysisFail, error.OutOfMemory, error.Canceled => |e| return e,
3496434965 error.ComptimeBreak, error.ComptimeReturn => unreachable,
3496534966 };
3496634967 union_type.setStatus(ip, .have_field_types);
......@@ -37027,6 +37028,7 @@ fn notePathToComptimeAllocPtr(
3702737028
3702837029 const derivation = comptime_ptr.pointerDerivationAdvanced(arena, pt, false, sema) catch |err| switch (err) {
3702937030 error.OutOfMemory => |e| return e,
37031 error.Canceled => @panic("TODO"), // pls don't be cancelable mlugg
3703037032 error.AnalysisFail => unreachable,
3703137033 };
3703237034
......@@ -37367,7 +37369,7 @@ pub fn resolveDeclaredEnum(
3736737369 ) catch |err| switch (err) {
3736837370 error.ComptimeBreak => unreachable,
3736937371 error.ComptimeReturn => unreachable,
37370 error.OutOfMemory => |e| return e,
37372 error.OutOfMemory, error.Canceled => |e| return e,
3737137373 error.AnalysisFail => {
3737237374 if (!zcu.failed_analysis.contains(sema.owner)) {
3737337375 try zcu.transitive_failed_analysis.put(gpa, sema.owner, {});
src/Type.zig+2-1
......@@ -3837,7 +3837,7 @@ fn resolveStructInner(
38373837 }
38383838 return error.AnalysisFail;
38393839 },
3840 error.OutOfMemory => |e| return e,
3840 error.OutOfMemory, error.Canceled => |e| return e,
38413841 };
38423842}
38433843
......@@ -3896,6 +3896,7 @@ fn resolveUnionInner(
38963896 return error.AnalysisFail;
38973897 },
38983898 error.OutOfMemory => |e| return e,
3899 error.Canceled => |e| return e,
38993900 };
39003901}
39013902
src/Value.zig+7-3
......@@ -1,12 +1,15 @@
1const std = @import("std");
2const builtin = @import("builtin");
31const build_options = @import("build_options");
4const Type = @import("Type.zig");
2const builtin = @import("builtin");
3
4const std = @import("std");
5const Io = std.Io;
56const assert = std.debug.assert;
67const BigIntConst = std.math.big.int.Const;
78const BigIntMutable = std.math.big.int.Mutable;
89const Target = std.Target;
910const Allocator = std.mem.Allocator;
11
12const Type = @import("Type.zig");
1013const Zcu = @import("Zcu.zig");
1114const Sema = @import("Sema.zig");
1215const InternPool = @import("InternPool.zig");
......@@ -2410,6 +2413,7 @@ pub const PointerDeriveStep = union(enum) {
24102413pub fn pointerDerivation(ptr_val: Value, arena: Allocator, pt: Zcu.PerThread) Allocator.Error!PointerDeriveStep {
24112414 return ptr_val.pointerDerivationAdvanced(arena, pt, false, null) catch |err| switch (err) {
24122415 error.OutOfMemory => |e| return e,
2416 error.Canceled => @panic("TODO"), // pls remove from error set mlugg
24132417 error.AnalysisFail => unreachable,
24142418 };
24152419}
src/Zcu.zig+3-1
......@@ -2755,9 +2755,11 @@ pub const LazySrcLoc = struct {
27552755 }
27562756};
27572757
2758pub const SemaError = error{ OutOfMemory, AnalysisFail };
2758pub const SemaError = error{ OutOfMemory, Canceled, AnalysisFail };
27592759pub const CompileError = error{
27602760 OutOfMemory,
2761 /// The compilation update is no longer desired.
2762 Canceled,
27612763 /// When this is returned, the compile error for the failure has already been recorded.
27622764 AnalysisFail,
27632765 /// In a comptime scope, a return instruction was encountered. This error is only seen when
src/Zcu/PerThread.zig+18-9
......@@ -1,26 +1,31 @@
11//! This type provides a wrapper around a `*Zcu` for uses which require a thread `Id`.
22//! Any operation which mutates `InternPool` state lives here rather than on `Zcu`.
33
4const Air = @import("../Air.zig");
4const std = @import("std");
55const Allocator = std.mem.Allocator;
66const assert = std.debug.assert;
77const Ast = std.zig.Ast;
88const AstGen = std.zig.AstGen;
99const BigIntConst = std.math.big.int.Const;
1010const BigIntMutable = std.math.big.int.Mutable;
11const Cache = std.Build.Cache;
12const log = std.log.scoped(.zcu);
13const mem = std.mem;
14const Zir = std.zig.Zir;
15const Zoir = std.zig.Zoir;
16const ZonGen = std.zig.ZonGen;
17const Io = std.Io;
18
19const Air = @import("../Air.zig");
1120const Builtin = @import("../Builtin.zig");
1221const build_options = @import("build_options");
1322const builtin = @import("builtin");
14const Cache = std.Build.Cache;
1523const dev = @import("../dev.zig");
1624const InternPool = @import("../InternPool.zig");
1725const AnalUnit = InternPool.AnalUnit;
1826const introspect = @import("../introspect.zig");
19const log = std.log.scoped(.zcu);
2027const Module = @import("../Package.zig").Module;
2128const Sema = @import("../Sema.zig");
22const std = @import("std");
23const mem = std.mem;
2429const target_util = @import("../target.zig");
2530const trace = @import("../tracy.zig").trace;
2631const Type = @import("../Type.zig");
......@@ -29,9 +34,6 @@ const Zcu = @import("../Zcu.zig");
2934const Compilation = @import("../Compilation.zig");
3035const codegen = @import("../codegen.zig");
3136const crash_report = @import("../crash_report.zig");
32const Zir = std.zig.Zir;
33const Zoir = std.zig.Zoir;
34const ZonGen = std.zig.ZonGen;
3537
3638zcu: *Zcu,
3739
......@@ -678,6 +680,7 @@ pub fn ensureMemoizedStateUpToDate(pt: Zcu.PerThread, stage: InternPool.Memoized
678680 // TODO: same as for `ensureComptimeUnitUpToDate` etc
679681 return error.OutOfMemory;
680682 },
683 error.Canceled => |e| return e,
681684 error.ComptimeReturn => unreachable,
682685 error.ComptimeBreak => unreachable,
683686 };
......@@ -842,6 +845,7 @@ pub fn ensureComptimeUnitUpToDate(pt: Zcu.PerThread, cu_id: InternPool.ComptimeU
842845 // for reporting OOM errors without allocating.
843846 return error.OutOfMemory;
844847 },
848 error.Canceled => |e| return e,
845849 error.ComptimeReturn => unreachable,
846850 error.ComptimeBreak => unreachable,
847851 };
......@@ -1030,6 +1034,7 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu
10301034 // for reporting OOM errors without allocating.
10311035 return error.OutOfMemory;
10321036 },
1037 error.Canceled => |e| return e,
10331038 error.ComptimeReturn => unreachable,
10341039 error.ComptimeBreak => unreachable,
10351040 };
......@@ -1443,6 +1448,7 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc
14431448 // for reporting OOM errors without allocating.
14441449 return error.OutOfMemory;
14451450 },
1451 error.Canceled => |e| return e,
14461452 error.ComptimeReturn => unreachable,
14471453 error.ComptimeBreak => unreachable,
14481454 };
......@@ -1668,6 +1674,7 @@ pub fn ensureFuncBodyUpToDate(pt: Zcu.PerThread, func_index: InternPool.Index) Z
16681674 // for reporting OOM errors without allocating.
16691675 return error.OutOfMemory;
16701676 },
1677 error.Canceled => |e| return e,
16711678 };
16721679
16731680 if (was_outdated) {
......@@ -2360,6 +2367,7 @@ pub fn embedFile(
23602367 import_string: []const u8,
23612368) error{
23622369 OutOfMemory,
2370 Canceled,
23632371 ImportOutsideModulePath,
23642372 CurrentWorkingDirectoryUnlinked,
23652373}!Zcu.EmbedFile.Index {
......@@ -4123,7 +4131,7 @@ fn recreateEnumType(
41234131 pt: Zcu.PerThread,
41244132 old_ty: InternPool.Index,
41254133 key: InternPool.Key.NamespaceType.Declared,
4126) Allocator.Error!InternPool.Index {
4134) (Allocator.Error || Io.Cancelable)!InternPool.Index {
41274135 const zcu = pt.zcu;
41284136 const gpa = zcu.gpa;
41294137 const ip = &zcu.intern_pool;
......@@ -4234,6 +4242,7 @@ fn recreateEnumType(
42344242 body_end,
42354243 ) catch |err| switch (err) {
42364244 error.OutOfMemory => |e| return e,
4245 error.Canceled => |e| return e,
42374246 error.AnalysisFail => {}, // call sites are responsible for checking `[transitive_]failed_analysis` to detect this
42384247 };
42394248
src/print_value.zig+2
......@@ -27,6 +27,7 @@ pub fn formatSema(ctx: FormatContext, writer: *Writer) Writer.Error!void {
2727 error.OutOfMemory => @panic("OOM"), // We're not allowed to return this from a format function
2828 error.ComptimeBreak, error.ComptimeReturn => unreachable,
2929 error.AnalysisFail => unreachable, // TODO: re-evaluate when we use `sema` more fully
30 error.Canceled => @panic("TODO"), // pls stop returning this error mlugg
3031 else => |e| return e,
3132 };
3233}
......@@ -36,6 +37,7 @@ pub fn format(ctx: FormatContext, writer: *Writer) Writer.Error!void {
3637 return print(ctx.val, writer, ctx.depth, ctx.pt, null) catch |err| switch (err) {
3738 error.OutOfMemory => @panic("OOM"), // We're not allowed to return this from a format function
3839 error.ComptimeBreak, error.ComptimeReturn, error.AnalysisFail => unreachable,
40 error.Canceled => @panic("TODO"), // pls stop returning this error mlugg
3941 else => |e| return e,
4042 };
4143}