authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-08 21:47:29+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-12 13:55:41+01:00
logdb5d85b8c89b755bd8865def3bd7114d5d9d4867
tree266fb0e6bea6977a76bb67047c69495dabb148ca
parentba53b140288b4518de38a8174ab7ad402607b8d4
signaturelock-open Commit is signed but in an unrecognized format.

compiler: improve progress output

* "Flush" nodes ("LLVM Emit Object", "ELF Flush") appear under "Linking" * "Code Generation" disappears when all analysis and codegen is done * We only show one node under "Semantic Analysis" to accurately convey that analysis isn't happening in parallel, but rather that we're pausing one task to do another

7 files changed, 126 insertions(+), 41 deletions(-)

lib/std/Progress.zig+22
......@@ -234,6 +234,28 @@ pub const Node = struct {
234234 _ = @atomicRmw(u32, &storage.completed_count, .Add, 1, .monotonic);
235235 }
236236
237 /// Thread-safe. Bytes after '0' in `new_name` are ignored.
238 pub fn setName(n: Node, new_name: []const u8) void {
239 const index = n.index.unwrap() orelse return;
240 const storage = storageByIndex(index);
241
242 const name_len = @min(max_name_len, std.mem.indexOfScalar(u8, new_name, 0) orelse new_name.len);
243
244 copyAtomicStore(storage.name[0..name_len], new_name[0..name_len]);
245 if (name_len < storage.name.len)
246 @atomicStore(u8, &storage.name[name_len], 0, .monotonic);
247 }
248
249 /// Gets the name of this `Node`.
250 /// A pointer to this array can later be passed to `setName` to restore the name.
251 pub fn getName(n: Node) [max_name_len]u8 {
252 var dest: [max_name_len]u8 align(@alignOf(usize)) = undefined;
253 if (n.index.unwrap()) |index| {
254 copyAtomicLoad(&dest, &storageByIndex(index).name);
255 }
256 return dest;
257 }
258
237259 /// Thread-safe.
238260 pub fn setCompletedItems(n: Node, completed_items: usize) void {
239261 const index = n.index.unwrap() orelse return;
src/Compilation.zig+36-22
......@@ -255,7 +255,7 @@ test_filters: []const []const u8,
255255test_name_prefix: ?[]const u8,
256256
257257link_task_wait_group: WaitGroup = .{},
258work_queue_progress_node: std.Progress.Node = .none,
258link_prog_node: std.Progress.Node = std.Progress.Node.none,
259259
260260llvm_opt_bisect_limit: c_int,
261261
......@@ -2795,6 +2795,17 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
27952795 }
27962796 }
27972797
2798 // The linker progress node is set up here instead of in `performAllTheWork`, because
2799 // we also want it around during `flush`.
2800 const have_link_node = comp.bin_file != null;
2801 if (have_link_node) {
2802 comp.link_prog_node = main_progress_node.start("Linking", 0);
2803 }
2804 defer if (have_link_node) {
2805 comp.link_prog_node.end();
2806 comp.link_prog_node = .none;
2807 };
2808
27982809 try comp.performAllTheWork(main_progress_node);
27992810
28002811 if (comp.zcu) |zcu| {
......@@ -2843,7 +2854,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
28432854
28442855 switch (comp.cache_use) {
28452856 .none, .incremental => {
2846 try flush(comp, arena, .main, main_progress_node);
2857 try flush(comp, arena, .main);
28472858 },
28482859 .whole => |whole| {
28492860 if (comp.file_system_inputs) |buf| try man.populateFileSystemInputs(buf);
......@@ -2919,7 +2930,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void {
29192930 }
29202931 }
29212932
2922 try flush(comp, arena, .main, main_progress_node);
2933 try flush(comp, arena, .main);
29232934
29242935 // Calling `flush` may have produced errors, in which case the
29252936 // cache manifest must not be written.
......@@ -3009,13 +3020,12 @@ fn flush(
30093020 comp: *Compilation,
30103021 arena: Allocator,
30113022 tid: Zcu.PerThread.Id,
3012 prog_node: std.Progress.Node,
30133023) !void {
30143024 if (comp.zcu) |zcu| {
30153025 if (zcu.llvm_object) |llvm_object| {
30163026 // Emit the ZCU object from LLVM now; it's required to flush the output file.
30173027 // If there's an output file, it wants to decide where the LLVM object goes!
3018 const sub_prog_node = prog_node.start("LLVM Emit Object", 0);
3028 const sub_prog_node = comp.link_prog_node.start("LLVM Emit Object", 0);
30193029 defer sub_prog_node.end();
30203030 try llvm_object.emit(.{
30213031 .pre_ir_path = comp.verbose_llvm_ir,
......@@ -3053,7 +3063,7 @@ fn flush(
30533063 }
30543064 if (comp.bin_file) |lf| {
30553065 // This is needed before reading the error flags.
3056 lf.flush(arena, tid, prog_node) catch |err| switch (err) {
3066 lf.flush(arena, tid, comp.link_prog_node) catch |err| switch (err) {
30573067 error.LinkFailure => {}, // Already reported.
30583068 error.OutOfMemory => return error.OutOfMemory,
30593069 };
......@@ -4172,28 +4182,15 @@ pub fn addWholeFileError(
41724182 }
41734183}
41744184
4175pub fn performAllTheWork(
4185fn performAllTheWork(
41764186 comp: *Compilation,
41774187 main_progress_node: std.Progress.Node,
41784188) JobError!void {
4179 comp.work_queue_progress_node = main_progress_node;
4180 defer comp.work_queue_progress_node = .none;
4181
4189 // Regardless of errors, `comp.zcu` needs to update its generation number.
41824190 defer if (comp.zcu) |zcu| {
4183 zcu.sema_prog_node.end();
4184 zcu.sema_prog_node = .none;
4185 zcu.codegen_prog_node.end();
4186 zcu.codegen_prog_node = .none;
4187
41884191 zcu.generation += 1;
41894192 };
4190 try comp.performAllTheWorkInner(main_progress_node);
4191}
41924193
4193fn performAllTheWorkInner(
4194 comp: *Compilation,
4195 main_progress_node: std.Progress.Node,
4196) JobError!void {
41974194 // Here we queue up all the AstGen tasks first, followed by C object compilation.
41984195 // We wait until the AstGen tasks are all completed before proceeding to the
41994196 // (at least for now) single-threaded main work queue. However, C object compilation
......@@ -4513,8 +4510,24 @@ fn performAllTheWorkInner(
45134510 }
45144511
45154512 zcu.sema_prog_node = main_progress_node.start("Semantic Analysis", 0);
4516 zcu.codegen_prog_node = if (comp.bin_file != null) main_progress_node.start("Code Generation", 0) else .none;
4513 if (comp.bin_file != null) {
4514 zcu.codegen_prog_node = main_progress_node.start("Code Generation", 0);
4515 }
4516 // We increment `pending_codegen_jobs` so that it doesn't reach 0 until after analysis finishes.
4517 // That prevents the "Code Generation" node from constantly disappearing and reappearing when
4518 // we're probably going to analyze more functions at some point.
4519 assert(zcu.pending_codegen_jobs.swap(1, .monotonic) == 0); // don't let this become 0 until analysis finishes
45174520 }
4521 // When analysis ends, delete the progress nodes for "Semantic Analysis" and possibly "Code Generation".
4522 defer if (comp.zcu) |zcu| {
4523 zcu.sema_prog_node.end();
4524 zcu.sema_prog_node = .none;
4525 if (zcu.pending_codegen_jobs.rmw(.Sub, 1, .monotonic) == 1) {
4526 // Decremented to 0, so all done.
4527 zcu.codegen_prog_node.end();
4528 zcu.codegen_prog_node = .none;
4529 }
4530 };
45184531
45194532 if (!comp.separateCodegenThreadOk()) {
45204533 // Waits until all input files have been parsed.
......@@ -4583,6 +4596,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
45834596 .status = .init(.pending),
45844597 .value = undefined,
45854598 };
4599 assert(zcu.pending_codegen_jobs.rmw(.Add, 1, .monotonic) > 0); // the "Code Generation" node hasn't been ended
45864600 if (comp.separateCodegenThreadOk()) {
45874601 // `workerZcuCodegen` takes ownership of `air`.
45884602 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, workerZcuCodegen, .{ comp, func.func, air, shared_mir });
src/Zcu.zig+36-2
......@@ -66,8 +66,18 @@ root_mod: *Package.Module,
6666/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests.
6767main_mod: *Package.Module,
6868std_mod: *Package.Module,
69sema_prog_node: std.Progress.Node = std.Progress.Node.none,
70codegen_prog_node: std.Progress.Node = std.Progress.Node.none,
69sema_prog_node: std.Progress.Node = .none,
70codegen_prog_node: std.Progress.Node = .none,
71/// The number of codegen jobs which are pending or in-progress. Whichever thread drops this value
72/// to 0 is responsible for ending `codegen_prog_node`. While semantic analysis is happening, this
73/// value bottoms out at 1 instead of 0, to ensure that it can only drop to 0 after analysis is
74/// completed (since semantic analysis could trigger more codegen work).
75pending_codegen_jobs: std.atomic.Value(u32) = .init(0),
76
77/// This is the progress node *under* `sema_prog_node` which is currently running.
78/// When we have to pause to analyze something else, we just temporarily rename this node.
79/// Eventually, when we thread semantic analysis, we will want one of these per thread.
80cur_sema_prog_node: std.Progress.Node = .none,
7181
7282/// Used by AstGen worker to load and store ZIR cache.
7383global_zir_cache: Cache.Directory,
......@@ -4753,3 +4763,27 @@ fn explainWhyFileIsInModule(
47534763 import = importer_ref.import;
47544764 }
47554765}
4766
4767const SemaProgNode = struct {
4768 /// `null` means we created the node, so should end it.
4769 old_name: ?[std.Progress.Node.max_name_len]u8,
4770 pub fn end(spn: SemaProgNode, zcu: *Zcu) void {
4771 if (spn.old_name) |old_name| {
4772 zcu.sema_prog_node.completeOne(); // we're just renaming, but it's effectively completion
4773 zcu.cur_sema_prog_node.setName(&old_name);
4774 } else {
4775 zcu.cur_sema_prog_node.end();
4776 zcu.cur_sema_prog_node = .none;
4777 }
4778 }
4779};
4780pub fn startSemaProgNode(zcu: *Zcu, name: []const u8) SemaProgNode {
4781 if (zcu.cur_sema_prog_node.index != .none) {
4782 const old_name = zcu.cur_sema_prog_node.getName();
4783 zcu.cur_sema_prog_node.setName(name);
4784 return .{ .old_name = old_name };
4785 } else {
4786 zcu.cur_sema_prog_node = zcu.sema_prog_node.start(name, 0);
4787 return .{ .old_name = null };
4788 }
4789}
src/Zcu/PerThread.zig+14-8
......@@ -796,8 +796,8 @@ pub fn ensureComptimeUnitUpToDate(pt: Zcu.PerThread, cu_id: InternPool.ComptimeU
796796 info.deps.clearRetainingCapacity();
797797 }
798798
799 const unit_prog_node = zcu.sema_prog_node.start("comptime", 0);
800 defer unit_prog_node.end();
799 const unit_prog_node = zcu.startSemaProgNode("comptime");
800 defer unit_prog_node.end(zcu);
801801
802802 return pt.analyzeComptimeUnit(cu_id) catch |err| switch (err) {
803803 error.AnalysisFail => {
......@@ -976,8 +976,8 @@ pub fn ensureNavValUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zcu
976976 info.deps.clearRetainingCapacity();
977977 }
978978
979 const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0);
980 defer unit_prog_node.end();
979 const unit_prog_node = zcu.startSemaProgNode(nav.fqn.toSlice(ip));
980 defer unit_prog_node.end(zcu);
981981
982982 const invalidate_value: bool, const new_failed: bool = if (pt.analyzeNavVal(nav_id)) |result| res: {
983983 break :res .{
......@@ -1396,8 +1396,8 @@ pub fn ensureNavTypeUpToDate(pt: Zcu.PerThread, nav_id: InternPool.Nav.Index) Zc
13961396 info.deps.clearRetainingCapacity();
13971397 }
13981398
1399 const unit_prog_node = zcu.sema_prog_node.start(nav.fqn.toSlice(ip), 0);
1400 defer unit_prog_node.end();
1399 const unit_prog_node = zcu.startSemaProgNode(nav.fqn.toSlice(ip));
1400 defer unit_prog_node.end(zcu);
14011401
14021402 const invalidate_type: bool, const new_failed: bool = if (pt.analyzeNavType(nav_id)) |result| res: {
14031403 break :res .{
......@@ -1617,8 +1617,8 @@ pub fn ensureFuncBodyUpToDate(pt: Zcu.PerThread, maybe_coerced_func_index: Inter
16171617 info.deps.clearRetainingCapacity();
16181618 }
16191619
1620 const func_prog_node = zcu.sema_prog_node.start(ip.getNav(func.owner_nav).fqn.toSlice(ip), 0);
1621 defer func_prog_node.end();
1620 const func_prog_node = zcu.startSemaProgNode(ip.getNav(func.owner_nav).fqn.toSlice(ip));
1621 defer func_prog_node.end(zcu);
16221622
16231623 const ies_outdated, const new_failed = if (pt.analyzeFuncBody(func_index)) |result|
16241624 .{ prev_failed or result.ies_outdated, false }
......@@ -3360,6 +3360,7 @@ pub fn populateTestFunctions(
33603360 ip.mutateVarInit(test_fns_val.toIntern(), new_init);
33613361 }
33623362 {
3363 assert(zcu.codegen_prog_node.index == .none);
33633364 zcu.codegen_prog_node = main_progress_node.start("Code Generation", 0);
33643365 defer {
33653366 zcu.codegen_prog_node.end();
......@@ -4393,6 +4394,11 @@ pub fn runCodegen(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air, ou
43934394 },
43944395 }
43954396 zcu.comp.link_task_queue.mirReady(zcu.comp, out);
4397 if (zcu.pending_codegen_jobs.rmw(.Sub, 1, .monotonic) == 1) {
4398 // Decremented to 0, so all done.
4399 zcu.codegen_prog_node.end();
4400 zcu.codegen_prog_node = .none;
4401 }
43964402}
43974403fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) error{
43984404 OutOfMemory,
src/link.zig+14-8
......@@ -1074,7 +1074,7 @@ pub const File = struct {
10741074
10751075 /// Called when all linker inputs have been sent via `loadInput`. After
10761076 /// this, `loadInput` will not be called anymore.
1077 pub fn prelink(base: *File, prog_node: std.Progress.Node) FlushError!void {
1077 pub fn prelink(base: *File) FlushError!void {
10781078 assert(!base.post_prelink);
10791079
10801080 // In this case, an object file is created by the LLVM backend, so
......@@ -1085,7 +1085,7 @@ pub const File = struct {
10851085 switch (base.tag) {
10861086 inline .wasm => |tag| {
10871087 dev.check(tag.devFeature());
1088 return @as(*tag.Type(), @fieldParentPtr("base", base)).prelink(prog_node);
1088 return @as(*tag.Type(), @fieldParentPtr("base", base)).prelink(base.comp.link_prog_node);
10891089 },
10901090 else => {},
10911091 }
......@@ -1293,7 +1293,7 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
12931293 const base = comp.bin_file orelse return;
12941294 switch (task) {
12951295 .load_explicitly_provided => {
1296 const prog_node = comp.work_queue_progress_node.start("Parse Linker Inputs", comp.link_inputs.len);
1296 const prog_node = comp.link_prog_node.start("Parse Inputs", comp.link_inputs.len);
12971297 defer prog_node.end();
12981298 for (comp.link_inputs) |input| {
12991299 base.loadInput(input) catch |err| switch (err) {
......@@ -1310,7 +1310,7 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
13101310 }
13111311 },
13121312 .load_host_libc => {
1313 const prog_node = comp.work_queue_progress_node.start("Linker Parse Host libc", 0);
1313 const prog_node = comp.link_prog_node.start("Parse Host libc", 0);
13141314 defer prog_node.end();
13151315
13161316 const target = comp.root_mod.resolved_target.result;
......@@ -1369,7 +1369,7 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
13691369 }
13701370 },
13711371 .load_object => |path| {
1372 const prog_node = comp.work_queue_progress_node.start("Linker Parse Object", 0);
1372 const prog_node = comp.link_prog_node.start("Parse Object", 0);
13731373 defer prog_node.end();
13741374 base.openLoadObject(path) catch |err| switch (err) {
13751375 error.LinkFailure => return, // error reported via diags
......@@ -1377,7 +1377,7 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
13771377 };
13781378 },
13791379 .load_archive => |path| {
1380 const prog_node = comp.work_queue_progress_node.start("Linker Parse Archive", 0);
1380 const prog_node = comp.link_prog_node.start("Parse Archive", 0);
13811381 defer prog_node.end();
13821382 base.openLoadArchive(path, null) catch |err| switch (err) {
13831383 error.LinkFailure => return, // error reported via link_diags
......@@ -1385,7 +1385,7 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
13851385 };
13861386 },
13871387 .load_dso => |path| {
1388 const prog_node = comp.work_queue_progress_node.start("Linker Parse Shared Library", 0);
1388 const prog_node = comp.link_prog_node.start("Parse Shared Library", 0);
13891389 defer prog_node.end();
13901390 base.openLoadDso(path, .{
13911391 .preferred_mode = .dynamic,
......@@ -1396,7 +1396,7 @@ pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
13961396 };
13971397 },
13981398 .load_input => |input| {
1399 const prog_node = comp.work_queue_progress_node.start("Linker Parse Input", 0);
1399 const prog_node = comp.link_prog_node.start("Parse Input", 0);
14001400 defer prog_node.end();
14011401 base.loadInput(input) catch |err| switch (err) {
14021402 error.LinkFailure => return, // error reported via link_diags
......@@ -1418,6 +1418,9 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
14181418 const zcu = comp.zcu.?;
14191419 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
14201420 defer pt.deactivate();
1421 const fqn_slice = zcu.intern_pool.getNav(nav_index).fqn.toSlice(&zcu.intern_pool);
1422 const nav_prog_node = comp.link_prog_node.start(fqn_slice, 0);
1423 defer nav_prog_node.end();
14211424 if (zcu.llvm_object) |llvm_object| {
14221425 llvm_object.updateNav(pt, nav_index) catch |err| switch (err) {
14231426 error.OutOfMemory => diags.setAllocFailure(),
......@@ -1441,6 +1444,9 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
14411444 const nav = zcu.funcInfo(func.func).owner_nav;
14421445 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
14431446 defer pt.deactivate();
1447 const fqn_slice = zcu.intern_pool.getNav(nav).fqn.toSlice(&zcu.intern_pool);
1448 const nav_prog_node = comp.link_prog_node.start(fqn_slice, 0);
1449 defer nav_prog_node.end();
14441450 switch (func.mir.status.load(.monotonic)) {
14451451 .pending => unreachable,
14461452 .ready => {},
src/link/Lld.zig+3
......@@ -267,6 +267,9 @@ pub fn flush(
267267
268268 const comp = lld.base.comp;
269269 const result = if (comp.config.output_mode == .Lib and comp.config.link_mode == .static) r: {
270 if (!@import("build_options").have_llvm or !comp.config.use_lib_llvm) {
271 return lld.base.comp.link_diags.fail("using lld without libllvm not implemented", .{});
272 }
270273 break :r linkAsArchive(lld, arena);
271274 } else switch (lld.ofmt) {
272275 .coff => coffLink(lld, arena),
src/link/Queue.zig+1-1
......@@ -180,7 +180,7 @@ fn flushTaskQueue(tid: usize, q: *Queue, comp: *Compilation) void {
180180 // We've finished the prelink tasks, so run prelink if necessary.
181181 if (comp.bin_file) |lf| {
182182 if (!lf.post_prelink) {
183 if (lf.prelink(comp.work_queue_progress_node)) |_| {
183 if (lf.prelink()) |_| {
184184 lf.post_prelink = true;
185185 } else |err| switch (err) {
186186 error.OutOfMemory => comp.link_diags.setAllocFailure(),