authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-14 02:10:20+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2025-06-14 02:10:20+01:00
log095c956c5c72927f428db370c30add007706b83e
tree6f40f8a11359e64ed4bed85625eaf304a25b06df
parent180e8442af0c29924e021aed327bb4070665af65
parent55b7187429fca74c60be80618fccec7253add6b9
signaturebadge-check Signed by PGP key B5690EEEBB952194

Merge pull request #24171 from mlugg/atomic-order-derp

compiler: fix races in link queue

3 files changed, 30 insertions(+), 34 deletions(-)

src/Zcu/PerThread.zig+18-23
...@@ -4373,34 +4373,29 @@ pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dep...@@ -4373,34 +4373,29 @@ pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dep
4373/// codegen thread, depending on whether the backend supports `Zcu.Feature.separate_thread`.4373/// codegen thread, depending on whether the backend supports `Zcu.Feature.separate_thread`.
4374pub fn runCodegen(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air, out: *@import("../link.zig").ZcuTask.LinkFunc.SharedMir) void {4374pub fn runCodegen(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air, out: *@import("../link.zig").ZcuTask.LinkFunc.SharedMir) void {
4375 const zcu = pt.zcu;4375 const zcu = pt.zcu;
4376 if (runCodegenInner(pt, func_index, air)) |mir| {4376 const success: bool = if (runCodegenInner(pt, func_index, air)) |mir| success: {
4377 out.value = mir;4377 out.value = mir;
4378 out.status.store(.ready, .release);4378 break :success true;
4379 } else |err| switch (err) {4379 } else |err| success: {
4380 error.OutOfMemory => {4380 switch (err) {
4381 zcu.comp.setAllocFailure();4381 error.OutOfMemory => zcu.comp.setAllocFailure(),
4382 out.status.store(.failed, .monotonic);4382 error.CodegenFail => zcu.assertCodegenFailed(zcu.funcInfo(func_index).owner_nav),
4383 },4383 error.NoLinkFile => assert(zcu.comp.bin_file == null),
4384 error.CodegenFail => {4384 error.BackendDoesNotProduceMir => switch (target_util.zigBackend(
4385 zcu.assertCodegenFailed(zcu.funcInfo(func_index).owner_nav);4385 zcu.root_mod.resolved_target.result,
4386 out.status.store(.failed, .monotonic);4386 zcu.comp.config.use_llvm,
4387 },4387 )) {
4388 error.NoLinkFile => {
4389 assert(zcu.comp.bin_file == null);
4390 out.status.store(.failed, .monotonic);
4391 },
4392 error.BackendDoesNotProduceMir => {
4393 const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);
4394 switch (backend) {
4395 else => unreachable, // assertion failure4388 else => unreachable, // assertion failure
4396 .stage2_spirv64,4389 .stage2_spirv64,
4397 .stage2_llvm,4390 .stage2_llvm,
4398 => {},4391 => {},
4399 }4392 },
4400 out.status.store(.failed, .monotonic);4393 }
4401 },4394 break :success false;
4402 }4395 };
4403 zcu.comp.link_task_queue.mirReady(zcu.comp, out);4396 // release `out.value` with this store; synchronizes with acquire loads in `link`
4397 out.status.store(if (success) .ready else .failed, .release);
4398 zcu.comp.link_task_queue.mirReady(zcu.comp, func_index, out);
4404 if (zcu.pending_codegen_jobs.rmw(.Sub, 1, .monotonic) == 1) {4399 if (zcu.pending_codegen_jobs.rmw(.Sub, 1, .monotonic) == 1) {
4405 // Decremented to 0, so all done.4400 // Decremented to 0, so all done.
4406 zcu.codegen_prog_node.end();4401 zcu.codegen_prog_node.end();
src/link.zig+2-2
...@@ -1249,7 +1249,7 @@ pub const ZcuTask = union(enum) {...@@ -1249,7 +1249,7 @@ pub const ZcuTask = union(enum) {
1249 .update_line_number,1249 .update_line_number,
1250 => {},1250 => {},
1251 .link_func => |link_func| {1251 .link_func => |link_func| {
1252 switch (link_func.mir.status.load(.monotonic)) {1252 switch (link_func.mir.status.load(.acquire)) {
1253 .pending => unreachable, // cannot deinit until MIR done1253 .pending => unreachable, // cannot deinit until MIR done
1254 .failed => {}, // MIR not populated so doesn't need freeing1254 .failed => {}, // MIR not populated so doesn't need freeing
1255 .ready => link_func.mir.value.deinit(zcu),1255 .ready => link_func.mir.value.deinit(zcu),
...@@ -1453,7 +1453,7 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {...@@ -1453,7 +1453,7 @@ pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
1453 const fqn_slice = ip.getNav(nav).fqn.toSlice(ip);1453 const fqn_slice = ip.getNav(nav).fqn.toSlice(ip);
1454 const nav_prog_node = comp.link_prog_node.start(fqn_slice, 0);1454 const nav_prog_node = comp.link_prog_node.start(fqn_slice, 0);
1455 defer nav_prog_node.end();1455 defer nav_prog_node.end();
1456 switch (func.mir.status.load(.monotonic)) {1456 switch (func.mir.status.load(.acquire)) {
1457 .pending => unreachable,1457 .pending => unreachable,
1458 .ready => {},1458 .ready => {},
1459 .failed => return,1459 .failed => return,
src/link/Queue.zig+10-9
...@@ -64,7 +64,7 @@ state: union(enum) {...@@ -64,7 +64,7 @@ state: union(enum) {
64 finished,64 finished,
65 /// The link thread is not running or queued, because it is waiting for this MIR to be populated.65 /// The link thread is not running or queued, because it is waiting for this MIR to be populated.
66 /// Once codegen completes, it must call `mirReady` which will restart the link thread.66 /// Once codegen completes, it must call `mirReady` which will restart the link thread.
67 wait_for_mir: *ZcuTask.LinkFunc.SharedMir,67 wait_for_mir: InternPool.Index,
68},68},
6969
70/// In the worst observed case, MIR is around 50 times as large as AIR. More typically, the ratio is70/// In the worst observed case, MIR is around 50 times as large as AIR. More typically, the ratio is
...@@ -113,7 +113,7 @@ pub fn start(q: *Queue, comp: *Compilation) void {...@@ -113,7 +113,7 @@ pub fn start(q: *Queue, comp: *Compilation) void {
113113
114/// Called by codegen workers after they have populated a `ZcuTask.LinkFunc.SharedMir`. If the link114/// Called by codegen workers after they have populated a `ZcuTask.LinkFunc.SharedMir`. If the link
115/// thread was waiting for this MIR, it can resume.115/// thread was waiting for this MIR, it can resume.
116pub fn mirReady(q: *Queue, comp: *Compilation, mir: *ZcuTask.LinkFunc.SharedMir) void {116pub fn mirReady(q: *Queue, comp: *Compilation, func_index: InternPool.Index, mir: *ZcuTask.LinkFunc.SharedMir) void {
117 // We would like to assert that `mir` is not pending, but that would race with a worker thread117 // We would like to assert that `mir` is not pending, but that would race with a worker thread
118 // potentially freeing it.118 // potentially freeing it.
119 {119 {
...@@ -121,12 +121,12 @@ pub fn mirReady(q: *Queue, comp: *Compilation, mir: *ZcuTask.LinkFunc.SharedMir)...@@ -121,12 +121,12 @@ pub fn mirReady(q: *Queue, comp: *Compilation, mir: *ZcuTask.LinkFunc.SharedMir)
121 defer q.mutex.unlock();121 defer q.mutex.unlock();
122 switch (q.state) {122 switch (q.state) {
123 .finished, .running => return,123 .finished, .running => return,
124 .wait_for_mir => |wait_for| if (wait_for != mir) return,124 .wait_for_mir => |wait_for| if (wait_for != func_index) return,
125 }125 }
126 // We were waiting for `mir`, so we will restart the linker thread.126 // We were waiting for `mir`, so we will restart the linker thread.
127 q.state = .running;127 q.state = .running;
128 }128 }
129 assert(mir.status.load(.monotonic) != .pending);129 assert(mir.status.load(.acquire) != .pending);
130 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, flushTaskQueue, .{ q, comp });130 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, flushTaskQueue, .{ q, comp });
131}131}
132132
...@@ -170,8 +170,8 @@ pub fn enqueueZcu(q: *Queue, comp: *Compilation, task: ZcuTask) Allocator.Error!...@@ -170,8 +170,8 @@ pub fn enqueueZcu(q: *Queue, comp: *Compilation, task: ZcuTask) Allocator.Error!
170 .finished => if (q.pending_prelink_tasks != 0) return,170 .finished => if (q.pending_prelink_tasks != 0) return,
171 }171 }
172 // Restart the linker thread, unless it would immediately be blocked172 // Restart the linker thread, unless it would immediately be blocked
173 if (task == .link_func and task.link_func.mir.status.load(.monotonic) == .pending) {173 if (task == .link_func and task.link_func.mir.status.load(.acquire) == .pending) {
174 q.state = .{ .wait_for_mir = task.link_func.mir };174 q.state = .{ .wait_for_mir = task.link_func.func };
175 return;175 return;
176 }176 }
177 q.state = .running;177 q.state = .running;
...@@ -243,12 +243,12 @@ fn flushTaskQueue(tid: usize, q: *Queue, comp: *Compilation) void {...@@ -243,12 +243,12 @@ fn flushTaskQueue(tid: usize, q: *Queue, comp: *Compilation) void {
243 if (task != .link_func) break :pending;243 if (task != .link_func) break :pending;
244 const status_ptr = &task.link_func.mir.status;244 const status_ptr = &task.link_func.mir.status;
245 // First check without the mutex to optimize for the common case where MIR is ready.245 // First check without the mutex to optimize for the common case where MIR is ready.
246 if (status_ptr.load(.monotonic) != .pending) break :pending;246 if (status_ptr.load(.acquire) != .pending) break :pending;
247 q.mutex.lock();247 q.mutex.lock();
248 defer q.mutex.unlock();248 defer q.mutex.unlock();
249 if (status_ptr.load(.monotonic) != .pending) break :pending;249 if (status_ptr.load(.acquire) != .pending) break :pending;
250 // We will stop for now, and get restarted once this MIR is ready.250 // We will stop for now, and get restarted once this MIR is ready.
251 q.state = .{ .wait_for_mir = task.link_func.mir };251 q.state = .{ .wait_for_mir = task.link_func.func };
252 q.flush_safety.unlock();252 q.flush_safety.unlock();
253 return;253 return;
254 }254 }
...@@ -273,6 +273,7 @@ const std = @import("std");...@@ -273,6 +273,7 @@ const std = @import("std");
273const assert = std.debug.assert;273const assert = std.debug.assert;
274const Allocator = std.mem.Allocator;274const Allocator = std.mem.Allocator;
275const Compilation = @import("../Compilation.zig");275const Compilation = @import("../Compilation.zig");
276const InternPool = @import("../InternPool.zig");
276const link = @import("../link.zig");277const link = @import("../link.zig");
277const PrelinkTask = link.PrelinkTask;278const PrelinkTask = link.PrelinkTask;
278const ZcuTask = link.ZcuTask;279const ZcuTask = link.ZcuTask;