authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-05-29 05:38:55+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2025-06-12 13:55:40+01:00
log9eb400ef19391261a3b61129d8665602c89959c5
treefc7046857c3271294a8ebfcd462ece05b0be5f46
parent66d15d9d0974e1b493b717cf02deb435ebd13858
signaturelock-open Commit is signed but in an unrecognized format.

compiler: rework backend pipeline to separate codegen and link

The idea here is that instead of the linker calling into codegen, instead codegen should run before we touch the linker, and after MIR is produced, it is sent to the linker. Aside from simplifying the call graph (by preventing N linkers from each calling into M codegen backends!), this has the huge benefit that it is possible to parallellize codegen separately from linking. The threading model can look like this: * 1 semantic analysis thread, which generates AIR * N codegen threads, which process AIR into MIR * 1 linker thread, which emits MIR to the binary The codegen threads are also responsible for `Air.Legalize` and `Air.Liveness`; it's more efficient to do this work here instead of blocking the main thread for this trivially parallel task. I have repurposed the `Zcu.Feature.separate_thread` backend feature to indicate support for this 1:N:1 threading pattern. This commit makes the C backend support this feature, since it was relatively easy to divorce from `link.C`: it just required eliminating some shared buffers. Other backends don't currently support this feature. In fact, they don't even compile -- the next few commits will fix them back up.

23 files changed, 918 insertions(+), 500 deletions(-)

src/Compilation.zig+140-94
...@@ -43,7 +43,6 @@ const Air = @import("Air.zig");...@@ -43,7 +43,6 @@ const Air = @import("Air.zig");
43const Builtin = @import("Builtin.zig");43const Builtin = @import("Builtin.zig");
44const LlvmObject = @import("codegen/llvm.zig").Object;44const LlvmObject = @import("codegen/llvm.zig").Object;
45const dev = @import("dev.zig");45const dev = @import("dev.zig");
46const ThreadSafeQueue = @import("ThreadSafeQueue.zig").ThreadSafeQueue;
4746
48pub const Config = @import("Compilation/Config.zig");47pub const Config = @import("Compilation/Config.zig");
4948
...@@ -113,17 +112,7 @@ win32_resource_table: if (dev.env.supports(.win32_resource)) std.AutoArrayHashMa...@@ -113,17 +112,7 @@ win32_resource_table: if (dev.env.supports(.win32_resource)) std.AutoArrayHashMa
113} = .{},112} = .{},
114113
115link_diags: link.Diags,114link_diags: link.Diags,
116link_task_queue: ThreadSafeQueue(link.Task) = .empty,115link_task_queue: link.Queue = .empty,
117/// Ensure only 1 simultaneous call to `flushTaskQueue`.
118link_task_queue_safety: std.debug.SafetyLock = .{},
119/// If any tasks are queued up that depend on prelink being finished, they are moved
120/// here until prelink finishes.
121link_task_queue_postponed: std.ArrayListUnmanaged(link.Task) = .empty,
122/// Initialized with how many link input tasks are expected. After this reaches zero
123/// the linker will begin the prelink phase.
124/// Initialized in the Compilation main thread before the pipeline; modified only in
125/// the linker task thread.
126remaining_prelink_tasks: u32,
127116
128/// Set of work that can be represented by only flags to determine whether the117/// Set of work that can be represented by only flags to determine whether the
129/// work is queued or not.118/// work is queued or not.
...@@ -846,15 +835,24 @@ pub const RcIncludes = enum {...@@ -846,15 +835,24 @@ pub const RcIncludes = enum {
846};835};
847836
848const Job = union(enum) {837const Job = union(enum) {
849 /// Corresponds to the task in `link.Task`.838 /// Given the generated AIR for a function, put it onto the code generation queue.
850 /// Only needed for backends that haven't yet been updated to not race against Sema.839 /// This `Job` exists (instead of the `link.ZcuTask` being directly queued) to ensure that
840 /// all types are resolved before the linker task is queued.
841 /// If the backend does not support `Zcu.Feature.separate_thread`, codegen and linking happen immediately.
842 codegen_func: struct {
843 func: InternPool.Index,
844 /// The AIR emitted from analyzing `func`; owned by this `Job` in `gpa`.
845 air: Air,
846 },
847 /// Queue a `link.ZcuTask` to emit this non-function `Nav` into the output binary.
848 /// This `Job` exists (instead of the `link.ZcuTask` being directly queued) to ensure that
849 /// all types are resolved before the linker task is queued.
850 /// If the backend does not support `Zcu.Feature.separate_thread`, the task is run immediately.
851 link_nav: InternPool.Nav.Index,851 link_nav: InternPool.Nav.Index,
852 /// Corresponds to the task in `link.Task`.852 /// Queue a `link.ZcuTask` to emit debug information for this container type.
853 /// TODO: this is currently also responsible for performing codegen.853 /// This `Job` exists (instead of the `link.ZcuTask` being directly queued) to ensure that
854 /// Only needed for backends that haven't yet been updated to not race against Sema.854 /// all types are resolved before the linker task is queued.
855 link_func: link.Task.CodegenFunc,855 /// If the backend does not support `Zcu.Feature.separate_thread`, the task is run immediately.
856 /// Corresponds to the task in `link.Task`.
857 /// Only needed for backends that haven't yet been updated to not race against Sema.
858 link_type: InternPool.Index,856 link_type: InternPool.Index,
859 update_line_number: InternPool.TrackedInst.Index,857 update_line_number: InternPool.TrackedInst.Index,
860 /// The `AnalUnit`, which is *not* a `func`, must be semantically analyzed.858 /// The `AnalUnit`, which is *not* a `func`, must be semantically analyzed.
...@@ -880,13 +878,13 @@ const Job = union(enum) {...@@ -880,13 +878,13 @@ const Job = union(enum) {
880 return switch (tag) {878 return switch (tag) {
881 // Prioritize functions so that codegen can get to work on them on a879 // Prioritize functions so that codegen can get to work on them on a
882 // separate thread, while Sema goes back to its own work.880 // separate thread, while Sema goes back to its own work.
883 .resolve_type_fully, .analyze_func, .link_func => 0,881 .resolve_type_fully, .analyze_func, .codegen_func => 0,
884 else => 1,882 else => 1,
885 };883 };
886 }884 }
887 comptime {885 comptime {
888 // Job dependencies886 // Job dependencies
889 assert(stage(.resolve_type_fully) <= stage(.link_func));887 assert(stage(.resolve_type_fully) <= stage(.codegen_func));
890 }888 }
891};889};
892890
...@@ -2004,7 +2002,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2004,7 +2002,6 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2004 .file_system_inputs = options.file_system_inputs,2002 .file_system_inputs = options.file_system_inputs,
2005 .parent_whole_cache = options.parent_whole_cache,2003 .parent_whole_cache = options.parent_whole_cache,
2006 .link_diags = .init(gpa),2004 .link_diags = .init(gpa),
2007 .remaining_prelink_tasks = 0,
2008 };2005 };
20092006
2010 // Prevent some footguns by making the "any" fields of config reflect2007 // Prevent some footguns by making the "any" fields of config reflect
...@@ -2213,7 +2210,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2213,7 +2210,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2213 };2210 };
2214 comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});2211 comp.c_object_table.putAssumeCapacityNoClobber(c_object, {});
2215 }2212 }
2216 comp.remaining_prelink_tasks += @intCast(comp.c_object_table.count());2213 comp.link_task_queue.pending_prelink_tasks += @intCast(comp.c_object_table.count());
22172214
2218 // Add a `Win32Resource` for each `rc_source_files` and one for `manifest_file`.2215 // Add a `Win32Resource` for each `rc_source_files` and one for `manifest_file`.
2219 const win32_resource_count =2216 const win32_resource_count =
...@@ -2224,7 +2221,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2224,7 +2221,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2224 // Add this after adding logic to updateWin32Resource to pass the2221 // Add this after adding logic to updateWin32Resource to pass the
2225 // result into link.loadInput. loadInput integration is not implemented2222 // result into link.loadInput. loadInput integration is not implemented
2226 // for Windows linking logic yet.2223 // for Windows linking logic yet.
2227 //comp.remaining_prelink_tasks += @intCast(win32_resource_count);2224 //comp.link_task_queue.pending_prelink_tasks += @intCast(win32_resource_count);
2228 for (options.rc_source_files) |rc_source_file| {2225 for (options.rc_source_files) |rc_source_file| {
2229 const win32_resource = try gpa.create(Win32Resource);2226 const win32_resource = try gpa.create(Win32Resource);
2230 errdefer gpa.destroy(win32_resource);2227 errdefer gpa.destroy(win32_resource);
...@@ -2275,78 +2272,76 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2275,78 +2272,76 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2275 const paths = try lci.resolveCrtPaths(arena, basenames, target);2272 const paths = try lci.resolveCrtPaths(arena, basenames, target);
22762273
2277 const fields = @typeInfo(@TypeOf(paths)).@"struct".fields;2274 const fields = @typeInfo(@TypeOf(paths)).@"struct".fields;
2278 try comp.link_task_queue.shared.ensureUnusedCapacity(gpa, fields.len + 1);2275 try comp.link_task_queue.queued_prelink.ensureUnusedCapacity(gpa, fields.len + 1);
2279 inline for (fields) |field| {2276 inline for (fields) |field| {
2280 if (@field(paths, field.name)) |path| {2277 if (@field(paths, field.name)) |path| {
2281 comp.link_task_queue.shared.appendAssumeCapacity(.{ .load_object = path });2278 comp.link_task_queue.queued_prelink.appendAssumeCapacity(.{ .load_object = path });
2282 comp.remaining_prelink_tasks += 1;
2283 }2279 }
2284 }2280 }
2285 // Loads the libraries provided by `target_util.libcFullLinkFlags(target)`.2281 // Loads the libraries provided by `target_util.libcFullLinkFlags(target)`.
2286 comp.link_task_queue.shared.appendAssumeCapacity(.load_host_libc);2282 comp.link_task_queue.queued_prelink.appendAssumeCapacity(.load_host_libc);
2287 comp.remaining_prelink_tasks += 1;
2288 } else if (target.isMuslLibC()) {2283 } else if (target.isMuslLibC()) {
2289 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;2284 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;
22902285
2291 if (musl.needsCrt0(comp.config.output_mode, comp.config.link_mode, comp.config.pie)) |f| {2286 if (musl.needsCrt0(comp.config.output_mode, comp.config.link_mode, comp.config.pie)) |f| {
2292 comp.queued_jobs.musl_crt_file[@intFromEnum(f)] = true;2287 comp.queued_jobs.musl_crt_file[@intFromEnum(f)] = true;
2293 comp.remaining_prelink_tasks += 1;2288 comp.link_task_queue.pending_prelink_tasks += 1;
2294 }2289 }
2295 switch (comp.config.link_mode) {2290 switch (comp.config.link_mode) {
2296 .static => comp.queued_jobs.musl_crt_file[@intFromEnum(musl.CrtFile.libc_a)] = true,2291 .static => comp.queued_jobs.musl_crt_file[@intFromEnum(musl.CrtFile.libc_a)] = true,
2297 .dynamic => comp.queued_jobs.musl_crt_file[@intFromEnum(musl.CrtFile.libc_so)] = true,2292 .dynamic => comp.queued_jobs.musl_crt_file[@intFromEnum(musl.CrtFile.libc_so)] = true,
2298 }2293 }
2299 comp.remaining_prelink_tasks += 1;2294 comp.link_task_queue.pending_prelink_tasks += 1;
2300 } else if (target.isGnuLibC()) {2295 } else if (target.isGnuLibC()) {
2301 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;2296 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;
23022297
2303 if (glibc.needsCrt0(comp.config.output_mode)) |f| {2298 if (glibc.needsCrt0(comp.config.output_mode)) |f| {
2304 comp.queued_jobs.glibc_crt_file[@intFromEnum(f)] = true;2299 comp.queued_jobs.glibc_crt_file[@intFromEnum(f)] = true;
2305 comp.remaining_prelink_tasks += 1;2300 comp.link_task_queue.pending_prelink_tasks += 1;
2306 }2301 }
2307 comp.queued_jobs.glibc_shared_objects = true;2302 comp.queued_jobs.glibc_shared_objects = true;
2308 comp.remaining_prelink_tasks += glibc.sharedObjectsCount(&target);2303 comp.link_task_queue.pending_prelink_tasks += glibc.sharedObjectsCount(&target);
23092304
2310 comp.queued_jobs.glibc_crt_file[@intFromEnum(glibc.CrtFile.libc_nonshared_a)] = true;2305 comp.queued_jobs.glibc_crt_file[@intFromEnum(glibc.CrtFile.libc_nonshared_a)] = true;
2311 comp.remaining_prelink_tasks += 1;2306 comp.link_task_queue.pending_prelink_tasks += 1;
2312 } else if (target.isFreeBSDLibC()) {2307 } else if (target.isFreeBSDLibC()) {
2313 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;2308 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;
23142309
2315 if (freebsd.needsCrt0(comp.config.output_mode)) |f| {2310 if (freebsd.needsCrt0(comp.config.output_mode)) |f| {
2316 comp.queued_jobs.freebsd_crt_file[@intFromEnum(f)] = true;2311 comp.queued_jobs.freebsd_crt_file[@intFromEnum(f)] = true;
2317 comp.remaining_prelink_tasks += 1;2312 comp.link_task_queue.pending_prelink_tasks += 1;
2318 }2313 }
23192314
2320 comp.queued_jobs.freebsd_shared_objects = true;2315 comp.queued_jobs.freebsd_shared_objects = true;
2321 comp.remaining_prelink_tasks += freebsd.sharedObjectsCount();2316 comp.link_task_queue.pending_prelink_tasks += freebsd.sharedObjectsCount();
2322 } else if (target.isNetBSDLibC()) {2317 } else if (target.isNetBSDLibC()) {
2323 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;2318 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;
23242319
2325 if (netbsd.needsCrt0(comp.config.output_mode)) |f| {2320 if (netbsd.needsCrt0(comp.config.output_mode)) |f| {
2326 comp.queued_jobs.netbsd_crt_file[@intFromEnum(f)] = true;2321 comp.queued_jobs.netbsd_crt_file[@intFromEnum(f)] = true;
2327 comp.remaining_prelink_tasks += 1;2322 comp.link_task_queue.pending_prelink_tasks += 1;
2328 }2323 }
23292324
2330 comp.queued_jobs.netbsd_shared_objects = true;2325 comp.queued_jobs.netbsd_shared_objects = true;
2331 comp.remaining_prelink_tasks += netbsd.sharedObjectsCount();2326 comp.link_task_queue.pending_prelink_tasks += netbsd.sharedObjectsCount();
2332 } else if (target.isWasiLibC()) {2327 } else if (target.isWasiLibC()) {
2333 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;2328 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;
23342329
2335 for (comp.wasi_emulated_libs) |crt_file| {2330 for (comp.wasi_emulated_libs) |crt_file| {
2336 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(crt_file)] = true;2331 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(crt_file)] = true;
2337 }2332 }
2338 comp.remaining_prelink_tasks += @intCast(comp.wasi_emulated_libs.len);2333 comp.link_task_queue.pending_prelink_tasks += @intCast(comp.wasi_emulated_libs.len);
23392334
2340 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(wasi_libc.execModelCrtFile(comp.config.wasi_exec_model))] = true;2335 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(wasi_libc.execModelCrtFile(comp.config.wasi_exec_model))] = true;
2341 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(wasi_libc.CrtFile.libc_a)] = true;2336 comp.queued_jobs.wasi_libc_crt_file[@intFromEnum(wasi_libc.CrtFile.libc_a)] = true;
2342 comp.remaining_prelink_tasks += 2;2337 comp.link_task_queue.pending_prelink_tasks += 2;
2343 } else if (target.isMinGW()) {2338 } else if (target.isMinGW()) {
2344 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;2339 if (!std.zig.target.canBuildLibC(target)) return error.LibCUnavailable;
23452340
2346 const main_crt_file: mingw.CrtFile = if (is_dyn_lib) .dllcrt2_o else .crt2_o;2341 const main_crt_file: mingw.CrtFile = if (is_dyn_lib) .dllcrt2_o else .crt2_o;
2347 comp.queued_jobs.mingw_crt_file[@intFromEnum(main_crt_file)] = true;2342 comp.queued_jobs.mingw_crt_file[@intFromEnum(main_crt_file)] = true;
2348 comp.queued_jobs.mingw_crt_file[@intFromEnum(mingw.CrtFile.libmingw32_lib)] = true;2343 comp.queued_jobs.mingw_crt_file[@intFromEnum(mingw.CrtFile.libmingw32_lib)] = true;
2349 comp.remaining_prelink_tasks += 2;2344 comp.link_task_queue.pending_prelink_tasks += 2;
23502345
2351 // When linking mingw-w64 there are some import libs we always need.2346 // When linking mingw-w64 there are some import libs we always need.
2352 try comp.windows_libs.ensureUnusedCapacity(gpa, mingw.always_link_libs.len);2347 try comp.windows_libs.ensureUnusedCapacity(gpa, mingw.always_link_libs.len);
...@@ -2360,7 +2355,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2360,7 +2355,7 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2360 target.isMinGW())2355 target.isMinGW())
2361 {2356 {
2362 comp.queued_jobs.zigc_lib = true;2357 comp.queued_jobs.zigc_lib = true;
2363 comp.remaining_prelink_tasks += 1;2358 comp.link_task_queue.pending_prelink_tasks += 1;
2364 }2359 }
2365 }2360 }
23662361
...@@ -2377,53 +2372,53 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2377,53 +2372,53 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2377 }2372 }
2378 if (comp.wantBuildLibUnwindFromSource()) {2373 if (comp.wantBuildLibUnwindFromSource()) {
2379 comp.queued_jobs.libunwind = true;2374 comp.queued_jobs.libunwind = true;
2380 comp.remaining_prelink_tasks += 1;2375 comp.link_task_queue.pending_prelink_tasks += 1;
2381 }2376 }
2382 if (build_options.have_llvm and is_exe_or_dyn_lib and comp.config.link_libcpp) {2377 if (build_options.have_llvm and is_exe_or_dyn_lib and comp.config.link_libcpp) {
2383 comp.queued_jobs.libcxx = true;2378 comp.queued_jobs.libcxx = true;
2384 comp.queued_jobs.libcxxabi = true;2379 comp.queued_jobs.libcxxabi = true;
2385 comp.remaining_prelink_tasks += 2;2380 comp.link_task_queue.pending_prelink_tasks += 2;
2386 }2381 }
2387 if (build_options.have_llvm and is_exe_or_dyn_lib and comp.config.any_sanitize_thread) {2382 if (build_options.have_llvm and is_exe_or_dyn_lib and comp.config.any_sanitize_thread) {
2388 comp.queued_jobs.libtsan = true;2383 comp.queued_jobs.libtsan = true;
2389 comp.remaining_prelink_tasks += 1;2384 comp.link_task_queue.pending_prelink_tasks += 1;
2390 }2385 }
23912386
2392 if (can_build_compiler_rt) {2387 if (can_build_compiler_rt) {
2393 if (comp.compiler_rt_strat == .lib) {2388 if (comp.compiler_rt_strat == .lib) {
2394 log.debug("queuing a job to build compiler_rt_lib", .{});2389 log.debug("queuing a job to build compiler_rt_lib", .{});
2395 comp.queued_jobs.compiler_rt_lib = true;2390 comp.queued_jobs.compiler_rt_lib = true;
2396 comp.remaining_prelink_tasks += 1;2391 comp.link_task_queue.pending_prelink_tasks += 1;
2397 } else if (comp.compiler_rt_strat == .obj) {2392 } else if (comp.compiler_rt_strat == .obj) {
2398 log.debug("queuing a job to build compiler_rt_obj", .{});2393 log.debug("queuing a job to build compiler_rt_obj", .{});
2399 // In this case we are making a static library, so we ask2394 // In this case we are making a static library, so we ask
2400 // for a compiler-rt object to put in it.2395 // for a compiler-rt object to put in it.
2401 comp.queued_jobs.compiler_rt_obj = true;2396 comp.queued_jobs.compiler_rt_obj = true;
2402 comp.remaining_prelink_tasks += 1;2397 comp.link_task_queue.pending_prelink_tasks += 1;
2403 }2398 }
24042399
2405 if (comp.ubsan_rt_strat == .lib) {2400 if (comp.ubsan_rt_strat == .lib) {
2406 log.debug("queuing a job to build ubsan_rt_lib", .{});2401 log.debug("queuing a job to build ubsan_rt_lib", .{});
2407 comp.queued_jobs.ubsan_rt_lib = true;2402 comp.queued_jobs.ubsan_rt_lib = true;
2408 comp.remaining_prelink_tasks += 1;2403 comp.link_task_queue.pending_prelink_tasks += 1;
2409 } else if (comp.ubsan_rt_strat == .obj) {2404 } else if (comp.ubsan_rt_strat == .obj) {
2410 log.debug("queuing a job to build ubsan_rt_obj", .{});2405 log.debug("queuing a job to build ubsan_rt_obj", .{});
2411 comp.queued_jobs.ubsan_rt_obj = true;2406 comp.queued_jobs.ubsan_rt_obj = true;
2412 comp.remaining_prelink_tasks += 1;2407 comp.link_task_queue.pending_prelink_tasks += 1;
2413 }2408 }
24142409
2415 if (is_exe_or_dyn_lib and comp.config.any_fuzz) {2410 if (is_exe_or_dyn_lib and comp.config.any_fuzz) {
2416 log.debug("queuing a job to build libfuzzer", .{});2411 log.debug("queuing a job to build libfuzzer", .{});
2417 comp.queued_jobs.fuzzer_lib = true;2412 comp.queued_jobs.fuzzer_lib = true;
2418 comp.remaining_prelink_tasks += 1;2413 comp.link_task_queue.pending_prelink_tasks += 1;
2419 }2414 }
2420 }2415 }
2421 }2416 }
24222417
2423 try comp.link_task_queue.shared.append(gpa, .load_explicitly_provided);2418 try comp.link_task_queue.queued_prelink.append(gpa, .load_explicitly_provided);
2424 comp.remaining_prelink_tasks += 1;
2425 }2419 }
2426 log.debug("total prelink tasks: {d}", .{comp.remaining_prelink_tasks});2420 log.debug("queued prelink tasks: {d}", .{comp.link_task_queue.queued_prelink.items.len});
2421 log.debug("pending prelink tasks: {d}", .{comp.link_task_queue.pending_prelink_tasks});
24272422
2428 return comp;2423 return comp;
2429}2424}
...@@ -2431,6 +2426,10 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil...@@ -2431,6 +2426,10 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil
2431pub fn destroy(comp: *Compilation) void {2426pub fn destroy(comp: *Compilation) void {
2432 const gpa = comp.gpa;2427 const gpa = comp.gpa;
24332428
2429 // This needs to be destroyed first, because it might contain MIR which we only know
2430 // how to interpret (which kind of MIR it is) from `comp.bin_file`.
2431 comp.link_task_queue.deinit(comp);
2432
2434 if (comp.bin_file) |lf| lf.destroy();2433 if (comp.bin_file) |lf| lf.destroy();
2435 if (comp.zcu) |zcu| zcu.deinit();2434 if (comp.zcu) |zcu| zcu.deinit();
2436 comp.cache_use.deinit();2435 comp.cache_use.deinit();
...@@ -2512,8 +2511,6 @@ pub fn destroy(comp: *Compilation) void {...@@ -2512,8 +2511,6 @@ pub fn destroy(comp: *Compilation) void {
2512 comp.failed_win32_resources.deinit(gpa);2511 comp.failed_win32_resources.deinit(gpa);
25132512
2514 comp.link_diags.deinit();2513 comp.link_diags.deinit();
2515 comp.link_task_queue.deinit(gpa);
2516 comp.link_task_queue_postponed.deinit(gpa);
25172514
2518 comp.clearMiscFailures();2515 comp.clearMiscFailures();
25192516
...@@ -4180,9 +4177,7 @@ fn performAllTheWorkInner(...@@ -4180,9 +4177,7 @@ fn performAllTheWorkInner(
4180 comp.link_task_wait_group.reset();4177 comp.link_task_wait_group.reset();
4181 defer comp.link_task_wait_group.wait();4178 defer comp.link_task_wait_group.wait();
41824179
4183 if (comp.link_task_queue.start()) {4180 comp.link_task_queue.start(comp);
4184 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, link.flushTaskQueue, .{comp});
4185 }
41864181
4187 if (comp.docs_emit != null) {4182 if (comp.docs_emit != null) {
4188 dev.check(.docs_emit);4183 dev.check(.docs_emit);
...@@ -4498,7 +4493,7 @@ fn performAllTheWorkInner(...@@ -4498,7 +4493,7 @@ fn performAllTheWorkInner(
4498 comp.link_task_wait_group.wait();4493 comp.link_task_wait_group.wait();
4499 comp.link_task_wait_group.reset();4494 comp.link_task_wait_group.reset();
4500 std.log.scoped(.link).debug("finished waiting for link_task_wait_group", .{});4495 std.log.scoped(.link).debug("finished waiting for link_task_wait_group", .{});
4501 if (comp.remaining_prelink_tasks > 0) {4496 if (comp.link_task_queue.pending_prelink_tasks > 0) {
4502 // Indicates an error occurred preventing prelink phase from completing.4497 // Indicates an error occurred preventing prelink phase from completing.
4503 return;4498 return;
4504 }4499 }
...@@ -4543,6 +4538,45 @@ pub fn queueJobs(comp: *Compilation, jobs: []const Job) !void {...@@ -4543,6 +4538,45 @@ pub fn queueJobs(comp: *Compilation, jobs: []const Job) !void {
45434538
4544fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {4539fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
4545 switch (job) {4540 switch (job) {
4541 .codegen_func => |func| {
4542 const zcu = comp.zcu.?;
4543 const gpa = zcu.gpa;
4544 var air = func.air;
4545 errdefer air.deinit(gpa);
4546 if (!air.typesFullyResolved(zcu)) {
4547 // Type resolution failed in a way which affects this function. This is a transitive
4548 // failure, but it doesn't need recording, because this function semantically depends
4549 // on the failed type, so when it is changed the function is updated.
4550 air.deinit(gpa);
4551 return;
4552 }
4553 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
4554 defer pt.deactivate();
4555 const shared_mir = try gpa.create(link.ZcuTask.LinkFunc.SharedMir);
4556 shared_mir.* = .{
4557 .status = .init(.pending),
4558 .value = undefined,
4559 };
4560 if (comp.separateCodegenThreadOk()) {
4561 // `workerZcuCodegen` takes ownership of `air`.
4562 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, workerZcuCodegen, .{ comp, func.func, air, shared_mir });
4563 comp.dispatchZcuLinkTask(tid, .{ .link_func = .{
4564 .func = func.func,
4565 .mir = shared_mir,
4566 .air = undefined,
4567 } });
4568 } else {
4569 const emit_needs_air = !zcu.backendSupportsFeature(.separate_thread);
4570 pt.runCodegen(func.func, &air, shared_mir);
4571 assert(shared_mir.status.load(.monotonic) != .pending);
4572 comp.dispatchZcuLinkTask(tid, .{ .link_func = .{
4573 .func = func.func,
4574 .mir = shared_mir,
4575 .air = if (emit_needs_air) &air else undefined,
4576 } });
4577 air.deinit(gpa);
4578 }
4579 },
4546 .link_nav => |nav_index| {4580 .link_nav => |nav_index| {
4547 const zcu = comp.zcu.?;4581 const zcu = comp.zcu.?;
4548 const nav = zcu.intern_pool.getNav(nav_index);4582 const nav = zcu.intern_pool.getNav(nav_index);
...@@ -4559,17 +4593,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {...@@ -4559,17 +4593,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
4559 // on the failed type, so when it is changed the `Nav` will be updated.4593 // on the failed type, so when it is changed the `Nav` will be updated.
4560 return;4594 return;
4561 }4595 }
4562 comp.dispatchLinkTask(tid, .{ .link_nav = nav_index });4596 comp.dispatchZcuLinkTask(tid, .{ .link_nav = nav_index });
4563 },
4564 .link_func => |func| {
4565 const zcu = comp.zcu.?;
4566 if (!func.air.typesFullyResolved(zcu)) {
4567 // Type resolution failed in a way which affects this function. This is a transitive
4568 // failure, but it doesn't need recording, because this function semantically depends
4569 // on the failed type, so when it is changed the function is updated.
4570 return;
4571 }
4572 comp.dispatchLinkTask(tid, .{ .link_func = func });
4573 },4597 },
4574 .link_type => |ty| {4598 .link_type => |ty| {
4575 const zcu = comp.zcu.?;4599 const zcu = comp.zcu.?;
...@@ -4580,10 +4604,10 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {...@@ -4580,10 +4604,10 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
4580 // on the failed type, so when that is changed, this type will be updated.4604 // on the failed type, so when that is changed, this type will be updated.
4581 return;4605 return;
4582 }4606 }
4583 comp.dispatchLinkTask(tid, .{ .link_type = ty });4607 comp.dispatchZcuLinkTask(tid, .{ .link_type = ty });
4584 },4608 },
4585 .update_line_number => |ti| {4609 .update_line_number => |ti| {
4586 comp.dispatchLinkTask(tid, .{ .update_line_number = ti });4610 comp.dispatchZcuLinkTask(tid, .{ .update_line_number = ti });
4587 },4611 },
4588 .analyze_func => |func| {4612 .analyze_func => |func| {
4589 const named_frame = tracy.namedFrame("analyze_func");4613 const named_frame = tracy.namedFrame("analyze_func");
...@@ -4675,18 +4699,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {...@@ -4675,18 +4699,7 @@ fn processOneJob(tid: usize, comp: *Compilation, job: Job) JobError!void {
4675 }4699 }
4676}4700}
46774701
4678/// The reason for the double-queue here is that the first queue ensures any4702pub fn separateCodegenThreadOk(comp: *const Compilation) bool {
4679/// resolve_type_fully tasks are complete before this dispatch function is called.
4680fn dispatchLinkTask(comp: *Compilation, tid: usize, link_task: link.Task) void {
4681 if (comp.separateCodegenThreadOk()) {
4682 comp.queueLinkTasks(&.{link_task});
4683 } else {
4684 assert(comp.remaining_prelink_tasks == 0);
4685 link.doTask(comp, tid, link_task);
4686 }
4687}
4688
4689fn separateCodegenThreadOk(comp: *const Compilation) bool {
4690 if (InternPool.single_threaded) return false;4703 if (InternPool.single_threaded) return false;
4691 const zcu = comp.zcu orelse return true;4704 const zcu = comp.zcu orelse return true;
4692 return zcu.backendSupportsFeature(.separate_thread);4705 return zcu.backendSupportsFeature(.separate_thread);
...@@ -5273,6 +5286,21 @@ pub const RtOptions = struct {...@@ -5273,6 +5286,21 @@ pub const RtOptions = struct {
5273 allow_lto: bool = true,5286 allow_lto: bool = true,
5274};5287};
52755288
5289fn workerZcuCodegen(
5290 tid: usize,
5291 comp: *Compilation,
5292 func_index: InternPool.Index,
5293 orig_air: Air,
5294 out: *link.ZcuTask.LinkFunc.SharedMir,
5295) void {
5296 var air = orig_air;
5297 // We own `air` now, so we are responsbile for freeing it.
5298 defer air.deinit(comp.gpa);
5299 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
5300 defer pt.deactivate();
5301 pt.runCodegen(func_index, &air, out);
5302}
5303
5276fn buildRt(5304fn buildRt(
5277 comp: *Compilation,5305 comp: *Compilation,
5278 root_source_name: []const u8,5306 root_source_name: []const u8,
...@@ -5804,7 +5832,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr...@@ -5804,7 +5832,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr
5804 },5832 },
5805 };5833 };
58065834
5807 comp.queueLinkTasks(&.{.{ .load_object = c_object.status.success.object_path }});5835 comp.queuePrelinkTasks(&.{.{ .load_object = c_object.status.success.object_path }});
5808}5836}
58095837
5810fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32_resource_prog_node: std.Progress.Node) !void {5838fn updateWin32Resource(comp: *Compilation, win32_resource: *Win32Resource, win32_resource_prog_node: std.Progress.Node) !void {
...@@ -7237,7 +7265,7 @@ fn buildOutputFromZig(...@@ -7237,7 +7265,7 @@ fn buildOutputFromZig(
7237 assert(out.* == null);7265 assert(out.* == null);
7238 out.* = crt_file;7266 out.* = crt_file;
72397267
7240 comp.queueLinkTaskMode(crt_file.full_object_path, &config);7268 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
7241}7269}
72427270
7243pub const CrtFileOptions = struct {7271pub const CrtFileOptions = struct {
...@@ -7361,7 +7389,7 @@ pub fn build_crt_file(...@@ -7361,7 +7389,7 @@ pub fn build_crt_file(
7361 try comp.updateSubCompilation(sub_compilation, misc_task_tag, prog_node);7389 try comp.updateSubCompilation(sub_compilation, misc_task_tag, prog_node);
73627390
7363 const crt_file = try sub_compilation.toCrtFile();7391 const crt_file = try sub_compilation.toCrtFile();
7364 comp.queueLinkTaskMode(crt_file.full_object_path, &config);7392 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
73657393
7366 {7394 {
7367 comp.mutex.lock();7395 comp.mutex.lock();
...@@ -7371,8 +7399,8 @@ pub fn build_crt_file(...@@ -7371,8 +7399,8 @@ pub fn build_crt_file(
7371 }7399 }
7372}7400}
73737401
7374pub fn queueLinkTaskMode(comp: *Compilation, path: Cache.Path, config: *const Compilation.Config) void {7402pub fn queuePrelinkTaskMode(comp: *Compilation, path: Cache.Path, config: *const Compilation.Config) void {
7375 comp.queueLinkTasks(switch (config.output_mode) {7403 comp.queuePrelinkTasks(switch (config.output_mode) {
7376 .Exe => unreachable,7404 .Exe => unreachable,
7377 .Obj => &.{.{ .load_object = path }},7405 .Obj => &.{.{ .load_object = path }},
7378 .Lib => &.{switch (config.link_mode) {7406 .Lib => &.{switch (config.link_mode) {
...@@ -7384,12 +7412,30 @@ pub fn queueLinkTaskMode(comp: *Compilation, path: Cache.Path, config: *const Co...@@ -7384,12 +7412,30 @@ pub fn queueLinkTaskMode(comp: *Compilation, path: Cache.Path, config: *const Co
73847412
7385/// Only valid to call during `update`. Automatically handles queuing up a7413/// Only valid to call during `update`. Automatically handles queuing up a
7386/// linker worker task if there is not already one.7414/// linker worker task if there is not already one.
7387pub fn queueLinkTasks(comp: *Compilation, tasks: []const link.Task) void {7415pub fn queuePrelinkTasks(comp: *Compilation, tasks: []const link.PrelinkTask) void {
7388 if (comp.link_task_queue.enqueue(comp.gpa, tasks) catch |err| switch (err) {7416 comp.link_task_queue.enqueuePrelink(comp, tasks) catch |err| switch (err) {
7389 error.OutOfMemory => return comp.setAllocFailure(),7417 error.OutOfMemory => return comp.setAllocFailure(),
7390 }) {7418 };
7391 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, link.flushTaskQueue, .{comp});7419}
7420
7421/// The reason for the double-queue here is that the first queue ensures any
7422/// resolve_type_fully tasks are complete before this dispatch function is called.
7423fn dispatchZcuLinkTask(comp: *Compilation, tid: usize, task: link.ZcuTask) void {
7424 if (!comp.separateCodegenThreadOk()) {
7425 assert(tid == 0);
7426 if (task == .link_func) {
7427 assert(task.link_func.mir.status.load(.monotonic) != .pending);
7428 }
7429 link.doZcuTask(comp, tid, task);
7430 task.deinit(comp.zcu.?);
7431 return;
7392 }7432 }
7433 comp.link_task_queue.enqueueZcu(comp, task) catch |err| switch (err) {
7434 error.OutOfMemory => {
7435 task.deinit(comp.zcu.?);
7436 comp.setAllocFailure();
7437 },
7438 };
7393}7439}
73947440
7395pub fn toCrtFile(comp: *Compilation) Allocator.Error!CrtFile {7441pub fn toCrtFile(comp: *Compilation) Allocator.Error!CrtFile {
src/ThreadSafeQueue.zig deleted-72
...@@ -1,72 +0,0 @@
1const std = @import("std");
2const assert = std.debug.assert;
3const Allocator = std.mem.Allocator;
4
5pub fn ThreadSafeQueue(comptime T: type) type {
6 return struct {
7 worker_owned: std.ArrayListUnmanaged(T),
8 /// Protected by `mutex`.
9 shared: std.ArrayListUnmanaged(T),
10 mutex: std.Thread.Mutex,
11 state: State,
12
13 const Self = @This();
14
15 pub const State = enum { wait, run };
16
17 pub const empty: Self = .{
18 .worker_owned = .empty,
19 .shared = .empty,
20 .mutex = .{},
21 .state = .wait,
22 };
23
24 pub fn deinit(self: *Self, gpa: Allocator) void {
25 self.worker_owned.deinit(gpa);
26 self.shared.deinit(gpa);
27 self.* = undefined;
28 }
29
30 /// Must be called from the worker thread.
31 pub fn check(self: *Self) ?[]T {
32 assert(self.worker_owned.items.len == 0);
33 {
34 self.mutex.lock();
35 defer self.mutex.unlock();
36 assert(self.state == .run);
37 if (self.shared.items.len == 0) {
38 self.state = .wait;
39 return null;
40 }
41 std.mem.swap(std.ArrayListUnmanaged(T), &self.worker_owned, &self.shared);
42 }
43 const result = self.worker_owned.items;
44 self.worker_owned.clearRetainingCapacity();
45 return result;
46 }
47
48 /// Adds items to the queue, returning true if and only if the worker
49 /// thread is waiting. Thread-safe.
50 /// Not safe to call from the worker thread.
51 pub fn enqueue(self: *Self, gpa: Allocator, items: []const T) error{OutOfMemory}!bool {
52 self.mutex.lock();
53 defer self.mutex.unlock();
54 try self.shared.appendSlice(gpa, items);
55 return switch (self.state) {
56 .run => false,
57 .wait => {
58 self.state = .run;
59 return true;
60 },
61 };
62 }
63
64 /// Safe only to call exactly once when initially starting the worker.
65 pub fn start(self: *Self) bool {
66 assert(self.state == .wait);
67 if (self.shared.items.len == 0) return false;
68 self.state = .run;
69 return true;
70 }
71 };
72}
src/Zcu.zig+44-6
...@@ -171,6 +171,8 @@ transitive_failed_analysis: std.AutoArrayHashMapUnmanaged(AnalUnit, void) = .emp...@@ -171,6 +171,8 @@ transitive_failed_analysis: std.AutoArrayHashMapUnmanaged(AnalUnit, void) = .emp
171/// This `Nav` succeeded analysis, but failed codegen.171/// This `Nav` succeeded analysis, but failed codegen.
172/// This may be a simple "value" `Nav`, or it may be a function.172/// This may be a simple "value" `Nav`, or it may be a function.
173/// The ErrorMsg memory is owned by the `AnalUnit`, using Module's general purpose allocator.173/// The ErrorMsg memory is owned by the `AnalUnit`, using Module's general purpose allocator.
174/// While multiple threads are active (most of the time!), this is guarded by `zcu.comp.mutex`, as
175/// codegen and linking run on a separate thread.
174failed_codegen: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, *ErrorMsg) = .empty,176failed_codegen: std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, *ErrorMsg) = .empty,
175failed_types: std.AutoArrayHashMapUnmanaged(InternPool.Index, *ErrorMsg) = .empty,177failed_types: std.AutoArrayHashMapUnmanaged(InternPool.Index, *ErrorMsg) = .empty,
176/// Keep track of `@compileLog`s per `AnalUnit`.178/// Keep track of `@compileLog`s per `AnalUnit`.
...@@ -3817,7 +3819,36 @@ pub const Feature = enum {...@@ -3817,7 +3819,36 @@ pub const Feature = enum {
3817 is_named_enum_value,3819 is_named_enum_value,
3818 error_set_has_value,3820 error_set_has_value,
3819 field_reordering,3821 field_reordering,
3820 /// If the backend supports running from another thread.3822 /// In theory, backends are supposed to work like this:
3823 ///
3824 /// * The AIR emitted by `Sema` is converted into MIR by `codegen.generateFunction`. This pass
3825 /// is "pure", in that it does not depend on or modify any external mutable state.
3826 ///
3827 /// * That MIR is sent to the linker, which calls `codegen.emitFunction` to convert the MIR to
3828 /// finalized machine code. This process is permitted to query and modify linker state.
3829 ///
3830 /// * The linker stores the resulting machine code in the binary as needed.
3831 ///
3832 /// The first stage described above can run in parallel to the rest of the compiler, and even to
3833 /// other code generation work; we can run as many codegen threads as we want in parallel because
3834 /// of the fact that this pass is pure. Emit and link must be single-threaded, but are generally
3835 /// very fast, so that isn't a problem.
3836 ///
3837 /// Unfortunately, some code generation implementations currently query and/or mutate linker state
3838 /// or even (in the case of the LLVM backend) semantic analysis state. Such backends cannot be run
3839 /// in parallel with each other, with linking, or (potentially) with semantic analysis.
3840 ///
3841 /// Additionally, some backends continue to need the AIR in the "emit" stage, despite this pass
3842 /// operating on MIR. This complicates memory management under the threading model above.
3843 ///
3844 /// These are both **bugs** in backend implementations, left over from legacy code. However, they
3845 /// are difficult to fix. So, this `Feature` currently guards correct threading of code generation:
3846 ///
3847 /// * With this feature enabled, the backend is threaded as described above. The "emit" stage does
3848 /// not have access to AIR (it will be `undefined`; see `codegen.emitFunction`).
3849 ///
3850 /// * With this feature disabled, semantic analysis, code generation, and linking all occur on the
3851 /// same thread, and the "emit" stage has access to AIR.
3821 separate_thread,3852 separate_thread,
3822};3853};
38233854
...@@ -4566,22 +4597,29 @@ pub fn codegenFail(...@@ -4566,22 +4597,29 @@ pub fn codegenFail(
4566 comptime format: []const u8,4597 comptime format: []const u8,
4567 args: anytype,4598 args: anytype,
4568) CodegenFailError {4599) CodegenFailError {
4569 const gpa = zcu.gpa;4600 const msg = try Zcu.ErrorMsg.create(zcu.gpa, zcu.navSrcLoc(nav_index), format, args);
4570 try zcu.failed_codegen.ensureUnusedCapacity(gpa, 1);4601 return zcu.codegenFailMsg(nav_index, msg);
4571 const msg = try Zcu.ErrorMsg.create(gpa, zcu.navSrcLoc(nav_index), format, args);
4572 zcu.failed_codegen.putAssumeCapacityNoClobber(nav_index, msg);
4573 return error.CodegenFail;
4574}4602}
45754603
4604/// Takes ownership of `msg`, even on OOM.
4576pub fn codegenFailMsg(zcu: *Zcu, nav_index: InternPool.Nav.Index, msg: *ErrorMsg) CodegenFailError {4605pub fn codegenFailMsg(zcu: *Zcu, nav_index: InternPool.Nav.Index, msg: *ErrorMsg) CodegenFailError {
4577 const gpa = zcu.gpa;4606 const gpa = zcu.gpa;
4578 {4607 {
4608 zcu.comp.mutex.lock();
4609 defer zcu.comp.mutex.unlock();
4579 errdefer msg.deinit(gpa);4610 errdefer msg.deinit(gpa);
4580 try zcu.failed_codegen.putNoClobber(gpa, nav_index, msg);4611 try zcu.failed_codegen.putNoClobber(gpa, nav_index, msg);
4581 }4612 }
4582 return error.CodegenFail;4613 return error.CodegenFail;
4583}4614}
45844615
4616/// Asserts that `zcu.failed_codegen` contains the key `nav`, with the necessary lock held.
4617pub fn assertCodegenFailed(zcu: *Zcu, nav: InternPool.Nav.Index) void {
4618 zcu.comp.mutex.lock();
4619 defer zcu.comp.mutex.unlock();
4620 assert(zcu.failed_codegen.contains(nav));
4621}
4622
4585pub fn codegenFailType(4623pub fn codegenFailType(
4586 zcu: *Zcu,4624 zcu: *Zcu,
4587 ty_index: InternPool.Index,4625 ty_index: InternPool.Index,
src/Zcu/PerThread.zig+87-75
...@@ -27,6 +27,7 @@ const Type = @import("../Type.zig");...@@ -27,6 +27,7 @@ const Type = @import("../Type.zig");
27const Value = @import("../Value.zig");27const Value = @import("../Value.zig");
28const Zcu = @import("../Zcu.zig");28const Zcu = @import("../Zcu.zig");
29const Compilation = @import("../Compilation.zig");29const Compilation = @import("../Compilation.zig");
30const codegen = @import("../codegen.zig");
30const Zir = std.zig.Zir;31const Zir = std.zig.Zir;
31const Zoir = std.zig.Zoir;32const Zoir = std.zig.Zoir;
32const ZonGen = std.zig.ZonGen;33const ZonGen = std.zig.ZonGen;
...@@ -1716,7 +1717,7 @@ fn analyzeFuncBody(...@@ -1716,7 +1717,7 @@ fn analyzeFuncBody(
1716 }1717 }
17171718
1718 // This job depends on any resolve_type_fully jobs queued up before it.1719 // This job depends on any resolve_type_fully jobs queued up before it.
1719 try comp.queueJob(.{ .link_func = .{1720 try comp.queueJob(.{ .codegen_func = .{
1720 .func = func_index,1721 .func = func_index,
1721 .air = air,1722 .air = air,
1722 } });1723 } });
...@@ -1724,79 +1725,6 @@ fn analyzeFuncBody(...@@ -1724,79 +1725,6 @@ fn analyzeFuncBody(
1724 return .{ .ies_outdated = ies_outdated };1725 return .{ .ies_outdated = ies_outdated };
1725}1726}
17261727
1727/// Takes ownership of `air`, even on error.
1728/// If any types referenced by `air` are unresolved, marks the codegen as failed.
1729pub fn linkerUpdateFunc(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) Allocator.Error!void {
1730 const zcu = pt.zcu;
1731 const gpa = zcu.gpa;
1732 const ip = &zcu.intern_pool;
1733 const comp = zcu.comp;
1734
1735 const func = zcu.funcInfo(func_index);
1736 const nav_index = func.owner_nav;
1737 const nav = ip.getNav(nav_index);
1738
1739 const codegen_prog_node = zcu.codegen_prog_node.start(nav.fqn.toSlice(ip), 0);
1740 defer codegen_prog_node.end();
1741
1742 legalize: {
1743 try air.legalize(pt, @import("../codegen.zig").legalizeFeatures(pt, nav_index) orelse break :legalize);
1744 }
1745
1746 var liveness = try Air.Liveness.analyze(zcu, air.*, ip);
1747 defer liveness.deinit(gpa);
1748
1749 if (build_options.enable_debug_extensions and comp.verbose_air) {
1750 std.debug.print("# Begin Function AIR: {}:\n", .{nav.fqn.fmt(ip)});
1751 air.dump(pt, liveness);
1752 std.debug.print("# End Function AIR: {}\n\n", .{nav.fqn.fmt(ip)});
1753 }
1754
1755 if (std.debug.runtime_safety) {
1756 var verify: Air.Liveness.Verify = .{
1757 .gpa = gpa,
1758 .zcu = zcu,
1759 .air = air.*,
1760 .liveness = liveness,
1761 .intern_pool = ip,
1762 };
1763 defer verify.deinit();
1764
1765 verify.verify() catch |err| switch (err) {
1766 error.OutOfMemory => return error.OutOfMemory,
1767 else => {
1768 try zcu.failed_codegen.putNoClobber(gpa, nav_index, try Zcu.ErrorMsg.create(
1769 gpa,
1770 zcu.navSrcLoc(nav_index),
1771 "invalid liveness: {s}",
1772 .{@errorName(err)},
1773 ));
1774 return;
1775 },
1776 };
1777 }
1778
1779 if (zcu.llvm_object) |llvm_object| {
1780 llvm_object.updateFunc(pt, func_index, air.*, liveness) catch |err| switch (err) {
1781 error.OutOfMemory => return error.OutOfMemory,
1782 };
1783 } else if (comp.bin_file) |lf| {
1784 lf.updateFunc(pt, func_index, air, liveness) catch |err| switch (err) {
1785 error.OutOfMemory => return error.OutOfMemory,
1786 error.CodegenFail => assert(zcu.failed_codegen.contains(nav_index)),
1787 error.Overflow, error.RelocationNotByteAligned => {
1788 try zcu.failed_codegen.putNoClobber(gpa, nav_index, try Zcu.ErrorMsg.create(
1789 gpa,
1790 zcu.navSrcLoc(nav_index),
1791 "unable to codegen: {s}",
1792 .{@errorName(err)},
1793 ));
1794 // Not a retryable failure.
1795 },
1796 };
1797 }
1798}
1799
1800pub fn semaMod(pt: Zcu.PerThread, mod: *Module) !void {1728pub fn semaMod(pt: Zcu.PerThread, mod: *Module) !void {
1801 dev.check(.sema);1729 dev.check(.sema);
1802 const file_index = pt.zcu.module_roots.get(mod).?.unwrap().?;1730 const file_index = pt.zcu.module_roots.get(mod).?.unwrap().?;
...@@ -3449,7 +3377,7 @@ pub fn populateTestFunctions(...@@ -3449,7 +3377,7 @@ pub fn populateTestFunctions(
3449 }3377 }
34503378
3451 // The linker thread is not running, so we actually need to dispatch this task directly.3379 // The linker thread is not running, so we actually need to dispatch this task directly.
3452 @import("../link.zig").doTask(zcu.comp, @intFromEnum(pt.tid), .{ .link_nav = nav_index });3380 @import("../link.zig").doZcuTask(zcu.comp, @intFromEnum(pt.tid), .{ .link_nav = nav_index });
3453 }3381 }
3454}3382}
34553383
...@@ -4442,3 +4370,87 @@ pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dep...@@ -4442,3 +4370,87 @@ pub fn addDependency(pt: Zcu.PerThread, unit: AnalUnit, dependee: InternPool.Dep
4442 try info.deps.append(gpa, dependee);4370 try info.deps.append(gpa, dependee);
4443 }4371 }
4444}4372}
4373
4374/// Performs code generation, which comes after `Sema` but before `link` in the pipeline.
4375/// This part of the pipeline is self-contained/"pure", so can be run in parallel with most
4376/// other code. This function is currently run either on the main thread, or on a separate
4377/// codegen thread, depending on whether the backend supports `Zcu.Feature.separate_thread`.
4378pub fn runCodegen(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air, out: *@import("../link.zig").ZcuTask.LinkFunc.SharedMir) void {
4379 if (runCodegenInner(pt, func_index, air)) |mir| {
4380 out.value = mir;
4381 out.status.store(.ready, .release);
4382 } else |err| switch (err) {
4383 error.OutOfMemory => {
4384 pt.zcu.comp.setAllocFailure();
4385 out.status.store(.failed, .monotonic);
4386 },
4387 error.CodegenFail => {
4388 pt.zcu.assertCodegenFailed(pt.zcu.funcInfo(func_index).owner_nav);
4389 out.status.store(.failed, .monotonic);
4390 },
4391 error.NoLinkFile => {
4392 assert(pt.zcu.comp.bin_file == null);
4393 out.status.store(.failed, .monotonic);
4394 },
4395 }
4396 pt.zcu.comp.link_task_queue.mirReady(pt.zcu.comp, out);
4397}
4398fn runCodegenInner(pt: Zcu.PerThread, func_index: InternPool.Index, air: *Air) error{ OutOfMemory, CodegenFail, NoLinkFile }!codegen.AnyMir {
4399 const zcu = pt.zcu;
4400 const gpa = zcu.gpa;
4401 const ip = &zcu.intern_pool;
4402 const comp = zcu.comp;
4403
4404 const nav = zcu.funcInfo(func_index).owner_nav;
4405 const fqn = ip.getNav(nav).fqn;
4406
4407 const codegen_prog_node = zcu.codegen_prog_node.start(fqn.toSlice(ip), 0);
4408 defer codegen_prog_node.end();
4409
4410 if (codegen.legalizeFeatures(pt, nav)) |features| {
4411 try air.legalize(pt, features);
4412 }
4413
4414 var liveness: Air.Liveness = try .analyze(zcu, air.*, ip);
4415 defer liveness.deinit(gpa);
4416
4417 // TODO: surely writing to stderr from n threads simultaneously will work flawlessly
4418 if (build_options.enable_debug_extensions and comp.verbose_air) {
4419 std.debug.print("# Begin Function AIR: {}:\n", .{fqn.fmt(ip)});
4420 air.dump(pt, liveness);
4421 std.debug.print("# End Function AIR: {}\n\n", .{fqn.fmt(ip)});
4422 }
4423
4424 if (std.debug.runtime_safety) {
4425 var verify: Air.Liveness.Verify = .{
4426 .gpa = gpa,
4427 .zcu = zcu,
4428 .air = air.*,
4429 .liveness = liveness,
4430 .intern_pool = ip,
4431 };
4432 defer verify.deinit();
4433
4434 verify.verify() catch |err| switch (err) {
4435 error.OutOfMemory => return error.OutOfMemory,
4436 else => return zcu.codegenFail(nav, "invalid liveness: {s}", .{@errorName(err)}),
4437 };
4438 }
4439
4440 // The LLVM backend is special, because we only need to do codegen. There is no equivalent to the
4441 // "emit" step because LLVM does not support incremental linking. Our linker (LLD or self-hosted)
4442 // will just see the ZCU object file which LLVM ultimately emits.
4443 if (zcu.llvm_object) |llvm_object| {
4444 return llvm_object.updateFunc(pt, func_index, air, &liveness);
4445 }
4446
4447 const lf = comp.bin_file orelse return error.NoLinkFile;
4448 return codegen.generateFunction(lf, pt, zcu.navSrcLoc(nav), func_index, air, &liveness) catch |err| switch (err) {
4449 error.OutOfMemory,
4450 error.CodegenFail,
4451 => |e| return e,
4452 error.Overflow,
4453 error.RelocationNotByteAligned,
4454 => return zcu.codegenFail(nav, "unable to codegen: {s}", .{@errorName(err)}),
4455 };
4456}
src/codegen.zig+93-4
...@@ -85,16 +85,104 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co...@@ -85,16 +85,104 @@ pub fn legalizeFeatures(pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) ?*co
85 }85 }
86}86}
8787
88/// Every code generation backend has a different MIR representation. However, we want to pass
89/// MIR from codegen to the linker *regardless* of which backend is in use. So, we use this: a
90/// union of all MIR types. The active tag is known from the backend in use; see `AnyMir.tag`.
91pub const AnyMir = union {
92 aarch64: @import("arch/aarch64/Mir.zig"),
93 arm: @import("arch/arm/Mir.zig"),
94 powerpc: noreturn, //@import("arch/powerpc/Mir.zig"),
95 riscv64: @import("arch/riscv64/Mir.zig"),
96 sparc64: @import("arch/sparc64/Mir.zig"),
97 x86_64: @import("arch/x86_64/Mir.zig"),
98 wasm: @import("arch/wasm/Mir.zig"),
99 c: @import("codegen/c.zig").Mir,
100
101 pub inline fn tag(comptime backend: std.builtin.CompilerBackend) []const u8 {
102 return switch (backend) {
103 .stage2_aarch64 => "aarch64",
104 .stage2_arm => "arm",
105 .stage2_powerpc => "powerpc",
106 .stage2_riscv64 => "riscv64",
107 .stage2_sparc64 => "sparc64",
108 .stage2_x86_64 => "x86_64",
109 .stage2_wasm => "wasm",
110 .stage2_c => "c",
111 else => unreachable,
112 };
113 }
114
115 pub fn deinit(mir: *AnyMir, zcu: *const Zcu) void {
116 const gpa = zcu.gpa;
117 const backend = target_util.zigBackend(zcu.root_mod.resolved_target.result, zcu.comp.config.use_llvm);
118 switch (backend) {
119 else => unreachable,
120 inline .stage2_aarch64,
121 .stage2_arm,
122 .stage2_powerpc,
123 .stage2_riscv64,
124 .stage2_sparc64,
125 .stage2_x86_64,
126 .stage2_c,
127 => |backend_ct| @field(mir, tag(backend_ct)).deinit(gpa),
128 }
129 }
130};
131
132/// Runs code generation for a function. This process converts the `Air` emitted by `Sema`,
133/// alongside annotated `Liveness` data, to machine code in the form of MIR (see `AnyMir`).
134///
135/// This is supposed to be a "pure" process, but some backends are currently buggy; see
136/// `Zcu.Feature.separate_thread` for details.
88pub fn generateFunction(137pub fn generateFunction(
89 lf: *link.File,138 lf: *link.File,
90 pt: Zcu.PerThread,139 pt: Zcu.PerThread,
91 src_loc: Zcu.LazySrcLoc,140 src_loc: Zcu.LazySrcLoc,
92 func_index: InternPool.Index,141 func_index: InternPool.Index,
93 air: Air,142 air: *const Air,
94 liveness: Air.Liveness,143 liveness: *const Air.Liveness,
144) CodeGenError!AnyMir {
145 const zcu = pt.zcu;
146 const func = zcu.funcInfo(func_index);
147 const target = zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;
148 switch (target_util.zigBackend(target, false)) {
149 else => unreachable,
150 inline .stage2_aarch64,
151 .stage2_arm,
152 .stage2_powerpc,
153 .stage2_riscv64,
154 .stage2_sparc64,
155 .stage2_x86_64,
156 .stage2_c,
157 => |backend| {
158 dev.check(devFeatureForBackend(backend));
159 const CodeGen = importBackend(backend);
160 const mir = try CodeGen.generate(lf, pt, src_loc, func_index, air, liveness);
161 return @unionInit(AnyMir, AnyMir.tag(backend), mir);
162 },
163 }
164}
165
166/// Converts the MIR returned by `generateFunction` to finalized machine code to be placed in
167/// the output binary. This is called from linker implementations, and may query linker state.
168///
169/// This function is not called for the C backend, as `link.C` directly understands its MIR.
170///
171/// The `air` parameter is not supposed to exist, but some backends are currently buggy; see
172/// `Zcu.Feature.separate_thread` for details.
173pub fn emitFunction(
174 lf: *link.File,
175 pt: Zcu.PerThread,
176 src_loc: Zcu.LazySrcLoc,
177 func_index: InternPool.Index,
178 any_mir: *const AnyMir,
95 code: *std.ArrayListUnmanaged(u8),179 code: *std.ArrayListUnmanaged(u8),
96 debug_output: link.File.DebugInfoOutput,180 debug_output: link.File.DebugInfoOutput,
97) CodeGenError!void {181 /// TODO: this parameter needs to be removed. We should not still hold AIR this late
182 /// in the pipeline. Any information needed to call emit must be stored in MIR.
183 /// This is `undefined` if the backend supports the `separate_thread` feature.
184 air: *const Air,
185) Allocator.Error!void {
98 const zcu = pt.zcu;186 const zcu = pt.zcu;
99 const func = zcu.funcInfo(func_index);187 const func = zcu.funcInfo(func_index);
100 const target = zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;188 const target = zcu.navFileScope(func.owner_nav).mod.?.resolved_target.result;
...@@ -108,7 +196,8 @@ pub fn generateFunction(...@@ -108,7 +196,8 @@ pub fn generateFunction(
108 .stage2_x86_64,196 .stage2_x86_64,
109 => |backend| {197 => |backend| {
110 dev.check(devFeatureForBackend(backend));198 dev.check(devFeatureForBackend(backend));
111 return importBackend(backend).generate(lf, pt, src_loc, func_index, air, liveness, code, debug_output);199 const mir = &@field(any_mir, AnyMir.tag(backend));
200 return mir.emit(lf, pt, src_loc, func_index, code, debug_output, air);
112 },201 },
113 }202 }
114}203}
src/codegen/c.zig+123-23
...@@ -3,6 +3,7 @@ const builtin = @import("builtin");...@@ -3,6 +3,7 @@ const builtin = @import("builtin");
3const assert = std.debug.assert;3const assert = std.debug.assert;
4const mem = std.mem;4const mem = std.mem;
5const log = std.log.scoped(.c);5const log = std.log.scoped(.c);
6const Allocator = mem.Allocator;
67
7const dev = @import("../dev.zig");8const dev = @import("../dev.zig");
8const link = @import("../link.zig");9const link = @import("../link.zig");
...@@ -30,6 +31,35 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {...@@ -30,6 +31,35 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features {
30 }) else null; // we don't currently ask zig1 to use safe optimization modes31 }) else null; // we don't currently ask zig1 to use safe optimization modes
31}32}
3233
34/// For most backends, MIR is basically a sequence of machine code instructions, perhaps with some
35/// "pseudo instructions" thrown in. For the C backend, it is instead the generated C code for a
36/// single function. We also need to track some information to get merged into the global `link.C`
37/// state, including:
38/// * The UAVs used, so declarations can be emitted in `flush`
39/// * The types used, so declarations can be emitted in `flush`
40/// * The lazy functions used, so definitions can be emitted in `flush`
41pub const Mir = struct {
42 /// This map contains all the UAVs we saw generating this function.
43 /// `link.C` will merge them into its `uavs`/`aligned_uavs` fields.
44 /// Key is the value of the UAV; value is the UAV's alignment, or
45 /// `.none` for natural alignment. The specified alignment is never
46 /// less than the natural alignment.
47 uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment),
48 // These remaining fields are essentially just an owned version of `link.C.AvBlock`.
49 code: []u8,
50 fwd_decl: []u8,
51 ctype_pool: CType.Pool,
52 lazy_fns: LazyFnMap,
53
54 pub fn deinit(mir: *Mir, gpa: Allocator) void {
55 mir.uavs.deinit(gpa);
56 gpa.free(mir.code);
57 gpa.free(mir.fwd_decl);
58 mir.ctype_pool.deinit(gpa);
59 mir.lazy_fns.deinit(gpa);
60 }
61};
62
33pub const CType = @import("c/Type.zig");63pub const CType = @import("c/Type.zig");
3464
35pub const CValue = union(enum) {65pub const CValue = union(enum) {
...@@ -671,7 +701,7 @@ pub const Object = struct {...@@ -671,7 +701,7 @@ pub const Object = struct {
671701
672/// This data is available both when outputting .c code and when outputting an .h file.702/// This data is available both when outputting .c code and when outputting an .h file.
673pub const DeclGen = struct {703pub const DeclGen = struct {
674 gpa: mem.Allocator,704 gpa: Allocator,
675 pt: Zcu.PerThread,705 pt: Zcu.PerThread,
676 mod: *Module,706 mod: *Module,
677 pass: Pass,707 pass: Pass,
...@@ -682,10 +712,12 @@ pub const DeclGen = struct {...@@ -682,10 +712,12 @@ pub const DeclGen = struct {
682 error_msg: ?*Zcu.ErrorMsg,712 error_msg: ?*Zcu.ErrorMsg,
683 ctype_pool: CType.Pool,713 ctype_pool: CType.Pool,
684 scratch: std.ArrayListUnmanaged(u32),714 scratch: std.ArrayListUnmanaged(u32),
685 /// Keeps track of anonymous decls that need to be rendered before this715 /// This map contains all the UAVs we saw generating this function.
686 /// (named) Decl in the output C code.716 /// `link.C` will merge them into its `uavs`/`aligned_uavs` fields.
687 uav_deps: std.AutoArrayHashMapUnmanaged(InternPool.Index, C.AvBlock),717 /// Key is the value of the UAV; value is the UAV's alignment, or
688 aligned_uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment),718 /// `.none` for natural alignment. The specified alignment is never
719 /// less than the natural alignment.
720 uavs: std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment),
689721
690 pub const Pass = union(enum) {722 pub const Pass = union(enum) {
691 nav: InternPool.Nav.Index,723 nav: InternPool.Nav.Index,
...@@ -753,21 +785,17 @@ pub const DeclGen = struct {...@@ -753,21 +785,17 @@ pub const DeclGen = struct {
753 // Indicate that the anon decl should be rendered to the output so that785 // Indicate that the anon decl should be rendered to the output so that
754 // our reference above is not undefined.786 // our reference above is not undefined.
755 const ptr_type = ip.indexToKey(uav.orig_ty).ptr_type;787 const ptr_type = ip.indexToKey(uav.orig_ty).ptr_type;
756 const gop = try dg.uav_deps.getOrPut(dg.gpa, uav.val);788 const gop = try dg.uavs.getOrPut(dg.gpa, uav.val);
757 if (!gop.found_existing) gop.value_ptr.* = .{};789 if (!gop.found_existing) gop.value_ptr.* = .none;
758790 // If there is an explicit alignment, greater than the current one, use it.
759 // Only insert an alignment entry if the alignment is greater than ABI791 // Note that we intentionally start at `.none`, so `gop.value_ptr.*` is never
760 // alignment. If there is already an entry, keep the greater alignment.792 // underaligned, so we don't need to worry about the `.none` case here.
761 const explicit_alignment = ptr_type.flags.alignment;793 if (ptr_type.flags.alignment != .none) {
762 if (explicit_alignment != .none) {794 // Resolve the current alignment so we can choose the bigger one.
763 const abi_alignment = Type.fromInterned(ptr_type.child).abiAlignment(zcu);795 const cur_alignment: Alignment = if (gop.value_ptr.* == .none) abi: {
764 if (explicit_alignment.order(abi_alignment).compare(.gt)) {796 break :abi Type.fromInterned(ptr_type.child).abiAlignment(zcu);
765 const aligned_gop = try dg.aligned_uavs.getOrPut(dg.gpa, uav.val);797 } else gop.value_ptr.*;
766 aligned_gop.value_ptr.* = if (aligned_gop.found_existing)798 gop.value_ptr.* = cur_alignment.maxStrict(ptr_type.flags.alignment);
767 aligned_gop.value_ptr.maxStrict(explicit_alignment)
768 else
769 explicit_alignment;
770 }
771 }799 }
772 }800 }
773801
...@@ -2895,7 +2923,79 @@ pub fn genLazyFn(o: *Object, lazy_ctype_pool: *const CType.Pool, lazy_fn: LazyFn...@@ -2895,7 +2923,79 @@ pub fn genLazyFn(o: *Object, lazy_ctype_pool: *const CType.Pool, lazy_fn: LazyFn
2895 }2923 }
2896}2924}
28972925
2898pub fn genFunc(f: *Function) !void {2926pub fn generate(
2927 lf: *link.File,
2928 pt: Zcu.PerThread,
2929 src_loc: Zcu.LazySrcLoc,
2930 func_index: InternPool.Index,
2931 air: *const Air,
2932 liveness: *const Air.Liveness,
2933) @import("../codegen.zig").CodeGenError!Mir {
2934 const zcu = pt.zcu;
2935 const gpa = zcu.gpa;
2936
2937 _ = src_loc;
2938 assert(lf.tag == .c);
2939
2940 const func = zcu.funcInfo(func_index);
2941
2942 var function: Function = .{
2943 .value_map = .init(gpa),
2944 .air = air.*,
2945 .liveness = liveness.*,
2946 .func_index = func_index,
2947 .object = .{
2948 .dg = .{
2949 .gpa = gpa,
2950 .pt = pt,
2951 .mod = zcu.navFileScope(func.owner_nav).mod.?,
2952 .error_msg = null,
2953 .pass = .{ .nav = func.owner_nav },
2954 .is_naked_fn = Type.fromInterned(func.ty).fnCallingConvention(zcu) == .naked,
2955 .expected_block = null,
2956 .fwd_decl = .init(gpa),
2957 .ctype_pool = .empty,
2958 .scratch = .empty,
2959 .uavs = .empty,
2960 },
2961 .code = .init(gpa),
2962 .indent_writer = undefined, // set later so we can get a pointer to object.code
2963 },
2964 .lazy_fns = .empty,
2965 };
2966 defer {
2967 function.object.code.deinit();
2968 function.object.dg.fwd_decl.deinit();
2969 function.object.dg.ctype_pool.deinit(gpa);
2970 function.object.dg.scratch.deinit(gpa);
2971 function.object.dg.uavs.deinit(gpa);
2972 function.deinit();
2973 }
2974 try function.object.dg.ctype_pool.init(gpa);
2975 function.object.indent_writer = .{ .underlying_writer = function.object.code.writer() };
2976
2977 genFunc(&function) catch |err| switch (err) {
2978 error.AnalysisFail => return zcu.codegenFailMsg(func.owner_nav, function.object.dg.error_msg.?),
2979 error.OutOfMemory => |e| return e,
2980 };
2981
2982 var mir: Mir = .{
2983 .uavs = .empty,
2984 .code = &.{},
2985 .fwd_decl = &.{},
2986 .ctype_pool = .empty,
2987 .lazy_fns = .empty,
2988 };
2989 errdefer mir.deinit(gpa);
2990 mir.uavs = function.object.dg.uavs.move();
2991 mir.code = try function.object.code.toOwnedSlice();
2992 mir.fwd_decl = try function.object.dg.fwd_decl.toOwnedSlice();
2993 mir.ctype_pool = function.object.dg.ctype_pool.move();
2994 mir.lazy_fns = function.lazy_fns.move();
2995 return mir;
2996}
2997
2998fn genFunc(f: *Function) !void {
2899 const tracy = trace(@src());2999 const tracy = trace(@src());
2900 defer tracy.end();3000 defer tracy.end();
29013001
...@@ -8482,7 +8582,7 @@ fn iterateBigTomb(f: *Function, inst: Air.Inst.Index) BigTomb {...@@ -8482,7 +8582,7 @@ fn iterateBigTomb(f: *Function, inst: Air.Inst.Index) BigTomb {
84828582
8483/// A naive clone of this map would create copies of the ArrayList which is8583/// A naive clone of this map would create copies of the ArrayList which is
8484/// stored in the values. This function additionally clones the values.8584/// stored in the values. This function additionally clones the values.
8485fn cloneFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) !LocalsMap {8585fn cloneFreeLocalsMap(gpa: Allocator, map: *LocalsMap) !LocalsMap {
8486 var cloned = try map.clone(gpa);8586 var cloned = try map.clone(gpa);
8487 const values = cloned.values();8587 const values = cloned.values();
8488 var i: usize = 0;8588 var i: usize = 0;
...@@ -8499,7 +8599,7 @@ fn cloneFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) !LocalsMap {...@@ -8499,7 +8599,7 @@ fn cloneFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) !LocalsMap {
8499 return cloned;8599 return cloned;
8500}8600}
85018601
8502fn deinitFreeLocalsMap(gpa: mem.Allocator, map: *LocalsMap) void {8602fn deinitFreeLocalsMap(gpa: Allocator, map: *LocalsMap) void {
8503 for (map.values()) |*value| {8603 for (map.values()) |*value| {
8504 value.deinit(gpa);8604 value.deinit(gpa);
8505 }8605 }
src/codegen/llvm.zig+10-12
...@@ -1121,8 +1121,8 @@ pub const Object = struct {...@@ -1121,8 +1121,8 @@ pub const Object = struct {
1121 o: *Object,1121 o: *Object,
1122 pt: Zcu.PerThread,1122 pt: Zcu.PerThread,
1123 func_index: InternPool.Index,1123 func_index: InternPool.Index,
1124 air: Air,1124 air: *const Air,
1125 liveness: Air.Liveness,1125 liveness: *const Air.Liveness,
1126 ) !void {1126 ) !void {
1127 assert(std.meta.eql(pt, o.pt));1127 assert(std.meta.eql(pt, o.pt));
1128 const zcu = pt.zcu;1128 const zcu = pt.zcu;
...@@ -1479,8 +1479,8 @@ pub const Object = struct {...@@ -1479,8 +1479,8 @@ pub const Object = struct {
14791479
1480 var fg: FuncGen = .{1480 var fg: FuncGen = .{
1481 .gpa = gpa,1481 .gpa = gpa,
1482 .air = air,1482 .air = air.*,
1483 .liveness = liveness,1483 .liveness = liveness.*,
1484 .ng = &ng,1484 .ng = &ng,
1485 .wip = wip,1485 .wip = wip,
1486 .is_naked = fn_info.cc == .naked,1486 .is_naked = fn_info.cc == .naked,
...@@ -1506,10 +1506,9 @@ pub const Object = struct {...@@ -1506,10 +1506,9 @@ pub const Object = struct {
1506 deinit_wip = false;1506 deinit_wip = false;
15071507
1508 fg.genBody(air.getMainBody(), .poi) catch |err| switch (err) {1508 fg.genBody(air.getMainBody(), .poi) catch |err| switch (err) {
1509 error.CodegenFail => {1509 error.CodegenFail => switch (zcu.codegenFailMsg(func.owner_nav, ng.err_msg.?)) {
1510 try zcu.failed_codegen.put(gpa, func.owner_nav, ng.err_msg.?);1510 error.CodegenFail => return,
1511 ng.err_msg = null;1511 error.OutOfMemory => |e| return e,
1512 return;
1513 },1512 },
1514 else => |e| return e,1513 else => |e| return e,
1515 };1514 };
...@@ -1561,10 +1560,9 @@ pub const Object = struct {...@@ -1561,10 +1560,9 @@ pub const Object = struct {
1561 .err_msg = null,1560 .err_msg = null,
1562 };1561 };
1563 ng.genDecl() catch |err| switch (err) {1562 ng.genDecl() catch |err| switch (err) {
1564 error.CodegenFail => {1563 error.CodegenFail => switch (pt.zcu.codegenFailMsg(nav_index, ng.err_msg.?)) {
1565 try pt.zcu.failed_codegen.put(pt.zcu.gpa, nav_index, ng.err_msg.?);1564 error.CodegenFail => return,
1566 ng.err_msg = null;1565 error.OutOfMemory => |e| return e,
1567 return;
1568 },1566 },
1569 else => |e| return e,1567 else => |e| return e,
1570 };1568 };
src/codegen/spirv.zig+3-2
...@@ -230,8 +230,9 @@ pub const Object = struct {...@@ -230,8 +230,9 @@ pub const Object = struct {
230 defer nav_gen.deinit();230 defer nav_gen.deinit();
231231
232 nav_gen.genNav(do_codegen) catch |err| switch (err) {232 nav_gen.genNav(do_codegen) catch |err| switch (err) {
233 error.CodegenFail => {233 error.CodegenFail => switch (zcu.codegenFailMsg(nav_index, nav_gen.error_msg.?)) {
234 try zcu.failed_codegen.put(gpa, nav_index, nav_gen.error_msg.?);234 error.CodegenFail => {},
235 error.OutOfMemory => |e| return e,
235 },236 },
236 else => |other| {237 else => |other| {
237 // There might be an error that happened *after* self.error_msg238 // There might be an error that happened *after* self.error_msg
src/dev.zig+9
...@@ -25,6 +25,9 @@ pub const Env = enum {...@@ -25,6 +25,9 @@ pub const Env = enum {
25 /// - `zig build-* -fno-emit-bin`25 /// - `zig build-* -fno-emit-bin`
26 sema,26 sema,
2727
28 /// - `zig build-* -ofmt=c`
29 cbe,
30
28 /// - sema31 /// - sema
29 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-linux --listen=-`32 /// - `zig build-* -fincremental -fno-llvm -fno-lld -target x86_64-linux --listen=-`
30 @"x86_64-linux",33 @"x86_64-linux",
...@@ -144,6 +147,12 @@ pub const Env = enum {...@@ -144,6 +147,12 @@ pub const Env = enum {
144 => true,147 => true,
145 else => Env.ast_gen.supports(feature),148 else => Env.ast_gen.supports(feature),
146 },149 },
150 .cbe => switch (feature) {
151 .c_backend,
152 .c_linker,
153 => true,
154 else => Env.sema.supports(feature),
155 },
147 .@"x86_64-linux" => switch (feature) {156 .@"x86_64-linux" => switch (feature) {
148 .build_command,157 .build_command,
149 .stdio_listen,158 .stdio_listen,
src/libs/freebsd.zig+1-1
...@@ -1004,7 +1004,7 @@ fn queueSharedObjects(comp: *Compilation, so_files: BuiltSharedObjects) void {...@@ -1004,7 +1004,7 @@ fn queueSharedObjects(comp: *Compilation, so_files: BuiltSharedObjects) void {
1004 }1004 }
1005 }1005 }
10061006
1007 comp.queueLinkTasks(task_buffer[0..task_buffer_i]);1007 comp.queuePrelinkTasks(task_buffer[0..task_buffer_i]);
1008}1008}
10091009
1010fn buildSharedLib(1010fn buildSharedLib(
src/libs/glibc.zig+1-1
...@@ -1170,7 +1170,7 @@ fn queueSharedObjects(comp: *Compilation, so_files: BuiltSharedObjects) void {...@@ -1170,7 +1170,7 @@ fn queueSharedObjects(comp: *Compilation, so_files: BuiltSharedObjects) void {
1170 }1170 }
1171 }1171 }
11721172
1173 comp.queueLinkTasks(task_buffer[0..task_buffer_i]);1173 comp.queuePrelinkTasks(task_buffer[0..task_buffer_i]);
1174}1174}
11751175
1176fn buildSharedLib(1176fn buildSharedLib(
src/libs/libcxx.zig+2-2
...@@ -308,7 +308,7 @@ pub fn buildLibCxx(comp: *Compilation, prog_node: std.Progress.Node) BuildError!...@@ -308,7 +308,7 @@ pub fn buildLibCxx(comp: *Compilation, prog_node: std.Progress.Node) BuildError!
308 assert(comp.libcxx_static_lib == null);308 assert(comp.libcxx_static_lib == null);
309 const crt_file = try sub_compilation.toCrtFile();309 const crt_file = try sub_compilation.toCrtFile();
310 comp.libcxx_static_lib = crt_file;310 comp.libcxx_static_lib = crt_file;
311 comp.queueLinkTaskMode(crt_file.full_object_path, &config);311 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
312}312}
313313
314pub fn buildLibCxxAbi(comp: *Compilation, prog_node: std.Progress.Node) BuildError!void {314pub fn buildLibCxxAbi(comp: *Compilation, prog_node: std.Progress.Node) BuildError!void {
...@@ -504,7 +504,7 @@ pub fn buildLibCxxAbi(comp: *Compilation, prog_node: std.Progress.Node) BuildErr...@@ -504,7 +504,7 @@ pub fn buildLibCxxAbi(comp: *Compilation, prog_node: std.Progress.Node) BuildErr
504 assert(comp.libcxxabi_static_lib == null);504 assert(comp.libcxxabi_static_lib == null);
505 const crt_file = try sub_compilation.toCrtFile();505 const crt_file = try sub_compilation.toCrtFile();
506 comp.libcxxabi_static_lib = crt_file;506 comp.libcxxabi_static_lib = crt_file;
507 comp.queueLinkTaskMode(crt_file.full_object_path, &config);507 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
508}508}
509509
510pub fn addCxxArgs(510pub fn addCxxArgs(
src/libs/libtsan.zig+1-1
...@@ -325,7 +325,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: std.Progress.Node) BuildError!vo...@@ -325,7 +325,7 @@ pub fn buildTsan(comp: *Compilation, prog_node: std.Progress.Node) BuildError!vo
325 };325 };
326326
327 const crt_file = try sub_compilation.toCrtFile();327 const crt_file = try sub_compilation.toCrtFile();
328 comp.queueLinkTaskMode(crt_file.full_object_path, &config);328 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
329 assert(comp.tsan_lib == null);329 assert(comp.tsan_lib == null);
330 comp.tsan_lib = crt_file;330 comp.tsan_lib = crt_file;
331}331}
src/libs/libunwind.zig+1-1
...@@ -195,7 +195,7 @@ pub fn buildStaticLib(comp: *Compilation, prog_node: std.Progress.Node) BuildErr...@@ -195,7 +195,7 @@ pub fn buildStaticLib(comp: *Compilation, prog_node: std.Progress.Node) BuildErr
195 };195 };
196196
197 const crt_file = try sub_compilation.toCrtFile();197 const crt_file = try sub_compilation.toCrtFile();
198 comp.queueLinkTaskMode(crt_file.full_object_path, &config);198 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
199 assert(comp.libunwind_static_lib == null);199 assert(comp.libunwind_static_lib == null);
200 comp.libunwind_static_lib = crt_file;200 comp.libunwind_static_lib = crt_file;
201}201}
src/libs/musl.zig+1-1
...@@ -278,7 +278,7 @@ pub fn buildCrtFile(comp: *Compilation, in_crt_file: CrtFile, prog_node: std.Pro...@@ -278,7 +278,7 @@ pub fn buildCrtFile(comp: *Compilation, in_crt_file: CrtFile, prog_node: std.Pro
278 errdefer comp.gpa.free(basename);278 errdefer comp.gpa.free(basename);
279279
280 const crt_file = try sub_compilation.toCrtFile();280 const crt_file = try sub_compilation.toCrtFile();
281 comp.queueLinkTaskMode(crt_file.full_object_path, &config);281 comp.queuePrelinkTaskMode(crt_file.full_object_path, &config);
282 {282 {
283 comp.mutex.lock();283 comp.mutex.lock();
284 defer comp.mutex.unlock();284 defer comp.mutex.unlock();
src/libs/netbsd.zig+1-1
...@@ -669,7 +669,7 @@ fn queueSharedObjects(comp: *Compilation, so_files: BuiltSharedObjects) void {...@@ -669,7 +669,7 @@ fn queueSharedObjects(comp: *Compilation, so_files: BuiltSharedObjects) void {
669 }669 }
670 }670 }
671671
672 comp.queueLinkTasks(task_buffer[0..task_buffer_i]);672 comp.queuePrelinkTasks(task_buffer[0..task_buffer_i]);
673}673}
674674
675fn buildSharedLib(675fn buildSharedLib(
src/link.zig+95-94
...@@ -21,11 +21,11 @@ const Type = @import("Type.zig");...@@ -21,11 +21,11 @@ const Type = @import("Type.zig");
21const Value = @import("Value.zig");21const Value = @import("Value.zig");
22const Package = @import("Package.zig");22const Package = @import("Package.zig");
23const dev = @import("dev.zig");23const dev = @import("dev.zig");
24const ThreadSafeQueue = @import("ThreadSafeQueue.zig").ThreadSafeQueue;
25const target_util = @import("target.zig");24const target_util = @import("target.zig");
26const codegen = @import("codegen.zig");25const codegen = @import("codegen.zig");
2726
28pub const LdScript = @import("link/LdScript.zig");27pub const LdScript = @import("link/LdScript.zig");
28pub const Queue = @import("link/Queue.zig");
2929
30pub const Diags = struct {30pub const Diags = struct {
31 /// Stored here so that function definitions can distinguish between31 /// Stored here so that function definitions can distinguish between
...@@ -741,21 +741,26 @@ pub const File = struct {...@@ -741,21 +741,26 @@ pub const File = struct {
741 }741 }
742742
743 /// May be called before or after updateExports for any given Decl.743 /// May be called before or after updateExports for any given Decl.
744 /// TODO: currently `pub` because `Zcu.PerThread` is calling this.744 /// The active tag of `mir` is determined by the backend used for the module this function is in.
745 /// Never called when LLVM is codegenning the ZCU.745 /// Never called when LLVM is codegenning the ZCU.
746 pub fn updateFunc(746 fn updateFunc(
747 base: *File,747 base: *File,
748 pt: Zcu.PerThread,748 pt: Zcu.PerThread,
749 func_index: InternPool.Index,749 func_index: InternPool.Index,
750 air: Air,750 /// This is owned by the caller, but the callee is permitted to mutate it provided
751 liveness: Air.Liveness,751 /// that `mir.deinit` remains legal for the caller. For instance, the callee can
752 /// take ownership of an embedded slice and replace it with `&.{}` in `mir`.
753 mir: *codegen.AnyMir,
754 /// This may be `undefined`; only pass it to `emitFunction`.
755 /// This parameter will eventually be removed.
756 maybe_undef_air: *const Air,
752 ) UpdateNavError!void {757 ) UpdateNavError!void {
753 assert(base.comp.zcu.?.llvm_object == null);758 assert(base.comp.zcu.?.llvm_object == null);
754 switch (base.tag) {759 switch (base.tag) {
755 .lld => unreachable,760 .lld => unreachable,
756 inline else => |tag| {761 inline else => |tag| {
757 dev.check(tag.devFeature());762 dev.check(tag.devFeature());
758 return @as(*tag.Type(), @fieldParentPtr("base", base)).updateFunc(pt, func_index, air, liveness);763 return @as(*tag.Type(), @fieldParentPtr("base", base)).updateFunc(pt, func_index, mir, maybe_undef_air);
759 },764 },
760 }765 }
761 }766 }
...@@ -1213,40 +1218,7 @@ pub const File = struct {...@@ -1213,40 +1218,7 @@ pub const File = struct {
1213 pub const Dwarf = @import("link/Dwarf.zig");1218 pub const Dwarf = @import("link/Dwarf.zig");
1214};1219};
12151220
1216/// Does all the tasks in the queue. Runs in exactly one separate thread1221pub const PrelinkTask = union(enum) {
1217/// from the rest of compilation. All tasks performed here are
1218/// single-threaded with respect to one another.
1219pub fn flushTaskQueue(tid: usize, comp: *Compilation) void {
1220 const diags = &comp.link_diags;
1221 // As soon as check() is called, another `flushTaskQueue` call could occur,
1222 // so the safety lock must go after the check.
1223 while (comp.link_task_queue.check()) |tasks| {
1224 comp.link_task_queue_safety.lock();
1225 defer comp.link_task_queue_safety.unlock();
1226
1227 if (comp.remaining_prelink_tasks > 0) {
1228 comp.link_task_queue_postponed.ensureUnusedCapacity(comp.gpa, tasks.len) catch |err| switch (err) {
1229 error.OutOfMemory => return diags.setAllocFailure(),
1230 };
1231 }
1232
1233 for (tasks) |task| doTask(comp, tid, task);
1234
1235 if (comp.remaining_prelink_tasks == 0) {
1236 if (comp.bin_file) |base| if (!base.post_prelink) {
1237 base.prelink(comp.work_queue_progress_node) catch |err| switch (err) {
1238 error.OutOfMemory => diags.setAllocFailure(),
1239 error.LinkFailure => continue,
1240 };
1241 base.post_prelink = true;
1242 for (comp.link_task_queue_postponed.items) |task| doTask(comp, tid, task);
1243 comp.link_task_queue_postponed.clearRetainingCapacity();
1244 };
1245 }
1246 }
1247}
1248
1249pub const Task = union(enum) {
1250 /// Loads the objects, shared objects, and archives that are already1222 /// Loads the objects, shared objects, and archives that are already
1251 /// known from the command line.1223 /// known from the command line.
1252 load_explicitly_provided,1224 load_explicitly_provided,
...@@ -1264,31 +1236,70 @@ pub const Task = union(enum) {...@@ -1264,31 +1236,70 @@ pub const Task = union(enum) {
1264 /// Tells the linker to load an input which could be an object file,1236 /// Tells the linker to load an input which could be an object file,
1265 /// archive, or shared library.1237 /// archive, or shared library.
1266 load_input: Input,1238 load_input: Input,
12671239};
1240pub const ZcuTask = union(enum) {
1268 /// Write the constant value for a Decl to the output file.1241 /// Write the constant value for a Decl to the output file.
1269 link_nav: InternPool.Nav.Index,1242 link_nav: InternPool.Nav.Index,
1270 /// Write the machine code for a function to the output file.1243 /// Write the machine code for a function to the output file.
1271 link_func: CodegenFunc,1244 link_func: LinkFunc,
1272 link_type: InternPool.Index,1245 link_type: InternPool.Index,
1273
1274 update_line_number: InternPool.TrackedInst.Index,1246 update_line_number: InternPool.TrackedInst.Index,
12751247 pub fn deinit(task: ZcuTask, zcu: *const Zcu) void {
1276 pub const CodegenFunc = struct {1248 switch (task) {
1249 .link_nav,
1250 .link_type,
1251 .update_line_number,
1252 => {},
1253 .link_func => |link_func| {
1254 switch (link_func.mir.status.load(.monotonic)) {
1255 .pending => unreachable, // cannot deinit until MIR done
1256 .failed => {}, // MIR not populated so doesn't need freeing
1257 .ready => link_func.mir.value.deinit(zcu),
1258 }
1259 zcu.gpa.destroy(link_func.mir);
1260 },
1261 }
1262 }
1263 pub const LinkFunc = struct {
1277 /// This will either be a non-generic `func_decl` or a `func_instance`.1264 /// This will either be a non-generic `func_decl` or a `func_instance`.
1278 func: InternPool.Index,1265 func: InternPool.Index,
1279 /// This `Air` is owned by the `Job` and allocated with `gpa`.1266 /// This pointer is allocated into `gpa` and must be freed when the `ZcuTask` is processed.
1280 /// It must be deinited when the job is processed.1267 /// The pointer is shared with the codegen worker, which will populate the MIR inside once
1281 air: Air,1268 /// it has been generated. It's important that the `link_func` is queued at the same time as
1269 /// the codegen job to ensure that the linker receives functions in a deterministic order,
1270 /// allowing reproducible builds.
1271 mir: *SharedMir,
1272 /// This field exists only due to deficiencies in some codegen implementations; it should
1273 /// be removed when the corresponding parameter of `CodeGen.emitFunction` can be removed.
1274 /// This is `undefined` if `Zcu.Feature.separate_thread` is supported.
1275 /// If this is defined, its memory is owned externally; do not `deinit` this `air`.
1276 air: *const Air,
1277
1278 pub const SharedMir = struct {
1279 /// This is initially `.pending`. When `value` is populated, the codegen thread will set
1280 /// this to `.ready`, and alert the queue if needed. It could also end up `.failed`.
1281 /// The action of storing a value (other than `.pending`) to this atomic transfers
1282 /// ownership of memory assoicated with `value` to this `ZcuTask`.
1283 status: std.atomic.Value(enum(u8) {
1284 /// We are waiting on codegen to generate MIR (or die trying).
1285 pending,
1286 /// `value` is not populated and will not be populated. Just drop the task from the queue and move on.
1287 failed,
1288 /// `value` is populated with the MIR from the backend in use, which is not LLVM.
1289 ready,
1290 }),
1291 /// This is `undefined` until `ready` is set to `true`. Once populated, this MIR belongs
1292 /// to the `ZcuTask`, and must be `deinit`ed when it is processed. Allocated into `gpa`.
1293 value: codegen.AnyMir,
1294 };
1282 };1295 };
1283};1296};
12841297
1285pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {1298pub fn doPrelinkTask(comp: *Compilation, task: PrelinkTask) void {
1286 const diags = &comp.link_diags;1299 const diags = &comp.link_diags;
1300 const base = comp.bin_file orelse return;
1287 switch (task) {1301 switch (task) {
1288 .load_explicitly_provided => {1302 .load_explicitly_provided => {
1289 comp.remaining_prelink_tasks -= 1;
1290 const base = comp.bin_file orelse return;
1291
1292 const prog_node = comp.work_queue_progress_node.start("Parse Linker Inputs", comp.link_inputs.len);1303 const prog_node = comp.work_queue_progress_node.start("Parse Linker Inputs", comp.link_inputs.len);
1293 defer prog_node.end();1304 defer prog_node.end();
1294 for (comp.link_inputs) |input| {1305 for (comp.link_inputs) |input| {
...@@ -1306,9 +1317,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1306,9 +1317,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1306 }1317 }
1307 },1318 },
1308 .load_host_libc => {1319 .load_host_libc => {
1309 comp.remaining_prelink_tasks -= 1;
1310 const base = comp.bin_file orelse return;
1311
1312 const prog_node = comp.work_queue_progress_node.start("Linker Parse Host libc", 0);1320 const prog_node = comp.work_queue_progress_node.start("Linker Parse Host libc", 0);
1313 defer prog_node.end();1321 defer prog_node.end();
13141322
...@@ -1368,8 +1376,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1368,8 +1376,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1368 }1376 }
1369 },1377 },
1370 .load_object => |path| {1378 .load_object => |path| {
1371 comp.remaining_prelink_tasks -= 1;
1372 const base = comp.bin_file orelse return;
1373 const prog_node = comp.work_queue_progress_node.start("Linker Parse Object", 0);1379 const prog_node = comp.work_queue_progress_node.start("Linker Parse Object", 0);
1374 defer prog_node.end();1380 defer prog_node.end();
1375 base.openLoadObject(path) catch |err| switch (err) {1381 base.openLoadObject(path) catch |err| switch (err) {
...@@ -1378,8 +1384,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1378,8 +1384,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1378 };1384 };
1379 },1385 },
1380 .load_archive => |path| {1386 .load_archive => |path| {
1381 comp.remaining_prelink_tasks -= 1;
1382 const base = comp.bin_file orelse return;
1383 const prog_node = comp.work_queue_progress_node.start("Linker Parse Archive", 0);1387 const prog_node = comp.work_queue_progress_node.start("Linker Parse Archive", 0);
1384 defer prog_node.end();1388 defer prog_node.end();
1385 base.openLoadArchive(path, null) catch |err| switch (err) {1389 base.openLoadArchive(path, null) catch |err| switch (err) {
...@@ -1388,8 +1392,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1388,8 +1392,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1388 };1392 };
1389 },1393 },
1390 .load_dso => |path| {1394 .load_dso => |path| {
1391 comp.remaining_prelink_tasks -= 1;
1392 const base = comp.bin_file orelse return;
1393 const prog_node = comp.work_queue_progress_node.start("Linker Parse Shared Library", 0);1395 const prog_node = comp.work_queue_progress_node.start("Linker Parse Shared Library", 0);
1394 defer prog_node.end();1396 defer prog_node.end();
1395 base.openLoadDso(path, .{1397 base.openLoadDso(path, .{
...@@ -1401,8 +1403,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1401,8 +1403,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1401 };1403 };
1402 },1404 },
1403 .load_input => |input| {1405 .load_input => |input| {
1404 comp.remaining_prelink_tasks -= 1;
1405 const base = comp.bin_file orelse return;
1406 const prog_node = comp.work_queue_progress_node.start("Linker Parse Input", 0);1406 const prog_node = comp.work_queue_progress_node.start("Linker Parse Input", 0);
1407 defer prog_node.end();1407 defer prog_node.end();
1408 base.loadInput(input) catch |err| switch (err) {1408 base.loadInput(input) catch |err| switch (err) {
...@@ -1416,11 +1416,12 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1416,11 +1416,12 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1416 },1416 },
1417 };1417 };
1418 },1418 },
1419 }
1420}
1421pub fn doZcuTask(comp: *Compilation, tid: usize, task: ZcuTask) void {
1422 const diags = &comp.link_diags;
1423 switch (task) {
1419 .link_nav => |nav_index| {1424 .link_nav => |nav_index| {
1420 if (comp.remaining_prelink_tasks != 0) {
1421 comp.link_task_queue_postponed.appendAssumeCapacity(task);
1422 return;
1423 }
1424 const zcu = comp.zcu.?;1425 const zcu = comp.zcu.?;
1425 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));1426 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
1426 defer pt.deactivate();1427 defer pt.deactivate();
...@@ -1431,39 +1432,43 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1431,39 +1432,43 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1431 } else if (comp.bin_file) |lf| {1432 } else if (comp.bin_file) |lf| {
1432 lf.updateNav(pt, nav_index) catch |err| switch (err) {1433 lf.updateNav(pt, nav_index) catch |err| switch (err) {
1433 error.OutOfMemory => diags.setAllocFailure(),1434 error.OutOfMemory => diags.setAllocFailure(),
1434 error.CodegenFail => assert(zcu.failed_codegen.contains(nav_index)),1435 error.CodegenFail => zcu.assertCodegenFailed(nav_index),
1435 error.Overflow, error.RelocationNotByteAligned => {1436 error.Overflow, error.RelocationNotByteAligned => {
1436 zcu.failed_codegen.ensureUnusedCapacity(zcu.gpa, 1) catch return diags.setAllocFailure();1437 switch (zcu.codegenFail(nav_index, "unable to codegen: {s}", .{@errorName(err)})) {
1437 const msg = Zcu.ErrorMsg.create(1438 error.CodegenFail => return,
1438 zcu.gpa,1439 error.OutOfMemory => return diags.setAllocFailure(),
1439 zcu.navSrcLoc(nav_index),1440 }
1440 "unable to codegen: {s}",
1441 .{@errorName(err)},
1442 ) catch return diags.setAllocFailure();
1443 zcu.failed_codegen.putAssumeCapacityNoClobber(nav_index, msg);
1444 // Not a retryable failure.1441 // Not a retryable failure.
1445 },1442 },
1446 };1443 };
1447 }1444 }
1448 },1445 },
1449 .link_func => |func| {1446 .link_func => |func| {
1450 if (comp.remaining_prelink_tasks != 0) {1447 const zcu = comp.zcu.?;
1451 comp.link_task_queue_postponed.appendAssumeCapacity(task);1448 const nav = zcu.funcInfo(func.func).owner_nav;
1452 return;1449 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
1453 }
1454 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
1455 defer pt.deactivate();1450 defer pt.deactivate();
1456 var air = func.air;1451 assert(zcu.llvm_object == null); // LLVM codegen doesn't produce MIR
1457 defer air.deinit(comp.gpa);1452 switch (func.mir.status.load(.monotonic)) {
1458 pt.linkerUpdateFunc(func.func, &air) catch |err| switch (err) {1453 .pending => unreachable,
1459 error.OutOfMemory => diags.setAllocFailure(),1454 .ready => {},
1460 };1455 .failed => return,
1456 }
1457 const mir = &func.mir.value;
1458 if (comp.bin_file) |lf| {
1459 lf.updateFunc(pt, func.func, mir, func.air) catch |err| switch (err) {
1460 error.OutOfMemory => return diags.setAllocFailure(),
1461 error.CodegenFail => return zcu.assertCodegenFailed(nav),
1462 error.Overflow, error.RelocationNotByteAligned => {
1463 switch (zcu.codegenFail(nav, "unable to codegen: {s}", .{@errorName(err)})) {
1464 error.OutOfMemory => return diags.setAllocFailure(),
1465 error.CodegenFail => return,
1466 }
1467 },
1468 };
1469 }
1461 },1470 },
1462 .link_type => |ty| {1471 .link_type => |ty| {
1463 if (comp.remaining_prelink_tasks != 0) {
1464 comp.link_task_queue_postponed.appendAssumeCapacity(task);
1465 return;
1466 }
1467 const zcu = comp.zcu.?;1472 const zcu = comp.zcu.?;
1468 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));1473 const pt: Zcu.PerThread = .activate(zcu, @enumFromInt(tid));
1469 defer pt.deactivate();1474 defer pt.deactivate();
...@@ -1477,10 +1482,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {...@@ -1477,10 +1482,6 @@ pub fn doTask(comp: *Compilation, tid: usize, task: Task) void {
1477 }1482 }
1478 },1483 },
1479 .update_line_number => |ti| {1484 .update_line_number => |ti| {
1480 if (comp.remaining_prelink_tasks != 0) {
1481 comp.link_task_queue_postponed.appendAssumeCapacity(task);
1482 return;
1483 }
1484 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));1485 const pt: Zcu.PerThread = .activate(comp.zcu.?, @enumFromInt(tid));
1485 defer pt.deactivate();1486 defer pt.deactivate();
1486 if (pt.zcu.llvm_object == null) {1487 if (pt.zcu.llvm_object == null) {
src/link/C.zig+57-86
...@@ -18,6 +18,7 @@ const trace = @import("../tracy.zig").trace;...@@ -18,6 +18,7 @@ const trace = @import("../tracy.zig").trace;
18const Type = @import("../Type.zig");18const Type = @import("../Type.zig");
19const Value = @import("../Value.zig");19const Value = @import("../Value.zig");
20const Air = @import("../Air.zig");20const Air = @import("../Air.zig");
21const AnyMir = @import("../codegen.zig").AnyMir;
2122
22pub const zig_h = "#include \"zig.h\"\n";23pub const zig_h = "#include \"zig.h\"\n";
2324
...@@ -166,6 +167,9 @@ pub fn deinit(self: *C) void {...@@ -166,6 +167,9 @@ pub fn deinit(self: *C) void {
166 self.uavs.deinit(gpa);167 self.uavs.deinit(gpa);
167 self.aligned_uavs.deinit(gpa);168 self.aligned_uavs.deinit(gpa);
168169
170 self.exported_navs.deinit(gpa);
171 self.exported_uavs.deinit(gpa);
172
169 self.string_bytes.deinit(gpa);173 self.string_bytes.deinit(gpa);
170 self.fwd_decl_buf.deinit(gpa);174 self.fwd_decl_buf.deinit(gpa);
171 self.code_buf.deinit(gpa);175 self.code_buf.deinit(gpa);
...@@ -177,73 +181,28 @@ pub fn updateFunc(...@@ -177,73 +181,28 @@ pub fn updateFunc(
177 self: *C,181 self: *C,
178 pt: Zcu.PerThread,182 pt: Zcu.PerThread,
179 func_index: InternPool.Index,183 func_index: InternPool.Index,
180 air: Air,184 mir: *AnyMir,
181 liveness: Air.Liveness,185 /// This may be `undefined`; only pass it to `emitFunction`.
186 /// This parameter will eventually be removed.
187 maybe_undef_air: *const Air,
182) link.File.UpdateNavError!void {188) link.File.UpdateNavError!void {
189 _ = maybe_undef_air; // It would be a bug to use this argument.
190
183 const zcu = pt.zcu;191 const zcu = pt.zcu;
184 const gpa = zcu.gpa;192 const gpa = zcu.gpa;
185 const func = zcu.funcInfo(func_index);193 const func = zcu.funcInfo(func_index);
186 const gop = try self.navs.getOrPut(gpa, func.owner_nav);
187 if (!gop.found_existing) gop.value_ptr.* = .{};
188 const ctype_pool = &gop.value_ptr.ctype_pool;
189 const lazy_fns = &gop.value_ptr.lazy_fns;
190 const fwd_decl = &self.fwd_decl_buf;
191 const code = &self.code_buf;
192 try ctype_pool.init(gpa);
193 ctype_pool.clearRetainingCapacity();
194 lazy_fns.clearRetainingCapacity();
195 fwd_decl.clearRetainingCapacity();
196 code.clearRetainingCapacity();
197194
198 var function: codegen.Function = .{195 const gop = try self.navs.getOrPut(gpa, func.owner_nav);
199 .value_map = codegen.CValueMap.init(gpa),196 if (gop.found_existing) gop.value_ptr.deinit(gpa);
200 .air = air,197 gop.value_ptr.* = .{
201 .liveness = liveness,198 .code = .empty,
202 .func_index = func_index,199 .fwd_decl = .empty,
203 .object = .{200 .ctype_pool = mir.c.ctype_pool.move(),
204 .dg = .{201 .lazy_fns = mir.c.lazy_fns.move(),
205 .gpa = gpa,
206 .pt = pt,
207 .mod = zcu.navFileScope(func.owner_nav).mod.?,
208 .error_msg = null,
209 .pass = .{ .nav = func.owner_nav },
210 .is_naked_fn = Type.fromInterned(func.ty).fnCallingConvention(zcu) == .naked,
211 .expected_block = null,
212 .fwd_decl = fwd_decl.toManaged(gpa),
213 .ctype_pool = ctype_pool.*,
214 .scratch = .{},
215 .uav_deps = self.uavs,
216 .aligned_uavs = self.aligned_uavs,
217 },
218 .code = code.toManaged(gpa),
219 .indent_writer = undefined, // set later so we can get a pointer to object.code
220 },
221 .lazy_fns = lazy_fns.*,
222 };
223 function.object.indent_writer = .{ .underlying_writer = function.object.code.writer() };
224 defer {
225 self.uavs = function.object.dg.uav_deps;
226 self.aligned_uavs = function.object.dg.aligned_uavs;
227 fwd_decl.* = function.object.dg.fwd_decl.moveToUnmanaged();
228 ctype_pool.* = function.object.dg.ctype_pool.move();
229 ctype_pool.freeUnusedCapacity(gpa);
230 function.object.dg.scratch.deinit(gpa);
231 lazy_fns.* = function.lazy_fns.move();
232 lazy_fns.shrinkAndFree(gpa, lazy_fns.count());
233 code.* = function.object.code.moveToUnmanaged();
234 function.deinit();
235 }
236
237 try zcu.failed_codegen.ensureUnusedCapacity(gpa, 1);
238 codegen.genFunc(&function) catch |err| switch (err) {
239 error.AnalysisFail => {
240 zcu.failed_codegen.putAssumeCapacityNoClobber(func.owner_nav, function.object.dg.error_msg.?);
241 return;
242 },
243 else => |e| return e,
244 };202 };
245 gop.value_ptr.fwd_decl = try self.addString(function.object.dg.fwd_decl.items);203 gop.value_ptr.code = try self.addString(mir.c.code);
246 gop.value_ptr.code = try self.addString(function.object.code.items);204 gop.value_ptr.fwd_decl = try self.addString(mir.c.fwd_decl);
205 try self.addUavsFromCodegen(&mir.c.uavs);
247}206}
248207
249fn updateUav(self: *C, pt: Zcu.PerThread, i: usize) !void {208fn updateUav(self: *C, pt: Zcu.PerThread, i: usize) !void {
...@@ -267,16 +226,14 @@ fn updateUav(self: *C, pt: Zcu.PerThread, i: usize) !void {...@@ -267,16 +226,14 @@ fn updateUav(self: *C, pt: Zcu.PerThread, i: usize) !void {
267 .fwd_decl = fwd_decl.toManaged(gpa),226 .fwd_decl = fwd_decl.toManaged(gpa),
268 .ctype_pool = codegen.CType.Pool.empty,227 .ctype_pool = codegen.CType.Pool.empty,
269 .scratch = .{},228 .scratch = .{},
270 .uav_deps = self.uavs,229 .uavs = .empty,
271 .aligned_uavs = self.aligned_uavs,
272 },230 },
273 .code = code.toManaged(gpa),231 .code = code.toManaged(gpa),
274 .indent_writer = undefined, // set later so we can get a pointer to object.code232 .indent_writer = undefined, // set later so we can get a pointer to object.code
275 };233 };
276 object.indent_writer = .{ .underlying_writer = object.code.writer() };234 object.indent_writer = .{ .underlying_writer = object.code.writer() };
277 defer {235 defer {
278 self.uavs = object.dg.uav_deps;236 object.dg.uavs.deinit(gpa);
279 self.aligned_uavs = object.dg.aligned_uavs;
280 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();237 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();
281 object.dg.ctype_pool.deinit(object.dg.gpa);238 object.dg.ctype_pool.deinit(object.dg.gpa);
282 object.dg.scratch.deinit(gpa);239 object.dg.scratch.deinit(gpa);
...@@ -295,8 +252,10 @@ fn updateUav(self: *C, pt: Zcu.PerThread, i: usize) !void {...@@ -295,8 +252,10 @@ fn updateUav(self: *C, pt: Zcu.PerThread, i: usize) !void {
295 else => |e| return e,252 else => |e| return e,
296 };253 };
297254
255 try self.addUavsFromCodegen(&object.dg.uavs);
256
298 object.dg.ctype_pool.freeUnusedCapacity(gpa);257 object.dg.ctype_pool.freeUnusedCapacity(gpa);
299 object.dg.uav_deps.values()[i] = .{258 self.uavs.values()[i] = .{
300 .code = try self.addString(object.code.items),259 .code = try self.addString(object.code.items),
301 .fwd_decl = try self.addString(object.dg.fwd_decl.items),260 .fwd_decl = try self.addString(object.dg.fwd_decl.items),
302 .ctype_pool = object.dg.ctype_pool.move(),261 .ctype_pool = object.dg.ctype_pool.move(),
...@@ -343,16 +302,14 @@ pub fn updateNav(self: *C, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) l...@@ -343,16 +302,14 @@ pub fn updateNav(self: *C, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) l
343 .fwd_decl = fwd_decl.toManaged(gpa),302 .fwd_decl = fwd_decl.toManaged(gpa),
344 .ctype_pool = ctype_pool.*,303 .ctype_pool = ctype_pool.*,
345 .scratch = .{},304 .scratch = .{},
346 .uav_deps = self.uavs,305 .uavs = .empty,
347 .aligned_uavs = self.aligned_uavs,
348 },306 },
349 .code = code.toManaged(gpa),307 .code = code.toManaged(gpa),
350 .indent_writer = undefined, // set later so we can get a pointer to object.code308 .indent_writer = undefined, // set later so we can get a pointer to object.code
351 };309 };
352 object.indent_writer = .{ .underlying_writer = object.code.writer() };310 object.indent_writer = .{ .underlying_writer = object.code.writer() };
353 defer {311 defer {
354 self.uavs = object.dg.uav_deps;312 object.dg.uavs.deinit(gpa);
355 self.aligned_uavs = object.dg.aligned_uavs;
356 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();313 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();
357 ctype_pool.* = object.dg.ctype_pool.move();314 ctype_pool.* = object.dg.ctype_pool.move();
358 ctype_pool.freeUnusedCapacity(gpa);315 ctype_pool.freeUnusedCapacity(gpa);
...@@ -360,16 +317,16 @@ pub fn updateNav(self: *C, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) l...@@ -360,16 +317,16 @@ pub fn updateNav(self: *C, pt: Zcu.PerThread, nav_index: InternPool.Nav.Index) l
360 code.* = object.code.moveToUnmanaged();317 code.* = object.code.moveToUnmanaged();
361 }318 }
362319
363 try zcu.failed_codegen.ensureUnusedCapacity(gpa, 1);
364 codegen.genDecl(&object) catch |err| switch (err) {320 codegen.genDecl(&object) catch |err| switch (err) {
365 error.AnalysisFail => {321 error.AnalysisFail => switch (zcu.codegenFailMsg(nav_index, object.dg.error_msg.?)) {
366 zcu.failed_codegen.putAssumeCapacityNoClobber(nav_index, object.dg.error_msg.?);322 error.CodegenFail => return,
367 return;323 error.OutOfMemory => |e| return e,
368 },324 },
369 else => |e| return e,325 else => |e| return e,
370 };326 };
371 gop.value_ptr.code = try self.addString(object.code.items);327 gop.value_ptr.code = try self.addString(object.code.items);
372 gop.value_ptr.fwd_decl = try self.addString(object.dg.fwd_decl.items);328 gop.value_ptr.fwd_decl = try self.addString(object.dg.fwd_decl.items);
329 try self.addUavsFromCodegen(&object.dg.uavs);
373}330}
374331
375pub fn updateLineNumber(self: *C, pt: Zcu.PerThread, ti_id: InternPool.TrackedInst.Index) !void {332pub fn updateLineNumber(self: *C, pt: Zcu.PerThread, ti_id: InternPool.TrackedInst.Index) !void {
...@@ -671,16 +628,14 @@ fn flushErrDecls(self: *C, pt: Zcu.PerThread, ctype_pool: *codegen.CType.Pool) F...@@ -671,16 +628,14 @@ fn flushErrDecls(self: *C, pt: Zcu.PerThread, ctype_pool: *codegen.CType.Pool) F
671 .fwd_decl = fwd_decl.toManaged(gpa),628 .fwd_decl = fwd_decl.toManaged(gpa),
672 .ctype_pool = ctype_pool.*,629 .ctype_pool = ctype_pool.*,
673 .scratch = .{},630 .scratch = .{},
674 .uav_deps = self.uavs,631 .uavs = .empty,
675 .aligned_uavs = self.aligned_uavs,
676 },632 },
677 .code = code.toManaged(gpa),633 .code = code.toManaged(gpa),
678 .indent_writer = undefined, // set later so we can get a pointer to object.code634 .indent_writer = undefined, // set later so we can get a pointer to object.code
679 };635 };
680 object.indent_writer = .{ .underlying_writer = object.code.writer() };636 object.indent_writer = .{ .underlying_writer = object.code.writer() };
681 defer {637 defer {
682 self.uavs = object.dg.uav_deps;638 object.dg.uavs.deinit(gpa);
683 self.aligned_uavs = object.dg.aligned_uavs;
684 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();639 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();
685 ctype_pool.* = object.dg.ctype_pool.move();640 ctype_pool.* = object.dg.ctype_pool.move();
686 ctype_pool.freeUnusedCapacity(gpa);641 ctype_pool.freeUnusedCapacity(gpa);
...@@ -692,6 +647,8 @@ fn flushErrDecls(self: *C, pt: Zcu.PerThread, ctype_pool: *codegen.CType.Pool) F...@@ -692,6 +647,8 @@ fn flushErrDecls(self: *C, pt: Zcu.PerThread, ctype_pool: *codegen.CType.Pool) F
692 error.AnalysisFail => unreachable,647 error.AnalysisFail => unreachable,
693 else => |e| return e,648 else => |e| return e,
694 };649 };
650
651 try self.addUavsFromCodegen(&object.dg.uavs);
695}652}
696653
697fn flushLazyFn(654fn flushLazyFn(
...@@ -719,8 +676,7 @@ fn flushLazyFn(...@@ -719,8 +676,7 @@ fn flushLazyFn(
719 .fwd_decl = fwd_decl.toManaged(gpa),676 .fwd_decl = fwd_decl.toManaged(gpa),
720 .ctype_pool = ctype_pool.*,677 .ctype_pool = ctype_pool.*,
721 .scratch = .{},678 .scratch = .{},
722 .uav_deps = .{},679 .uavs = .empty,
723 .aligned_uavs = .{},
724 },680 },
725 .code = code.toManaged(gpa),681 .code = code.toManaged(gpa),
726 .indent_writer = undefined, // set later so we can get a pointer to object.code682 .indent_writer = undefined, // set later so we can get a pointer to object.code
...@@ -729,8 +685,7 @@ fn flushLazyFn(...@@ -729,8 +685,7 @@ fn flushLazyFn(
729 defer {685 defer {
730 // If this assert trips just handle the anon_decl_deps the same as686 // If this assert trips just handle the anon_decl_deps the same as
731 // `updateFunc()` does.687 // `updateFunc()` does.
732 assert(object.dg.uav_deps.count() == 0);688 assert(object.dg.uavs.count() == 0);
733 assert(object.dg.aligned_uavs.count() == 0);
734 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();689 fwd_decl.* = object.dg.fwd_decl.moveToUnmanaged();
735 ctype_pool.* = object.dg.ctype_pool.move();690 ctype_pool.* = object.dg.ctype_pool.move();
736 ctype_pool.freeUnusedCapacity(gpa);691 ctype_pool.freeUnusedCapacity(gpa);
...@@ -866,12 +821,10 @@ pub fn updateExports(...@@ -866,12 +821,10 @@ pub fn updateExports(
866 .fwd_decl = fwd_decl.toManaged(gpa),821 .fwd_decl = fwd_decl.toManaged(gpa),
867 .ctype_pool = decl_block.ctype_pool,822 .ctype_pool = decl_block.ctype_pool,
868 .scratch = .{},823 .scratch = .{},
869 .uav_deps = .{},824 .uavs = .empty,
870 .aligned_uavs = .{},
871 };825 };
872 defer {826 defer {
873 assert(dg.uav_deps.count() == 0);827 assert(dg.uavs.count() == 0);
874 assert(dg.aligned_uavs.count() == 0);
875 fwd_decl.* = dg.fwd_decl.moveToUnmanaged();828 fwd_decl.* = dg.fwd_decl.moveToUnmanaged();
876 ctype_pool.* = dg.ctype_pool.move();829 ctype_pool.* = dg.ctype_pool.move();
877 ctype_pool.freeUnusedCapacity(gpa);830 ctype_pool.freeUnusedCapacity(gpa);
...@@ -891,3 +844,21 @@ pub fn deleteExport(...@@ -891,3 +844,21 @@ pub fn deleteExport(
891 .uav => |uav| _ = self.exported_uavs.swapRemove(uav),844 .uav => |uav| _ = self.exported_uavs.swapRemove(uav),
892 }845 }
893}846}
847
848fn addUavsFromCodegen(c: *C, uavs: *const std.AutoArrayHashMapUnmanaged(InternPool.Index, Alignment)) Allocator.Error!void {
849 const gpa = c.base.comp.gpa;
850 try c.uavs.ensureUnusedCapacity(gpa, uavs.count());
851 try c.aligned_uavs.ensureUnusedCapacity(gpa, uavs.count());
852 for (uavs.keys(), uavs.values()) |uav_val, uav_align| {
853 {
854 const gop = c.uavs.getOrPutAssumeCapacity(uav_val);
855 if (!gop.found_existing) gop.value_ptr.* = .{};
856 }
857 if (uav_align != .none) {
858 const gop = c.aligned_uavs.getOrPutAssumeCapacity(uav_val);
859 gop.value_ptr.* = if (gop.found_existing) max: {
860 break :max gop.value_ptr.*.maxStrict(uav_align);
861 } else uav_align;
862 }
863 }
864}
src/link/Coff.zig+2-15
...@@ -1079,7 +1079,7 @@ pub fn updateFunc(...@@ -1079,7 +1079,7 @@ pub fn updateFunc(
1079 var code_buffer: std.ArrayListUnmanaged(u8) = .empty;1079 var code_buffer: std.ArrayListUnmanaged(u8) = .empty;
1080 defer code_buffer.deinit(gpa);1080 defer code_buffer.deinit(gpa);
10811081
1082 codegen.generateFunction(1082 try codegen.generateFunction(
1083 &coff.base,1083 &coff.base,
1084 pt,1084 pt,
1085 zcu.navSrcLoc(nav_index),1085 zcu.navSrcLoc(nav_index),
...@@ -1088,20 +1088,7 @@ pub fn updateFunc(...@@ -1088,20 +1088,7 @@ pub fn updateFunc(
1088 liveness,1088 liveness,
1089 &code_buffer,1089 &code_buffer,
1090 .none,1090 .none,
1091 ) catch |err| switch (err) {1091 );
1092 error.CodegenFail => return error.CodegenFail,
1093 error.OutOfMemory => return error.OutOfMemory,
1094 error.Overflow, error.RelocationNotByteAligned => |e| {
1095 try zcu.failed_codegen.putNoClobber(gpa, nav_index, try Zcu.ErrorMsg.create(
1096 gpa,
1097 zcu.navSrcLoc(nav_index),
1098 "unable to codegen: {s}",
1099 .{@errorName(e)},
1100 ));
1101 try zcu.retryable_failures.append(zcu.gpa, AnalUnit.wrap(.{ .func = func_index }));
1102 return error.CodegenFail;
1103 },
1104 };
11051092
1106 try coff.updateNavCode(pt, nav_index, code_buffer.items, .FUNCTION);1093 try coff.updateNavCode(pt, nav_index, code_buffer.items, .FUNCTION);
11071094
src/link/Elf.zig+3-3
...@@ -1691,13 +1691,13 @@ pub fn updateFunc(...@@ -1691,13 +1691,13 @@ pub fn updateFunc(
1691 self: *Elf,1691 self: *Elf,
1692 pt: Zcu.PerThread,1692 pt: Zcu.PerThread,
1693 func_index: InternPool.Index,1693 func_index: InternPool.Index,
1694 air: Air,1694 mir: *const codegen.AnyMir,
1695 liveness: Air.Liveness,1695 maybe_undef_air: *const Air,
1696) link.File.UpdateNavError!void {1696) link.File.UpdateNavError!void {
1697 if (build_options.skip_non_native and builtin.object_format != .elf) {1697 if (build_options.skip_non_native and builtin.object_format != .elf) {
1698 @panic("Attempted to compile for object format that was disabled by build configuration");1698 @panic("Attempted to compile for object format that was disabled by build configuration");
1699 }1699 }
1700 return self.zigObjectPtr().?.updateFunc(self, pt, func_index, air, liveness);1700 return self.zigObjectPtr().?.updateFunc(self, pt, func_index, mir, maybe_undef_air);
1701}1701}
17021702
1703pub fn updateNav(1703pub fn updateNav(
src/link/Elf/ZigObject.zig+7-5
...@@ -1416,8 +1416,10 @@ pub fn updateFunc(...@@ -1416,8 +1416,10 @@ pub fn updateFunc(
1416 elf_file: *Elf,1416 elf_file: *Elf,
1417 pt: Zcu.PerThread,1417 pt: Zcu.PerThread,
1418 func_index: InternPool.Index,1418 func_index: InternPool.Index,
1419 air: Air,1419 mir: *const codegen.AnyMir,
1420 liveness: Air.Liveness,1420 /// This may be `undefined`; only pass it to `emitFunction`.
1421 /// This parameter will eventually be removed.
1422 maybe_undef_air: *const Air,
1421) link.File.UpdateNavError!void {1423) link.File.UpdateNavError!void {
1422 const tracy = trace(@src());1424 const tracy = trace(@src());
1423 defer tracy.end();1425 defer tracy.end();
...@@ -1438,15 +1440,15 @@ pub fn updateFunc(...@@ -1438,15 +1440,15 @@ pub fn updateFunc(
1438 var debug_wip_nav = if (self.dwarf) |*dwarf| try dwarf.initWipNav(pt, func.owner_nav, sym_index) else null;1440 var debug_wip_nav = if (self.dwarf) |*dwarf| try dwarf.initWipNav(pt, func.owner_nav, sym_index) else null;
1439 defer if (debug_wip_nav) |*wip_nav| wip_nav.deinit();1441 defer if (debug_wip_nav) |*wip_nav| wip_nav.deinit();
14401442
1441 try codegen.generateFunction(1443 try codegen.emitFunction(
1442 &elf_file.base,1444 &elf_file.base,
1443 pt,1445 pt,
1444 zcu.navSrcLoc(func.owner_nav),1446 zcu.navSrcLoc(func.owner_nav),
1445 func_index,1447 func_index,
1446 air,1448 mir,
1447 liveness,
1448 &code_buffer,1449 &code_buffer,
1449 if (debug_wip_nav) |*dn| .{ .dwarf = dn } else .none,1450 if (debug_wip_nav) |*dn| .{ .dwarf = dn } else .none,
1451 maybe_undef_air,
1450 );1452 );
1451 const code = code_buffer.items;1453 const code = code_buffer.items;
14521454
src/link/Queue.zig created+234
...@@ -0,0 +1,234 @@
1//! Stores and manages the queue of link tasks. Each task is either a `PrelinkTask` or a `ZcuTask`.
2//!
3//! There must be at most one link thread (the thread processing these tasks) active at a time. If
4//! `!comp.separateCodegenThreadOk()`, then ZCU tasks will be run on the main thread, bypassing this
5//! queue entirely.
6//!
7//! All prelink tasks must be processed before any ZCU tasks are processed. After all prelink tasks
8//! are run, but before any ZCU tasks are run, `prelink` must be called on the `link.File`.
9//!
10//! There will sometimes be a `ZcuTask` in the queue which is not yet ready because it depends on
11//! MIR which has not yet been generated by any codegen thread. In this case, we must pause
12//! processing of linker tasks until the MIR is ready. It would be incorrect to run any other link
13//! tasks first, since this would make builds unreproducible.
14
15mutex: std.Thread.Mutex,
16/// Validates that only one `flushTaskQueue` thread is running at a time.
17flush_safety: std.debug.SafetyLock,
18
19/// This is the number of prelink tasks which are expected but have not yet been enqueued.
20/// Guarded by `mutex`.
21pending_prelink_tasks: u32,
22
23/// Prelink tasks which have been enqueued and are not yet owned by the worker thread.
24/// Allocated into `gpa`, guarded by `mutex`.
25queued_prelink: std.ArrayListUnmanaged(PrelinkTask),
26/// The worker thread moves items from `queued_prelink` into this array in order to process them.
27/// Allocated into `gpa`, accessed only by the worker thread.
28wip_prelink: std.ArrayListUnmanaged(PrelinkTask),
29
30/// Like `queued_prelink`, but for ZCU tasks.
31/// Allocated into `gpa`, guarded by `mutex`.
32queued_zcu: std.ArrayListUnmanaged(ZcuTask),
33/// Like `wip_prelink`, but for ZCU tasks.
34/// Allocated into `gpa`, accessed only by the worker thread.
35wip_zcu: std.ArrayListUnmanaged(ZcuTask),
36
37/// When processing ZCU link tasks, we might have to block due to unpopulated MIR. When this
38/// happens, some tasks in `wip_zcu` have been run, and some are still pending. This is the
39/// index into `wip_zcu` which we have reached.
40wip_zcu_idx: usize,
41
42/// Guarded by `mutex`.
43state: union(enum) {
44 /// The link thread is currently running or queued to run.
45 running,
46 /// The link thread is not running or queued, because it has exhausted all immediately available
47 /// tasks. It should be spawned when more tasks are enqueued. If `pending_prelink_tasks` is not
48 /// zero, we are specifically waiting for prelink tasks.
49 finished,
50 /// The link thread is not running or queued, because it is waiting for this MIR to be populated.
51 /// Once codegen completes, it must call `mirReady` which will restart the link thread.
52 wait_for_mir: *ZcuTask.LinkFunc.SharedMir,
53},
54
55/// The initial `Queue` state, containing no tasks, expecting no prelink tasks, and with no running worker thread.
56/// The `pending_prelink_tasks` and `queued_prelink` fields may be modified as needed before calling `start`.
57pub const empty: Queue = .{
58 .mutex = .{},
59 .flush_safety = .{},
60 .pending_prelink_tasks = 0,
61 .queued_prelink = .empty,
62 .wip_prelink = .empty,
63 .queued_zcu = .empty,
64 .wip_zcu = .empty,
65 .wip_zcu_idx = 0,
66 .state = .finished,
67};
68/// `lf` is needed to correctly deinit any pending `ZcuTask`s.
69pub fn deinit(q: *Queue, comp: *Compilation) void {
70 const gpa = comp.gpa;
71 for (q.queued_zcu.items) |t| t.deinit(comp.zcu.?);
72 for (q.wip_zcu.items[q.wip_zcu_idx..]) |t| t.deinit(comp.zcu.?);
73 q.queued_prelink.deinit(gpa);
74 q.wip_prelink.deinit(gpa);
75 q.queued_zcu.deinit(gpa);
76 q.wip_zcu.deinit(gpa);
77}
78
79/// This is expected to be called exactly once, after which the caller must not directly access
80/// `queued_prelink` or `pending_prelink_tasks` any longer. This will spawn the link thread if
81/// necessary.
82pub fn start(q: *Queue, comp: *Compilation) void {
83 assert(q.state == .finished);
84 assert(q.queued_zcu.items.len == 0);
85 if (q.queued_prelink.items.len != 0) {
86 q.state = .running;
87 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, flushTaskQueue, .{ q, comp });
88 }
89}
90
91/// Called by codegen workers after they have populated a `ZcuTask.LinkFunc.SharedMir`. If the link
92/// thread was waiting for this MIR, it can resume.
93pub fn mirReady(q: *Queue, comp: *Compilation, mir: *ZcuTask.LinkFunc.SharedMir) void {
94 // We would like to assert that `mir` is not pending, but that would race with a worker thread
95 // potentially freeing it.
96 {
97 q.mutex.lock();
98 defer q.mutex.unlock();
99 switch (q.state) {
100 .finished => unreachable, // there's definitely a task queued
101 .running => return,
102 .wait_for_mir => |wait_for| if (wait_for != mir) return,
103 }
104 // We were waiting for `mir`, so we will restart the linker thread.
105 q.state = .running;
106 }
107 assert(mir.status.load(.monotonic) != .pending);
108 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, flushTaskQueue, .{ q, comp });
109}
110
111/// Enqueues all prelink tasks in `tasks`. Asserts that they were expected, i.e. that `tasks.len` is
112/// less than or equal to `q.pending_prelink_tasks`. Also asserts that `tasks.len` is not 0.
113pub fn enqueuePrelink(q: *Queue, comp: *Compilation, tasks: []const PrelinkTask) Allocator.Error!void {
114 {
115 q.mutex.lock();
116 defer q.mutex.unlock();
117 try q.queued_prelink.appendSlice(comp.gpa, tasks);
118 q.pending_prelink_tasks -= @intCast(tasks.len);
119 switch (q.state) {
120 .wait_for_mir => unreachable, // we've not started zcu tasks yet
121 .running => return,
122 .finished => {},
123 }
124 // Restart the linker thread, because it was waiting for a task
125 q.state = .running;
126 }
127 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, flushTaskQueue, .{ q, comp });
128}
129
130pub fn enqueueZcu(q: *Queue, comp: *Compilation, task: ZcuTask) Allocator.Error!void {
131 assert(comp.separateCodegenThreadOk());
132 {
133 q.mutex.lock();
134 defer q.mutex.unlock();
135 try q.queued_zcu.append(comp.gpa, task);
136 switch (q.state) {
137 .running, .wait_for_mir => return,
138 .finished => if (q.pending_prelink_tasks != 0) return,
139 }
140 // Restart the linker thread, unless it would immediately be blocked
141 if (task == .link_func and task.link_func.mir.status.load(.monotonic) == .pending) {
142 q.state = .{ .wait_for_mir = task.link_func.mir };
143 return;
144 }
145 q.state = .running;
146 }
147 comp.thread_pool.spawnWgId(&comp.link_task_wait_group, flushTaskQueue, .{ q, comp });
148}
149
150fn flushTaskQueue(tid: usize, q: *Queue, comp: *Compilation) void {
151 q.flush_safety.lock();
152 defer q.flush_safety.unlock();
153
154 if (std.debug.runtime_safety) {
155 q.mutex.lock();
156 defer q.mutex.unlock();
157 assert(q.state == .running);
158 }
159 prelink: while (true) {
160 assert(q.wip_prelink.items.len == 0);
161 {
162 q.mutex.lock();
163 defer q.mutex.unlock();
164 std.mem.swap(std.ArrayListUnmanaged(PrelinkTask), &q.queued_prelink, &q.wip_prelink);
165 if (q.wip_prelink.items.len == 0) {
166 if (q.pending_prelink_tasks == 0) {
167 break :prelink; // prelink is done
168 } else {
169 // We're expecting more prelink tasks so can't move on to ZCU tasks.
170 q.state = .finished;
171 return;
172 }
173 }
174 }
175 for (q.wip_prelink.items) |task| {
176 link.doPrelinkTask(comp, task);
177 }
178 q.wip_prelink.clearRetainingCapacity();
179 }
180
181 // We've finished the prelink tasks, so run prelink if necessary.
182 if (comp.bin_file) |lf| {
183 if (!lf.post_prelink) {
184 if (lf.prelink(comp.work_queue_progress_node)) |_| {
185 lf.post_prelink = true;
186 } else |err| switch (err) {
187 error.OutOfMemory => comp.link_diags.setAllocFailure(),
188 error.LinkFailure => {},
189 }
190 }
191 }
192
193 // Now we can run ZCU tasks.
194 while (true) {
195 if (q.wip_zcu.items.len == q.wip_zcu_idx) {
196 q.wip_zcu.clearRetainingCapacity();
197 q.wip_zcu_idx = 0;
198 q.mutex.lock();
199 defer q.mutex.unlock();
200 std.mem.swap(std.ArrayListUnmanaged(ZcuTask), &q.queued_zcu, &q.wip_zcu);
201 if (q.wip_zcu.items.len == 0) {
202 // We've exhausted all available tasks.
203 q.state = .finished;
204 return;
205 }
206 }
207 const task = q.wip_zcu.items[q.wip_zcu_idx];
208 // If the task is a `link_func`, we might have to stop until its MIR is populated.
209 pending: {
210 if (task != .link_func) break :pending;
211 const status_ptr = &task.link_func.mir.status;
212 // First check without the mutex to optimize for the common case where MIR is ready.
213 if (status_ptr.load(.monotonic) != .pending) break :pending;
214 q.mutex.lock();
215 defer q.mutex.unlock();
216 if (status_ptr.load(.monotonic) != .pending) break :pending;
217 // We will stop for now, and get restarted once this MIR is ready.
218 q.state = .{ .wait_for_mir = task.link_func.mir };
219 return;
220 }
221 link.doZcuTask(comp, tid, task);
222 task.deinit(comp.zcu.?);
223 q.wip_zcu_idx += 1;
224 }
225}
226
227const std = @import("std");
228const assert = std.debug.assert;
229const Allocator = std.mem.Allocator;
230const Compilation = @import("../Compilation.zig");
231const link = @import("../link.zig");
232const PrelinkTask = link.PrelinkTask;
233const ZcuTask = link.ZcuTask;
234const Queue = @This();
src/target.zig+3-1
...@@ -850,7 +850,9 @@ pub inline fn backendSupportsFeature(backend: std.builtin.CompilerBackend, compt...@@ -850,7 +850,9 @@ pub inline fn backendSupportsFeature(backend: std.builtin.CompilerBackend, compt
850 },850 },
851 .separate_thread => switch (backend) {851 .separate_thread => switch (backend) {
852 .stage2_llvm => false,852 .stage2_llvm => false,
853 else => true,853 // MLUGG TODO
854 .stage2_c => true,
855 else => false,
854 },856 },
855 };857 };
856}858}