authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-02-04 03:00:13+00:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-02-04 18:38:40+00:00
log269c1ae649017836f15313d1d4977402be11eed5
treec38b9e06df7c14f90db02cb6afa0ee42c7e7fbfa
parenta0004cebc255405764e889effb25a42fe07d8463
signaturelock-open Commit is signed but in an unrecognized format.

Zcu: incremental compilation improvements

* Mark root Decls for re-analysis separately * Check for re-analysis of root Decls * Remove `outdated` entry when analyzing fn body * Remove legacy `outdated` field from Decl analysis state

2 files changed, 160 insertions(+), 136 deletions(-)

src/Compilation.zig-1
...@@ -3548,7 +3548,6 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v...@@ -3548,7 +3548,6 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v
3548 switch (decl.analysis) {3548 switch (decl.analysis) {
3549 .unreferenced => unreachable,3549 .unreferenced => unreachable,
3550 .in_progress => unreachable,3550 .in_progress => unreachable,
3551 .outdated => unreachable,
35523551
3553 .file_failure,3552 .file_failure,
3554 .sema_failure,3553 .sema_failure,
src/Module.zig+160-135
...@@ -156,7 +156,14 @@ potentially_outdated: std.AutoArrayHashMapUnmanaged(InternPool.Depender, u32) =...@@ -156,7 +156,14 @@ potentially_outdated: std.AutoArrayHashMapUnmanaged(InternPool.Depender, u32) =
156outdated: std.AutoArrayHashMapUnmanaged(InternPool.Depender, u32) = .{},156outdated: std.AutoArrayHashMapUnmanaged(InternPool.Depender, u32) = .{},
157/// This contains all `Depender`s in `outdated` whose PO dependency count is 0.157/// This contains all `Depender`s in `outdated` whose PO dependency count is 0.
158/// Such `Depender`s are ready for immediate re-analysis.158/// Such `Depender`s are ready for immediate re-analysis.
159/// See `findOutdatedToAnalyze` for details.
159outdated_ready: std.AutoArrayHashMapUnmanaged(InternPool.Depender, void) = .{},160outdated_ready: std.AutoArrayHashMapUnmanaged(InternPool.Depender, void) = .{},
161/// This contains a set of Decls which may not be in `outdated`, but are the
162/// root Decls of files which have updated source and thus must be re-analyzed.
163/// If such a Decl is only in this set, the struct type index may be preserved
164/// (only the namespace might change). If such a Decl is also `outdated`, the
165/// struct type index must be recreated.
166outdated_file_root: std.AutoArrayHashMapUnmanaged(Decl.Index, void) = .{},
160167
161stage1_flags: packed struct {168stage1_flags: packed struct {
162 have_winmain: bool = false,169 have_winmain: bool = false,
...@@ -431,13 +438,9 @@ pub const Decl = struct {...@@ -431,13 +438,9 @@ pub const Decl = struct {
431 /// This indicates the failure was something like running out of disk space,438 /// This indicates the failure was something like running out of disk space,
432 /// and attempting codegen again may succeed.439 /// and attempting codegen again may succeed.
433 codegen_failure_retryable,440 codegen_failure_retryable,
434 /// Everything is done. During an update, this Decl may be out of date, depending441 /// Sematic analysis of this Decl has succeeded. However, the Decl may
435 /// on its dependencies. The `generation` field can be used to determine if this442 /// be outdated due to an incomplete update!
436 /// completion status occurred before or after a given update.
437 complete,443 complete,
438 /// A Module update is in progress, and this Decl has been flagged as being known
439 /// to require re-analysis.
440 outdated,
441 },444 },
442 /// Whether `typed_value`, `align`, `linksection` and `addrspace` are populated.445 /// Whether `typed_value`, `align`, `linksection` and `addrspace` are populated.
443 has_tv: bool,446 has_tv: bool,
...@@ -735,8 +738,7 @@ pub const Namespace = struct {...@@ -735,8 +738,7 @@ pub const Namespace = struct {
735 file_scope: *File,738 file_scope: *File,
736 /// Will be a struct, enum, union, or opaque.739 /// Will be a struct, enum, union, or opaque.
737 ty: Type,740 ty: Type,
738 /// Direct children of the namespace. Used during an update to detect741 /// Direct children of the namespace.
739 /// which decls have been added/removed from source.
740 /// Declaration order is preserved via entry order.742 /// Declaration order is preserved via entry order.
741 /// These are only declarations named directly by the AST; anonymous743 /// These are only declarations named directly by the AST; anonymous
742 /// declarations are not stored here.744 /// declarations are not stored here.
...@@ -2492,6 +2494,7 @@ pub fn deinit(zcu: *Zcu) void {...@@ -2492,6 +2494,7 @@ pub fn deinit(zcu: *Zcu) void {
2492 zcu.potentially_outdated.deinit(gpa);2494 zcu.potentially_outdated.deinit(gpa);
2493 zcu.outdated.deinit(gpa);2495 zcu.outdated.deinit(gpa);
2494 zcu.outdated_ready.deinit(gpa);2496 zcu.outdated_ready.deinit(gpa);
2497 zcu.outdated_file_root.deinit(gpa);
24952498
2496 zcu.test_functions.deinit(gpa);2499 zcu.test_functions.deinit(gpa);
24972500
...@@ -2858,27 +2861,13 @@ pub fn astGenFile(mod: *Module, file: *File) !void {...@@ -2858,27 +2861,13 @@ pub fn astGenFile(mod: *Module, file: *File) !void {
2858 file.prev_zir = null;2861 file.prev_zir = null;
2859 }2862 }
28602863
2861 if (file.root_decl.unwrap()) |root_decl| mark_outdated: {2864 if (file.root_decl.unwrap()) |root_decl| {
2862 // The root of this file must be re-analyzed, since the file has changed.2865 // The root of this file must be re-analyzed, since the file has changed.
2863 comp.mutex.lock();2866 comp.mutex.lock();
2864 defer comp.mutex.unlock();2867 defer comp.mutex.unlock();
28652868
2866 const root_decl_depender = InternPool.Depender.wrap(.{ .decl = root_decl });2869 log.debug("outdated root Decl: {}", .{root_decl});
28672870 try mod.outdated_file_root.put(gpa, root_decl, {});
2868 const gop = try mod.outdated.getOrPut(gpa, root_decl_depender);
2869 // If this Decl is already marked as outdated, nothing needs to be done.
2870 if (gop.found_existing) break :mark_outdated;
2871
2872 log.debug("outdated: {} (root Decl)", .{root_decl});
2873
2874 // If it's already PO, forward its existing PO dependency count.
2875 // Otherwise, it has no PO dependencies yet.
2876 if (mod.potentially_outdated.fetchSwapRemove(root_decl_depender)) |kv| {
2877 gop.value_ptr.* = kv.value;
2878 } else {
2879 gop.value_ptr.* = 0;
2880 try mod.outdated_ready.put(mod.gpa, root_decl_depender, {});
2881 }
2882 }2871 }
2883}2872}
28842873
...@@ -3187,6 +3176,26 @@ pub fn findOutdatedToAnalyze(zcu: *Zcu) Allocator.Error!?InternPool.Depender {...@@ -3187,6 +3176,26 @@ pub fn findOutdatedToAnalyze(zcu: *Zcu) Allocator.Error!?InternPool.Depender {
3187 return zcu.outdated_ready.keys()[0];3176 return zcu.outdated_ready.keys()[0];
3188 }3177 }
31893178
3179 // Next, we will see if there is any outdated file root which was not in
3180 // `outdated`. This set will be small (number of files changed in this
3181 // update), so it's alright for us to just iterate here.
3182 for (zcu.outdated_file_root.keys()) |file_decl| {
3183 const decl_depender = InternPool.Depender.wrap(.{ .decl = file_decl });
3184 if (zcu.outdated.contains(decl_depender)) {
3185 // Since we didn't hit this in the first loop, this Decl must have
3186 // pending dependencies, so is ineligible.
3187 continue;
3188 }
3189 if (zcu.potentially_outdated.contains(decl_depender)) {
3190 // This Decl's struct may or may not need to be recreated depending
3191 // on whether it is outdated. If we analyzed it now, we would have
3192 // to assume it was outdated and recreate it!
3193 continue;
3194 }
3195 log.debug("findOutdatedToAnalyze: outdated file root decl '{d}'", .{file_decl});
3196 return decl_depender;
3197 }
3198
3190 // There is no single Depender which is ready for re-analysis. Instead, we3199 // There is no single Depender which is ready for re-analysis. Instead, we
3191 // must assume that some Decl with PO dependencies is outdated - e.g. in the3200 // must assume that some Decl with PO dependencies is outdated - e.g. in the
3192 // above example we arbitrarily pick one of A or B. We should select a Decl,3201 // above example we arbitrarily pick one of A or B. We should select a Decl,
...@@ -3407,26 +3416,27 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl_index: Decl.Index) SemaError!void {...@@ -3407,26 +3416,27 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl_index: Decl.Index) SemaError!void {
3407 .in_progress => unreachable,3416 .in_progress => unreachable,
34083417
3409 .file_failure,3418 .file_failure,
3410 .sema_failure,
3411 .sema_failure_retryable,
3412 .liveness_failure,3419 .liveness_failure,
3413 .codegen_failure,3420 .codegen_failure,
3414 .dependency_failure,
3415 .codegen_failure_retryable,3421 .codegen_failure_retryable,
3422 .dependency_failure,
3416 => return error.AnalysisFail,3423 => return error.AnalysisFail,
34173424
3418 .complete => if (was_outdated) {3425 .sema_failure,
3419 if (build_options.only_c) unreachable;3426 .sema_failure_retryable,
3420 // The exports this Decl performs will be re-discovered, so we remove them here3427 => if (!was_outdated) return error.AnalysisFail,
3421 // prior to re-analysis.
3422 try mod.deleteDeclExports(decl_index);
3423 } else return,
34243428
3425 .outdated => unreachable, // TODO: remove this field3429 .complete => if (!was_outdated) return,
34263430
3427 .unreferenced => {},3431 .unreferenced => {},
3428 }3432 }
34293433
3434 if (was_outdated) {
3435 // The exports this Decl performs will be re-discovered, so we remove them here
3436 // prior to re-analysis.
3437 try mod.deleteDeclExports(decl_index);
3438 }
3439
3430 var decl_prog_node = mod.sema_prog_node.start("", 0);3440 var decl_prog_node = mod.sema_prog_node.start("", 0);
3431 decl_prog_node.activate();3441 decl_prog_node.activate();
3432 defer decl_prog_node.end();3442 defer decl_prog_node.end();
...@@ -3493,7 +3503,6 @@ pub fn ensureFuncBodyAnalyzed(zcu: *Zcu, func_index: InternPool.Index) SemaError...@@ -3493,7 +3503,6 @@ pub fn ensureFuncBodyAnalyzed(zcu: *Zcu, func_index: InternPool.Index) SemaError
3493 switch (decl.analysis) {3503 switch (decl.analysis) {
3494 .unreferenced => unreachable,3504 .unreferenced => unreachable,
3495 .in_progress => unreachable,3505 .in_progress => unreachable,
3496 .outdated => unreachable,
34973506
3498 .file_failure,3507 .file_failure,
3499 .sema_failure,3508 .sema_failure,
...@@ -3503,109 +3512,117 @@ pub fn ensureFuncBodyAnalyzed(zcu: *Zcu, func_index: InternPool.Index) SemaError...@@ -3503,109 +3512,117 @@ pub fn ensureFuncBodyAnalyzed(zcu: *Zcu, func_index: InternPool.Index) SemaError
3503 .sema_failure_retryable,3512 .sema_failure_retryable,
3504 => return error.AnalysisFail,3513 => return error.AnalysisFail,
35053514
3506 .complete, .codegen_failure_retryable => {3515 .complete, .codegen_failure_retryable => {},
3507 switch (func.analysis(ip).state) {3516 }
3508 .sema_failure, .dependency_failure => return error.AnalysisFail,
3509 .none, .queued => {},
3510 .in_progress => unreachable,
3511 .inline_only => unreachable, // don't queue work for this
3512 .success => return,
3513 }
35143517
3515 const gpa = zcu.gpa;3518 const func_as_depender = InternPool.Depender.wrap(.{ .func = func_index });
3519 const was_outdated = zcu.outdated.swapRemove(func_as_depender) or
3520 zcu.potentially_outdated.swapRemove(func_as_depender);
35163521
3517 var tmp_arena = std.heap.ArenaAllocator.init(gpa);3522 if (was_outdated) {
3518 defer tmp_arena.deinit();3523 _ = zcu.outdated_ready.swapRemove(func_as_depender);
3519 const sema_arena = tmp_arena.allocator();3524 }
35203525
3521 var air = zcu.analyzeFnBody(func_index, sema_arena) catch |err| switch (err) {3526 switch (func.analysis(ip).state) {
3522 error.AnalysisFail => {3527 .sema_failure, .dependency_failure => if (!was_outdated) return error.AnalysisFail,
3523 if (func.analysis(ip).state == .in_progress) {3528 .none, .queued => {},
3524 // If this decl caused the compile error, the analysis field would3529 .in_progress => unreachable,
3525 // be changed to indicate it was this Decl's fault. Because this3530 .inline_only => unreachable, // don't queue work for this
3526 // did not happen, we infer here that it was a dependency failure.3531 .success => if (!was_outdated) return,
3527 func.analysis(ip).state = .dependency_failure;3532 }
3528 }
3529 return error.AnalysisFail;
3530 },
3531 error.OutOfMemory => return error.OutOfMemory,
3532 };
3533 defer air.deinit(gpa);
35343533
3535 const comp = zcu.comp;3534 const gpa = zcu.gpa;
35363535
3537 const dump_air = builtin.mode == .Debug and comp.verbose_air;3536 var tmp_arena = std.heap.ArenaAllocator.init(gpa);
3538 const dump_llvm_ir = builtin.mode == .Debug and (comp.verbose_llvm_ir != null or comp.verbose_llvm_bc != null);3537 defer tmp_arena.deinit();
3538 const sema_arena = tmp_arena.allocator();
35393539
3540 if (comp.bin_file == null and zcu.llvm_object == null and !dump_air and !dump_llvm_ir) {3540 var air = zcu.analyzeFnBody(func_index, sema_arena) catch |err| switch (err) {
3541 return;3541 error.AnalysisFail => {
3542 if (func.analysis(ip).state == .in_progress) {
3543 // If this decl caused the compile error, the analysis field would
3544 // be changed to indicate it was this Decl's fault. Because this
3545 // did not happen, we infer here that it was a dependency failure.
3546 func.analysis(ip).state = .dependency_failure;
3542 }3547 }
3548 return error.AnalysisFail;
3549 },
3550 error.OutOfMemory => return error.OutOfMemory,
3551 };
3552 defer air.deinit(gpa);
35433553
3544 var liveness = try Liveness.analyze(gpa, air, ip);3554 const comp = zcu.comp;
3545 defer liveness.deinit(gpa);
35463555
3547 if (dump_air) {3556 const dump_air = builtin.mode == .Debug and comp.verbose_air;
3548 const fqn = try decl.getFullyQualifiedName(zcu);3557 const dump_llvm_ir = builtin.mode == .Debug and (comp.verbose_llvm_ir != null or comp.verbose_llvm_bc != null);
3549 std.debug.print("# Begin Function AIR: {}:\n", .{fqn.fmt(ip)});
3550 @import("print_air.zig").dump(zcu, air, liveness);
3551 std.debug.print("# End Function AIR: {}\n\n", .{fqn.fmt(ip)});
3552 }
35533558
3554 if (std.debug.runtime_safety) {3559 if (comp.bin_file == null and zcu.llvm_object == null and !dump_air and !dump_llvm_ir) {
3555 var verify = Liveness.Verify{3560 return;
3556 .gpa = gpa,3561 }
3557 .air = air,
3558 .liveness = liveness,
3559 .intern_pool = ip,
3560 };
3561 defer verify.deinit();
3562
3563 verify.verify() catch |err| switch (err) {
3564 error.OutOfMemory => return error.OutOfMemory,
3565 else => {
3566 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3567 zcu.failed_decls.putAssumeCapacityNoClobber(
3568 decl_index,
3569 try Module.ErrorMsg.create(
3570 gpa,
3571 decl.srcLoc(zcu),
3572 "invalid liveness: {s}",
3573 .{@errorName(err)},
3574 ),
3575 );
3576 decl.analysis = .liveness_failure;
3577 return error.AnalysisFail;
3578 },
3579 };
3580 }
35813562
3582 if (comp.bin_file) |lf| {3563 var liveness = try Liveness.analyze(gpa, air, ip);
3583 lf.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {3564 defer liveness.deinit(gpa);
3584 error.OutOfMemory => return error.OutOfMemory,3565
3585 error.AnalysisFail => {3566 if (dump_air) {
3586 decl.analysis = .codegen_failure;3567 const fqn = try decl.getFullyQualifiedName(zcu);
3587 },3568 std.debug.print("# Begin Function AIR: {}:\n", .{fqn.fmt(ip)});
3588 else => {3569 @import("print_air.zig").dump(zcu, air, liveness);
3589 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);3570 std.debug.print("# End Function AIR: {}\n\n", .{fqn.fmt(ip)});
3590 zcu.failed_decls.putAssumeCapacityNoClobber(decl_index, try Module.ErrorMsg.create(3571 }
3591 gpa,3572
3592 decl.srcLoc(zcu),3573 if (std.debug.runtime_safety) {
3593 "unable to codegen: {s}",3574 var verify = Liveness.Verify{
3594 .{@errorName(err)},3575 .gpa = gpa,
3595 ));3576 .air = air,
3596 decl.analysis = .codegen_failure_retryable;3577 .liveness = liveness,
3597 },3578 .intern_pool = ip,
3598 };3579 };
3599 } else if (zcu.llvm_object) |llvm_object| {3580 defer verify.deinit();
3600 if (build_options.only_c) unreachable;3581
3601 llvm_object.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {3582 verify.verify() catch |err| switch (err) {
3602 error.OutOfMemory => return error.OutOfMemory,3583 error.OutOfMemory => return error.OutOfMemory,
3603 error.AnalysisFail => {3584 else => {
3604 decl.analysis = .codegen_failure;3585 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3605 },3586 zcu.failed_decls.putAssumeCapacityNoClobber(
3606 };3587 decl_index,
3607 }3588 try Module.ErrorMsg.create(
3608 },3589 gpa,
3590 decl.srcLoc(zcu),
3591 "invalid liveness: {s}",
3592 .{@errorName(err)},
3593 ),
3594 );
3595 decl.analysis = .liveness_failure;
3596 return error.AnalysisFail;
3597 },
3598 };
3599 }
3600
3601 if (comp.bin_file) |lf| {
3602 lf.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {
3603 error.OutOfMemory => return error.OutOfMemory,
3604 error.AnalysisFail => {
3605 decl.analysis = .codegen_failure;
3606 },
3607 else => {
3608 try zcu.failed_decls.ensureUnusedCapacity(gpa, 1);
3609 zcu.failed_decls.putAssumeCapacityNoClobber(decl_index, try Module.ErrorMsg.create(
3610 gpa,
3611 decl.srcLoc(zcu),
3612 "unable to codegen: {s}",
3613 .{@errorName(err)},
3614 ));
3615 decl.analysis = .codegen_failure_retryable;
3616 },
3617 };
3618 } else if (zcu.llvm_object) |llvm_object| {
3619 if (build_options.only_c) unreachable;
3620 llvm_object.updateFunc(zcu, func_index, air, liveness) catch |err| switch (err) {
3621 error.OutOfMemory => return error.OutOfMemory,
3622 error.AnalysisFail => {
3623 decl.analysis = .codegen_failure;
3624 },
3625 };
3609 }3626 }
3610}3627}
36113628
...@@ -3625,7 +3642,6 @@ pub fn ensureFuncBodyAnalysisQueued(mod: *Module, func_index: InternPool.Index)...@@ -3625,7 +3642,6 @@ pub fn ensureFuncBodyAnalysisQueued(mod: *Module, func_index: InternPool.Index)
3625 switch (decl.analysis) {3642 switch (decl.analysis) {
3626 .unreferenced => unreachable,3643 .unreferenced => unreachable,
3627 .in_progress => unreachable,3644 .in_progress => unreachable,
3628 .outdated => unreachable,
36293645
3630 .file_failure,3646 .file_failure,
3631 .sema_failure,3647 .sema_failure,
...@@ -3813,6 +3829,15 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !SemaDeclResult {...@@ -3813,6 +3829,15 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !SemaDeclResult {
3813 return error.AnalysisFail;3829 return error.AnalysisFail;
3814 }3830 }
38153831
3832 if (mod.declIsRoot(decl_index)) {
3833 // This comes from an `analyze_decl` job on an incremental update where
3834 // this file changed.
3835 @panic("TODO: update root Decl of modified file");
3836 } else if (decl.owns_tv) {
3837 // We are re-analyzing an owner Decl (for a function or a namespace type).
3838 @panic("TODO: update owner Decl");
3839 }
3840
3816 const gpa = mod.gpa;3841 const gpa = mod.gpa;
3817 const zir = decl.getFileScope(mod).zir;3842 const zir = decl.getFileScope(mod).zir;
38183843
...@@ -3880,12 +3905,12 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !SemaDeclResult {...@@ -3880,12 +3905,12 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !SemaDeclResult {
3880 };3905 };
3881 defer sema.deinit();3906 defer sema.deinit();
38823907
3883 // Every Decl other (than file root Decls, which do not have a ZIR index) has a dependency on its own source.3908 // Every Decl (other than file root Decls, which do not have a ZIR index) has a dependency on its own source.
3884 if (decl.zir_decl_index.unwrap()) |zir_decl_index| {3909 try sema.declareDependency(.{ .src_hash = try ip.trackZir(
3885 try sema.declareDependency(.{ .src_hash = try ip.trackZir(sema.gpa, decl.getFileScope(mod), zir_decl_index) });3910 sema.gpa,
3886 }3911 decl.getFileScope(mod),
38873912 decl.zir_decl_index.unwrap().?,
3888 assert(!mod.declIsRoot(decl_index));3913 ) });
38893914
3890 var block_scope: Sema.Block = .{3915 var block_scope: Sema.Block = .{
3891 .parent = null,3916 .parent = null,