authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-11 17:34:13-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-11 17:34:13-07:00
logcbbc7cc8b1edeeae1715248a5d13304027698367
tree08305439f53718e48bd65da4ae8751949dab218b
parent5d9fc11d18bef9142cb32af3d1515b6de9f125ab

stage2: better handling of file-level compile errors across updates

* `Module.File` now retains the most recent successful ZIR in the event that an update causes a ZIR compile error. This way Zig does not throw out useful semantic analysis results when an update temporarily introduces a ZIR compile error. * Semantic analysis of a File now unconditionally creates a Decl object for the File. The Decl object is marked as `file_failed` in case of File-level compile errors. This allows detecting of the File being outdated, and dependency tracking just like any other Decl.

4 files changed, 106 insertions(+), 65 deletions(-)

BRANCH_TODO-3
......@@ -12,9 +12,6 @@
1212 their indexes starting at 0 so that we can use an array to store Sema
1313 results rather than a map.
1414
15 * keep track of file dependencies/dependants
16 * unload files from memory when a dependency is dropped
17
1815 * implement the new AstGen compile errors
1916
2017 * get rid of failed_root_src_file
src/Compilation.zig+2
......@@ -1902,6 +1902,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
19021902 .in_progress => unreachable,
19031903 .outdated => unreachable,
19041904
1905 .file_failure,
19051906 .sema_failure,
19061907 .codegen_failure,
19071908 .dependency_failure,
......@@ -1970,6 +1971,7 @@ pub fn performAllTheWork(self: *Compilation) error{ TimerUnsupported, OutOfMemor
19701971 .in_progress => unreachable,
19711972 .outdated => unreachable,
19721973
1974 .file_failure,
19731975 .sema_failure,
19741976 .dependency_failure,
19751977 .sema_failure_retryable,
src/Module.zig+102-60
......@@ -199,8 +199,12 @@ pub const Decl = struct {
199199 /// This Decl corresponds to an AST Node that has not been referenced yet, and therefore
200200 /// because of Zig's lazy declaration analysis, it will remain unanalyzed until referenced.
201201 unreferenced,
202 /// Semantic analysis for this Decl is running right now. This state detects dependency loops.
202 /// Semantic analysis for this Decl is running right now.
203 /// This state detects dependency loops.
203204 in_progress,
205 /// The file corresponding to this Decl had a parse error or ZIR error.
206 /// There will be a corresponding ErrorMsg in Module.failed_files.
207 file_failure,
204208 /// This Decl might be OK but it depends on another one which did not successfully complete
205209 /// semantic analysis.
206210 dependency_failure,
......@@ -548,6 +552,7 @@ pub const Decl = struct {
548552 .unreferenced,
549553 .in_progress,
550554 .dependency_failure,
555 .file_failure,
551556 .sema_failure,
552557 .sema_failure_retryable,
553558 .codegen_failure,
......@@ -836,7 +841,7 @@ pub const Scope = struct {
836841 pub fn namespace(scope: *Scope) *Namespace {
837842 switch (scope.tag) {
838843 .block => return scope.cast(Block).?.sema.owner_decl.namespace,
839 .file => return scope.cast(File).?.namespace,
844 .file => return scope.cast(File).?.namespace.?,
840845 .namespace => return scope.cast(Namespace).?,
841846 .decl_ref => return scope.cast(DeclRef).?.decl.namespace,
842847 }
......@@ -965,7 +970,6 @@ pub const Scope = struct {
965970 parse_failure,
966971 astgen_failure,
967972 success_zir,
968 success_air,
969973 },
970974 source_loaded: bool,
971975 tree_loaded: bool,
......@@ -988,9 +992,9 @@ pub const Scope = struct {
988992 /// Package that this file is a part of, managed externally.
989993 pkg: *Package,
990994 /// The namespace of the struct that represents this file.
991 /// Populated only when status is `success_air`.
995 /// Populated only when `have_decl` is true.
992996 /// Owned by its owner Decl Value.
993 namespace: *Namespace,
997 namespace: ?*Namespace,
994998
995999 /// Used by change detection algorithm, after astgen, contains the
9961000 /// set of decls that existed in the previous ZIR but not in the new one.
......@@ -1000,6 +1004,12 @@ pub const Scope = struct {
10001004 /// but their source code has been modified.
10011005 outdated_decls: std.ArrayListUnmanaged(*Decl) = .{},
10021006
1007 /// The most recent successful ZIR for this file, with no errors.
1008 /// This is only populated when a previously successful ZIR
1009 /// newly introduces compile errors during an update. When ZIR is
1010 /// successful, this field is unloaded.
1011 prev_zir: ?*Zir = null,
1012
10031013 pub fn unload(file: *File, gpa: *Allocator) void {
10041014 file.unloadTree(gpa);
10051015 file.unloadSource(gpa);
......@@ -1032,11 +1042,15 @@ pub const Scope = struct {
10321042 log.debug("deinit File {s}", .{file.sub_file_path});
10331043 file.deleted_decls.deinit(gpa);
10341044 file.outdated_decls.deinit(gpa);
1035 if (file.status == .success_air) {
1036 file.namespace.getDecl().destroy(mod);
1045 if (file.namespace) |ns| {
1046 ns.getDecl().destroy(mod);
10371047 }
10381048 gpa.free(file.sub_file_path);
10391049 file.unload(gpa);
1050 if (file.prev_zir) |prev_zir| {
1051 prev_zir.deinit(gpa);
1052 gpa.destroy(prev_zir);
1053 }
10401054 file.* = undefined;
10411055 }
10421056
......@@ -2370,7 +2384,7 @@ pub fn astGenFile(mod: *Module, file: *Scope.File, prog_node: *std.Progress.Node
23702384 }
23712385 return;
23722386 },
2373 .parse_failure, .astgen_failure, .success_zir, .success_air => {
2387 .parse_failure, .astgen_failure, .success_zir => {
23742388 const unchanged_metadata =
23752389 stat.size == file.stat_size and
23762390 stat.mtime == file.stat_mtime and
......@@ -2411,7 +2425,7 @@ pub fn astGenFile(mod: *Module, file: *Scope.File, prog_node: *std.Progress.Node
24112425
24122426 // Move previous ZIR to a local variable so we can compare it with the new one.
24132427 var prev_zir = file.zir;
2414 const prev_zir_loaded = file.zir_loaded;
2428 var prev_zir_loaded = file.zir_loaded;
24152429 file.zir_loaded = false;
24162430 file.zir = undefined;
24172431 defer if (prev_zir_loaded) prev_zir.deinit(gpa);
......@@ -2536,8 +2550,34 @@ pub fn astGenFile(mod: *Module, file: *Scope.File, prog_node: *std.Progress.Node
25362550 // We do not need to hold any locks at this time because all the Decl and Namespace
25372551 // objects being touched are specific to this File, and the only other concurrent
25382552 // tasks are touching other File objects.
2539 try updateZirRefs(gpa, file, prev_zir);
2540
2553 if (file.zir.hasCompileErrors()) {
2554 // In this case, we keep the previous ZIR loaded so that we can use it
2555 // for the update next time it does not have any compile errors. This avoids
2556 // needlessly tossing out semantic analysis work when a ZIR error is
2557 // temporarily introduced.
2558 if (!prev_zir.hasCompileErrors()) {
2559 assert(file.prev_zir == null);
2560 const prev_zir_ptr = try gpa.create(Zir);
2561 file.prev_zir = prev_zir_ptr;
2562 prev_zir_ptr.* = prev_zir;
2563 prev_zir_loaded = false;
2564 }
2565 } else if (prev_zir.hasCompileErrors()) {
2566 if (file.prev_zir) |file_prev_zir| {
2567 prev_zir.deinit(gpa);
2568 prev_zir = file_prev_zir.*;
2569 gpa.destroy(file_prev_zir);
2570 file.prev_zir = null;
2571 try updateZirRefs(gpa, file, prev_zir);
2572 } else if (file.namespace) |ns| {
2573 // First time the File has succeeded ZIR. We must mark it outdated since
2574 // we have already tried to semantically analyze it.
2575 try file.outdated_decls.resize(gpa, 1);
2576 file.outdated_decls.items[0] = ns.getDecl();
2577 }
2578 } else {
2579 try updateZirRefs(gpa, file, prev_zir);
2580 }
25412581 // At this point, `file.outdated_decls` and `file.deleted_decls` are populated,
25422582 // and semantic analysis will deal with them properly.
25432583 }
......@@ -2579,7 +2619,7 @@ fn updateZirRefs(gpa: *Allocator, file: *Scope.File, old_zir: Zir) !void {
25792619 var decl_stack: std.ArrayListUnmanaged(*Decl) = .{};
25802620 defer decl_stack.deinit(gpa);
25812621
2582 const root_decl = file.namespace.getDecl();
2622 const root_decl = file.namespace.?.getDecl();
25832623 try decl_stack.append(gpa, root_decl);
25842624
25852625 file.deleted_decls.clearRetainingCapacity();
......@@ -2712,6 +2752,7 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
27122752 const subsequent_analysis = switch (decl.analysis) {
27132753 .in_progress => unreachable,
27142754
2755 .file_failure,
27152756 .sema_failure,
27162757 .sema_failure_retryable,
27172758 .codegen_failure,
......@@ -2773,6 +2814,7 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
27732814 .in_progress => unreachable,
27742815 .outdated => continue, // already queued for update
27752816
2817 .file_failure,
27762818 .dependency_failure,
27772819 .sema_failure,
27782820 .sema_failure_retryable,
......@@ -2793,24 +2835,13 @@ pub fn semaPkg(mod: *Module, pkg: *Package) !void {
27932835 return mod.semaFile(file);
27942836}
27952837
2838/// Regardless of the file status, will create a `Decl` so that we
2839/// can track dependencies and re-analyze when the file becomes outdated.
27962840pub fn semaFile(mod: *Module, file: *Scope.File) InnerError!void {
27972841 const tracy = trace(@src());
27982842 defer tracy.end();
27992843
2800 switch (file.status) {
2801 .never_loaded => unreachable,
2802
2803 .retryable_failure,
2804 .parse_failure,
2805 .astgen_failure,
2806 => return error.AnalysisFail,
2807
2808 .success_zir => {},
2809 .success_air => return,
2810 }
2811
2812 assert(file.zir_loaded);
2813 const main_struct_inst = file.zir.getMainStruct();
2844 if (file.namespace != null) return;
28142845
28152846 const gpa = mod.gpa;
28162847 var new_decl_arena = std.heap.ArenaAllocator.init(gpa);
......@@ -2823,7 +2854,7 @@ pub fn semaFile(mod: *Module, file: *Scope.File) InnerError!void {
28232854 .owner_decl = undefined, // set below
28242855 .fields = .{},
28252856 .node_offset = 0, // it's the struct for the root file
2826 .zir_index = main_struct_inst,
2857 .zir_index = undefined, // set below
28272858 .layout = .Auto,
28282859 .status = .none,
28292860 .namespace = .{
......@@ -2845,39 +2876,48 @@ pub fn semaFile(mod: *Module, file: *Scope.File) InnerError!void {
28452876 new_decl.val = struct_val;
28462877 new_decl.has_tv = true;
28472878 new_decl.owns_tv = true;
2848 new_decl.analysis = .complete;
2879 new_decl.analysis = .in_progress;
28492880 new_decl.generation = mod.generation;
28502881
2851 var sema_arena = std.heap.ArenaAllocator.init(gpa);
2852 defer sema_arena.deinit();
2882 if (file.status == .success_zir) {
2883 assert(file.zir_loaded);
2884 const main_struct_inst = file.zir.getMainStruct();
2885 struct_obj.zir_index = main_struct_inst;
2886
2887 var sema_arena = std.heap.ArenaAllocator.init(gpa);
2888 defer sema_arena.deinit();
2889
2890 var sema: Sema = .{
2891 .mod = mod,
2892 .gpa = gpa,
2893 .arena = &sema_arena.allocator,
2894 .code = file.zir,
2895 // TODO use a map because this array is too big
2896 .inst_map = try sema_arena.allocator.alloc(*ir.Inst, file.zir.instructions.len),
2897 .owner_decl = new_decl,
2898 .namespace = &struct_obj.namespace,
2899 .func = null,
2900 .owner_func = null,
2901 .param_inst_list = &.{},
2902 };
2903 var block_scope: Scope.Block = .{
2904 .parent = null,
2905 .sema = &sema,
2906 .src_decl = new_decl,
2907 .instructions = .{},
2908 .inlining = null,
2909 .is_comptime = true,
2910 };
2911 defer block_scope.instructions.deinit(gpa);
28532912
2854 var sema: Sema = .{
2855 .mod = mod,
2856 .gpa = gpa,
2857 .arena = &sema_arena.allocator,
2858 .code = file.zir,
2859 // TODO use a map because this array is too big
2860 .inst_map = try sema_arena.allocator.alloc(*ir.Inst, file.zir.instructions.len),
2861 .owner_decl = new_decl,
2862 .namespace = &struct_obj.namespace,
2863 .func = null,
2864 .owner_func = null,
2865 .param_inst_list = &.{},
2866 };
2867 var block_scope: Scope.Block = .{
2868 .parent = null,
2869 .sema = &sema,
2870 .src_decl = new_decl,
2871 .instructions = .{},
2872 .inlining = null,
2873 .is_comptime = true,
2874 };
2875 defer block_scope.instructions.deinit(gpa);
2913 try sema.analyzeStructDecl(new_decl, main_struct_inst, struct_obj);
28762914
2877 try sema.analyzeStructDecl(new_decl, main_struct_inst, struct_obj);
2878 try new_decl.finalizeNewArena(&new_decl_arena);
2915 new_decl.analysis = .complete;
2916 } else {
2917 new_decl.analysis = .file_failure;
2918 }
28792919
2880 file.status = .success_air;
2920 try new_decl.finalizeNewArena(&new_decl_arena);
28812921}
28822922
28832923/// Returns `true` if the Decl type changed.
......@@ -2887,6 +2927,10 @@ fn semaDecl(mod: *Module, decl: *Decl) !bool {
28872927 const tracy = trace(@src());
28882928 defer tracy.end();
28892929
2930 if (decl.namespace.file_scope.status != .success_zir) {
2931 return error.AnalysisFail;
2932 }
2933
28902934 const gpa = mod.gpa;
28912935 const zir = decl.namespace.file_scope.zir;
28922936 const zir_datas = zir.instructions.items(.data);
......@@ -2914,8 +2958,6 @@ fn semaDecl(mod: *Module, decl: *Decl) !bool {
29142958 const main_struct_inst = zir.getMainStruct();
29152959 const struct_obj = decl.getStruct().?;
29162960 try sema.analyzeStructDecl(decl, main_struct_inst, struct_obj);
2917 assert(decl.namespace.file_scope.status == .success_zir);
2918 decl.namespace.file_scope.status = .success_air;
29192961 decl.analysis = .complete;
29202962 decl.generation = mod.generation;
29212963 return false;
......@@ -3117,7 +3159,7 @@ pub fn importPkg(mod: *Module, cur_pkg: *Package, pkg: *Package) !ImportFileResu
31173159 .zir = undefined,
31183160 .status = .never_loaded,
31193161 .pkg = pkg,
3120 .namespace = undefined,
3162 .namespace = null,
31213163 };
31223164 return ImportFileResult{
31233165 .file = new_file,
......@@ -3183,7 +3225,7 @@ pub fn importFile(
31833225 .zir = undefined,
31843226 .status = .never_loaded,
31853227 .pkg = cur_file.pkg,
3186 .namespace = undefined,
3228 .namespace = null,
31873229 };
31883230 return ImportFileResult{
31893231 .file = new_file,
......@@ -4337,7 +4379,7 @@ pub fn optimizeMode(mod: Module) std.builtin.Mode {
43374379
43384380fn lockAndClearFileCompileError(mod: *Module, file: *Scope.File) void {
43394381 switch (file.status) {
4340 .success_zir, .success_air, .retryable_failure => {},
4382 .success_zir, .retryable_failure => {},
43414383 .never_loaded, .parse_failure, .astgen_failure => {
43424384 const lock = mod.comp.mutex.acquire();
43434385 defer lock.release();
src/Sema.zig+2-2
......@@ -4387,7 +4387,7 @@ fn zirImport(sema: *Sema, block: *Scope.Block, inst: Zir.Inst.Index) InnerError!
43874387 },
43884388 };
43894389 try mod.semaFile(result.file);
4390 return mod.constType(sema.arena, src, result.file.namespace.ty);
4390 return mod.constType(sema.arena, src, result.file.namespace.?.ty);
43914391}
43924392
43934393fn zirShl(sema: *Sema, block: *Scope.Block, inst: Zir.Inst.Index) InnerError!*Inst {
......@@ -7285,7 +7285,7 @@ fn getBuiltinType(
72857285 const opt_builtin_inst = try sema.analyzeNamespaceLookup(
72867286 block,
72877287 src,
7288 std_file.namespace,
7288 std_file.namespace.?,
72897289 "builtin",
72907290 );
72917291 const builtin_inst = try sema.analyzeLoad(block, src, opt_builtin_inst.?, src);