authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-04-07 19:38:00-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-04-07 19:54:28-07:00
log4996c2b6a94b042d86b50eb61c9d8d98e63415af
tree3857fef9a889a57d20925005579e7ef6784ea2e8
parent8f28e26e7a4f770f8d8e700386e2ade111948891

stage2: fix incremental compilation Decl deletion logic

* `analyzeContainer` now has an `outdated_decls` set as well as `deleted_decls`. Instead of queuing up outdated Decls for re-analysis right away, they are added to this new set. When processing the `deleted_decls` set, we remove deleted Decls from the `outdated_decls` set, to avoid deleted Decl pointers from being in the work_queue. Only after processing the deleted decls do we add analyze_decl work items to the queue. * Module.deletion_set is now an `AutoArrayHashMap` rather than `ArrayList`. `declareDeclDependency` will now remove a Decl from it as appropriate. When processing the `deletion_set` in `Compilation.performAllTheWork`, it now assumes all Decl in the set are to be deleted. * Fix crash when handling parse errors. Currently we unload the `ast.Tree` if any parse errors occur. Previously the code emitted a LazySrcLoc pointing to a token index, but then when we try to resolve the token index to a byte offset to create a compile error message, the ast.Tree` would be unloaded. Now we use `LazySrcLoc.byte_abs` instead of `token_abs` so the error message can be created even with the `ast.Tree` unloaded. Together, these changes solve a crash that happened with incremental compilation when Decls were added and removed in some combinations.

5 files changed, 145 insertions(+), 70 deletions(-)

src/AstGen.zig+3-3
......@@ -1899,9 +1899,9 @@ fn containerDecl(
18991899 if (member.ast.type_expr != 0) {
19001900 return mod.failNode(scope, member.ast.type_expr, "enum fields do not have types", .{});
19011901 }
1902 if (member.ast.align_expr != 0) {
1903 return mod.failNode(scope, member.ast.align_expr, "enum fields do not have alignments", .{});
1904 }
1902 // Alignment expressions in enums are caught by the parser.
1903 assert(member.ast.align_expr == 0);
1904
19051905 const name_token = member.ast.name_token;
19061906 if (mem.eql(u8, tree.tokenSlice(name_token), "_")) {
19071907 if (nonexhaustive_node != 0) {
src/Compilation.zig+10-7
......@@ -1377,14 +1377,17 @@ pub fn update(self: *Compilation) !void {
13771377
13781378 if (!use_stage1) {
13791379 if (self.bin_file.options.module) |module| {
1380 // Process the deletion set.
1381 while (module.deletion_set.popOrNull()) |decl| {
1382 if (decl.dependants.items().len != 0) {
1383 decl.deletion_flag = false;
1384 continue;
1385 }
1386 try module.deleteDecl(decl);
1380 // Process the deletion set. We use a while loop here because the
1381 // deletion set may grow as we call `deleteDecl` within this loop,
1382 // and more unreferenced Decls are revealed.
1383 var entry_i: usize = 0;
1384 while (entry_i < module.deletion_set.entries.items.len) : (entry_i += 1) {
1385 const decl = module.deletion_set.entries.items[entry_i].key;
1386 assert(decl.deletion_flag);
1387 assert(decl.dependants.items().len == 0);
1388 try module.deleteDecl(decl, null);
13871389 }
1390 module.deletion_set.shrinkRetainingCapacity(0);
13881391 }
13891392 }
13901393
src/Module.zig+79-18
......@@ -75,7 +75,7 @@ next_anon_name_index: usize = 0,
7575
7676/// Candidates for deletion. After a semantic analysis update completes, this list
7777/// contains Decls that need to be deleted if they end up having no references to them.
78deletion_set: ArrayListUnmanaged(*Decl) = .{},
78deletion_set: std.AutoArrayHashMapUnmanaged(*Decl, void) = .{},
7979
8080/// Error tags and their values, tag names are duped with mod.gpa.
8181/// Corresponds with `error_name_list`.
......@@ -192,7 +192,7 @@ pub const Decl = struct {
192192 /// to require re-analysis.
193193 outdated,
194194 },
195 /// This flag is set when this Decl is added to a check_for_deletion set, and cleared
195 /// This flag is set when this Decl is added to `Module.deletion_set`, and cleared
196196 /// when removed.
197197 deletion_flag: bool,
198198 /// Whether the corresponding AST decl has a `pub` keyword.
......@@ -2393,7 +2393,7 @@ pub fn ensureDeclAnalyzed(mod: *Module, decl: *Decl) InnerError!void {
23932393 // We don't perform a deletion here, because this Decl or another one
23942394 // may end up referencing it before the update is complete.
23952395 dep.deletion_flag = true;
2396 try mod.deletion_set.append(mod.gpa, dep);
2396 try mod.deletion_set.put(mod.gpa, dep, {});
23972397 }
23982398 }
23992399 decl.dependencies.clearRetainingCapacity();
......@@ -3197,6 +3197,11 @@ pub fn declareDeclDependency(mod: *Module, depender: *Decl, dependee: *Decl) !u3
31973197 try depender.dependencies.ensureCapacity(mod.gpa, depender.dependencies.count() + 1);
31983198 try dependee.dependants.ensureCapacity(mod.gpa, dependee.dependants.count() + 1);
31993199
3200 if (dependee.deletion_flag) {
3201 dependee.deletion_flag = false;
3202 mod.deletion_set.removeAssertDiscard(dependee);
3203 }
3204
32003205 dependee.dependants.putAssumeCapacity(depender, {});
32013206 const gop = depender.dependencies.getOrPutAssumeCapacity(dependee);
32023207 return @intCast(u32, gop.index);
......@@ -3224,12 +3229,14 @@ pub fn getAstTree(mod: *Module, root_scope: *Scope.File) !*const ast.Tree {
32243229 var msg = std.ArrayList(u8).init(mod.gpa);
32253230 defer msg.deinit();
32263231
3232 const token_starts = tree.tokens.items(.start);
3233
32273234 try tree.renderError(parse_err, msg.writer());
32283235 const err_msg = try mod.gpa.create(ErrorMsg);
32293236 err_msg.* = .{
32303237 .src_loc = .{
32313238 .container = .{ .file_scope = root_scope },
3232 .lazy = .{ .token_abs = parse_err.token },
3239 .lazy = .{ .byte_abs = token_starts[parse_err.token] },
32333240 },
32343241 .msg = msg.toOwnedSlice(),
32353242 };
......@@ -3274,6 +3281,14 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
32743281 deleted_decls.putAssumeCapacityNoClobber(entry.key, {});
32753282 }
32763283
3284 // Keep track of decls that are invalidated from the update. Ultimately,
3285 // the goal is to queue up `analyze_decl` tasks in the work queue for
3286 // the outdated decls, but we cannot queue up the tasks until after
3287 // we find out which ones have been deleted, otherwise there would be
3288 // deleted Decl pointers in the work queue.
3289 var outdated_decls = std.AutoArrayHashMap(*Decl, void).init(mod.gpa);
3290 defer outdated_decls.deinit();
3291
32773292 for (decls) |decl_node, decl_i| switch (node_tags[decl_node]) {
32783293 .fn_decl => {
32793294 const fn_proto = node_datas[decl_node].lhs;
......@@ -3284,6 +3299,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
32843299 try mod.semaContainerFn(
32853300 container_scope,
32863301 &deleted_decls,
3302 &outdated_decls,
32873303 decl_node,
32883304 decl_i,
32893305 tree.*,
......@@ -3294,6 +3310,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
32943310 .fn_proto_multi => try mod.semaContainerFn(
32953311 container_scope,
32963312 &deleted_decls,
3313 &outdated_decls,
32973314 decl_node,
32983315 decl_i,
32993316 tree.*,
......@@ -3305,6 +3322,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33053322 try mod.semaContainerFn(
33063323 container_scope,
33073324 &deleted_decls,
3325 &outdated_decls,
33083326 decl_node,
33093327 decl_i,
33103328 tree.*,
......@@ -3315,6 +3333,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33153333 .fn_proto => try mod.semaContainerFn(
33163334 container_scope,
33173335 &deleted_decls,
3336 &outdated_decls,
33183337 decl_node,
33193338 decl_i,
33203339 tree.*,
......@@ -3329,6 +3348,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33293348 try mod.semaContainerFn(
33303349 container_scope,
33313350 &deleted_decls,
3351 &outdated_decls,
33323352 decl_node,
33333353 decl_i,
33343354 tree.*,
......@@ -3339,6 +3359,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33393359 .fn_proto_multi => try mod.semaContainerFn(
33403360 container_scope,
33413361 &deleted_decls,
3362 &outdated_decls,
33423363 decl_node,
33433364 decl_i,
33443365 tree.*,
......@@ -3350,6 +3371,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33503371 try mod.semaContainerFn(
33513372 container_scope,
33523373 &deleted_decls,
3374 &outdated_decls,
33533375 decl_node,
33543376 decl_i,
33553377 tree.*,
......@@ -3360,6 +3382,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33603382 .fn_proto => try mod.semaContainerFn(
33613383 container_scope,
33623384 &deleted_decls,
3385 &outdated_decls,
33633386 decl_node,
33643387 decl_i,
33653388 tree.*,
......@@ -3370,6 +3393,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33703393 .global_var_decl => try mod.semaContainerVar(
33713394 container_scope,
33723395 &deleted_decls,
3396 &outdated_decls,
33733397 decl_node,
33743398 decl_i,
33753399 tree.*,
......@@ -3378,6 +3402,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33783402 .local_var_decl => try mod.semaContainerVar(
33793403 container_scope,
33803404 &deleted_decls,
3405 &outdated_decls,
33813406 decl_node,
33823407 decl_i,
33833408 tree.*,
......@@ -3386,6 +3411,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33863411 .simple_var_decl => try mod.semaContainerVar(
33873412 container_scope,
33883413 &deleted_decls,
3414 &outdated_decls,
33893415 decl_node,
33903416 decl_i,
33913417 tree.*,
......@@ -3394,6 +3420,7 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
33943420 .aligned_var_decl => try mod.semaContainerVar(
33953421 container_scope,
33963422 &deleted_decls,
3423 &outdated_decls,
33973424 decl_node,
33983425 decl_i,
33993426 tree.*,
......@@ -3446,11 +3473,27 @@ pub fn analyzeContainer(mod: *Module, container_scope: *Scope.Container) !void {
34463473 },
34473474 else => unreachable,
34483475 };
3449 // Handle explicitly deleted decls from the source code. Not to be confused
3450 // with when we delete decls because they are no longer referenced.
3476 // Handle explicitly deleted decls from the source code. This is one of two
3477 // places that Decl deletions happen. The other is in `Compilation`, after
3478 // `performAllTheWork`, where we iterate over `Module.deletion_set` and
3479 // delete Decls which are no longer referenced.
3480 // If a Decl is explicitly deleted from source, and also no longer referenced,
3481 // it may be both in this `deleted_decls` set, as well as in the
3482 // `Module.deletion_set`. To avoid deleting it twice, we remove it from the
3483 // deletion set at this time.
34513484 for (deleted_decls.items()) |entry| {
3452 log.debug("noticed '{s}' deleted from source", .{entry.key.name});
3453 try mod.deleteDecl(entry.key);
3485 const decl = entry.key;
3486 log.debug("'{s}' deleted from source", .{decl.name});
3487 if (decl.deletion_flag) {
3488 log.debug("'{s}' redundantly in deletion set; removing", .{decl.name});
3489 mod.deletion_set.removeAssertDiscard(decl);
3490 }
3491 try mod.deleteDecl(decl, &outdated_decls);
3492 }
3493 // Finally we can queue up re-analysis tasks after we have processed
3494 // the deleted decls.
3495 for (outdated_decls.items()) |entry| {
3496 try mod.markOutdatedDecl(entry.key);
34543497 }
34553498}
34563499
......@@ -3458,6 +3501,7 @@ fn semaContainerFn(
34583501 mod: *Module,
34593502 container_scope: *Scope.Container,
34603503 deleted_decls: *std.AutoArrayHashMap(*Decl, void),
3504 outdated_decls: *std.AutoArrayHashMap(*Decl, void),
34613505 decl_node: ast.Node.Index,
34623506 decl_i: usize,
34633507 tree: ast.Tree,
......@@ -3489,7 +3533,7 @@ fn semaContainerFn(
34893533 try mod.failed_decls.putNoClobber(mod.gpa, decl, msg);
34903534 } else {
34913535 if (!srcHashEql(decl.contents_hash, contents_hash)) {
3492 try mod.markOutdatedDecl(decl);
3536 try outdated_decls.put(decl, {});
34933537 decl.contents_hash = contents_hash;
34943538 } else switch (mod.comp.bin_file.tag) {
34953539 .coff => {
......@@ -3524,6 +3568,7 @@ fn semaContainerVar(
35243568 mod: *Module,
35253569 container_scope: *Scope.Container,
35263570 deleted_decls: *std.AutoArrayHashMap(*Decl, void),
3571 outdated_decls: *std.AutoArrayHashMap(*Decl, void),
35273572 decl_node: ast.Node.Index,
35283573 decl_i: usize,
35293574 tree: ast.Tree,
......@@ -3549,7 +3594,7 @@ fn semaContainerVar(
35493594 errdefer err_msg.destroy(mod.gpa);
35503595 try mod.failed_decls.putNoClobber(mod.gpa, decl, err_msg);
35513596 } else if (!srcHashEql(decl.contents_hash, contents_hash)) {
3552 try mod.markOutdatedDecl(decl);
3597 try outdated_decls.put(decl, {});
35533598 decl.contents_hash = contents_hash;
35543599 }
35553600 } else {
......@@ -3579,17 +3624,27 @@ fn semaContainerField(
35793624 log.err("TODO: analyze container field", .{});
35803625}
35813626
3582pub fn deleteDecl(mod: *Module, decl: *Decl) !void {
3627pub fn deleteDecl(
3628 mod: *Module,
3629 decl: *Decl,
3630 outdated_decls: ?*std.AutoArrayHashMap(*Decl, void),
3631) !void {
35833632 const tracy = trace(@src());
35843633 defer tracy.end();
35853634
3586 try mod.deletion_set.ensureCapacity(mod.gpa, mod.deletion_set.items.len + decl.dependencies.items().len);
3635 log.debug("deleting decl '{s}'", .{decl.name});
3636
3637 if (outdated_decls) |map| {
3638 _ = map.swapRemove(decl);
3639 try map.ensureCapacity(map.count() + decl.dependants.count());
3640 }
3641 try mod.deletion_set.ensureCapacity(mod.gpa, mod.deletion_set.count() +
3642 decl.dependencies.count());
35873643
35883644 // Remove from the namespace it resides in. In the case of an anonymous Decl it will
35893645 // not be present in the set, and this does nothing.
35903646 decl.container.removeDecl(decl);
35913647
3592 log.debug("deleting decl '{s}'", .{decl.name});
35933648 const name_hash = decl.fullyQualifiedNameHash();
35943649 mod.decl_table.removeAssertDiscard(name_hash);
35953650 // Remove itself from its dependencies, because we are about to destroy the decl pointer.
......@@ -3600,16 +3655,22 @@ pub fn deleteDecl(mod: *Module, decl: *Decl) !void {
36003655 // We don't recursively perform a deletion here, because during the update,
36013656 // another reference to it may turn up.
36023657 dep.deletion_flag = true;
3603 mod.deletion_set.appendAssumeCapacity(dep);
3658 mod.deletion_set.putAssumeCapacity(dep, {});
36043659 }
36053660 }
3606 // Anything that depends on this deleted decl certainly needs to be re-analyzed.
3661 // Anything that depends on this deleted decl needs to be re-analyzed.
36073662 for (decl.dependants.items()) |entry| {
36083663 const dep = entry.key;
36093664 dep.removeDependency(decl);
3610 if (dep.analysis != .outdated) {
3611 // TODO Move this failure possibility to the top of the function.
3612 try mod.markOutdatedDecl(dep);
3665 if (outdated_decls) |map| {
3666 map.putAssumeCapacity(dep, {});
3667 } else if (std.debug.runtime_safety) {
3668 // If `outdated_decls` is `null`, it means we're being called from
3669 // `Compilation` after `performAllTheWork` and we cannot queue up any
3670 // more work. `dep` must necessarily be another Decl that is no longer
3671 // being referenced, and will be in the `deletion_set`. Otherwise,
3672 // something has gone wrong.
3673 assert(mod.deletion_set.contains(dep));
36133674 }
36143675 }
36153676 if (mod.failed_decls.swapRemove(decl)) |entry| {
test/stage2/cbe.zig+53
......@@ -538,6 +538,45 @@ pub fn addCases(ctx: *TestContext) !void {
538538
539539 {
540540 var case = ctx.exeFromCompiledC("enums", .{});
541
542 case.addError(
543 \\const E1 = packed enum { a, b, c };
544 \\const E2 = extern enum { a, b, c };
545 \\export fn foo() void {
546 \\ const x = E1.a;
547 \\}
548 \\export fn bar() void {
549 \\ const x = E2.a;
550 \\}
551 , &.{
552 ":1:12: error: enums do not support 'packed' or 'extern'; instead provide an explicit integer tag type",
553 ":2:12: error: enums do not support 'packed' or 'extern'; instead provide an explicit integer tag type",
554 });
555
556 // comptime and types are caught in AstGen.
557 case.addError(
558 \\const E1 = enum {
559 \\ a,
560 \\ comptime b,
561 \\ c,
562 \\};
563 \\const E2 = enum {
564 \\ a,
565 \\ b: i32,
566 \\ c,
567 \\};
568 \\export fn foo() void {
569 \\ const x = E1.a;
570 \\}
571 \\export fn bar() void {
572 \\ const x = E2.a;
573 \\}
574 , &.{
575 ":3:5: error: enum fields cannot be marked comptime",
576 ":8:8: error: enum fields do not have types",
577 });
578
579 // @enumToInt, @intToEnum, enum literal coercion, field access syntax, comparison, switch
541580 case.addCompareOutput(
542581 \\const Number = enum { One, Two, Three };
543582 \\
......@@ -559,6 +598,20 @@ pub fn addCases(ctx: *TestContext) !void {
559598 \\ }
560599 \\}
561600 , "");
601
602 // Specifying alignment is a parse error.
603 case.addError(
604 \\const E1 = enum {
605 \\ a,
606 \\ b align(4),
607 \\ c,
608 \\};
609 \\export fn foo() void {
610 \\ const x = E1.a;
611 \\}
612 , &.{
613 ":3:7: error: expected ',', found 'align'",
614 });
562615 }
563616
564617 ctx.c("empty start function", linux_x64,
test/stage2/test.zig-42
......@@ -1598,46 +1598,4 @@ pub fn addCases(ctx: *TestContext) !void {
15981598 "",
15991599 );
16001600 }
1601 {
1602 var case = ctx.exe("enum_literal -> enum", linux_x64);
1603
1604 case.addCompareOutput(
1605 \\const E = enum { a, b };
1606 \\export fn _start() noreturn {
1607 \\ const a: E = .a;
1608 \\ const b: E = .b;
1609 \\ exit();
1610 \\}
1611 \\fn exit() noreturn {
1612 \\ asm volatile ("syscall"
1613 \\ :
1614 \\ : [number] "{rax}" (231),
1615 \\ [arg1] "{rdi}" (0)
1616 \\ : "rcx", "r11", "memory"
1617 \\ );
1618 \\ unreachable;
1619 \\}
1620 ,
1621 "",
1622 );
1623 case.addError(
1624 \\export fn _start() noreturn {
1625 \\ const a: E = .c;
1626 \\ exit();
1627 \\}
1628 \\const E = enum { a, b };
1629 \\fn exit() noreturn {
1630 \\ asm volatile ("syscall"
1631 \\ :
1632 \\ : [number] "{rax}" (231),
1633 \\ [arg1] "{rdi}" (0)
1634 \\ : "rcx", "r11", "memory"
1635 \\ );
1636 \\ unreachable;
1637 \\}
1638 , &.{
1639 ":2:19: error: enum 'E' has no field named 'c'",
1640 ":5:11: note: enum declared here",
1641 });
1642 }
16431601}