authorgravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-08-19 08:45:30+01:00
committergravatar for mlugg@mlugg.co.ukMatthew Lugg <mlugg@mlugg.co.uk> 2024-08-21 01:30:46+01:00
log43fdd061f7430794c142283101f9c97a0829d446
tree9d37f8b99735e51b290472d45f5e9b2b61aa7b3e
parent95fbfde9daa165ca74a4d7267720fd32b42af45e
signaturelock-open Commit is signed but in an unrecognized format.

AstGen: incorporate extra information into source hashes

* Indices of referenced captures * Line and column of `@src()` The second point aligns with a reversal of the "incremental compilation" section of https://github.com/ziglang/zig/issues/2029#issuecomment-645793168. This reversal was already done as #17688 (46a6d50), with the idea to push incremental compilation down the line. My proposal is to keep it as comptime-known, and simply re-analyze uses of `@src()` whenever their line/column change. I think this decision is reasonable for a few reasons: * The Zig compiler is quite fast. Occasionally re-analyzing a few functions containing `@src()` calls is perfectly acceptable and won't noticably impact update times. * The system described by Andrew in #2029 is currently vaporware. * The system described by Andrew in #2029 is non-trivial to implement. In particular, it requires some way to have backends update a single global in certain cases, without re-doing semantic analysis. There is no other part of incremental compilation which requires this. * Having `@src().line` be comptime-known is useful. For instance, #17688 was justified by broken Tracy integration because the source line couldn't be comptime-known.

1 files changed, 107 insertions(+), 30 deletions(-)

lib/std/zig/AstGen.zig+107-30
......@@ -66,6 +66,10 @@ scratch: std.ArrayListUnmanaged(u32) = .{},
6666/// of ZIR.
6767/// The key is the ref operand; the value is the ref instruction.
6868ref_table: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .{},
69/// Any information which should trigger invalidation of incremental compilation
70/// data should be used to update this hasher. The result is the final source
71/// hash of the enclosing declaration/etc.
72src_hasher: std.zig.SrcHasher,
6973
7074const InnerError = error{ OutOfMemory, AnalysisFail };
7175
......@@ -137,6 +141,7 @@ pub fn generate(gpa: Allocator, tree: Ast) Allocator.Error!Zir {
137141 .arena = arena.allocator(),
138142 .tree = &tree,
139143 .nodes_need_rl = &nodes_need_rl,
144 .src_hasher = undefined, // `structDeclInner` for the root struct will set this
140145 };
141146 defer astgen.deinit(gpa);
142147
......@@ -1422,6 +1427,8 @@ fn fnProtoExpr(
14221427 .is_extern = false,
14231428 .is_noinline = false,
14241429 .noalias_bits = noalias_bits,
1430
1431 .proto_hash = undefined, // ignored for `body_gz == null`
14251432 });
14261433
14271434 _ = try block_scope.addBreak(.break_inline, block_inst, result);
......@@ -4007,6 +4014,13 @@ fn fnDecl(
40074014 const tree = astgen.tree;
40084015 const token_tags = tree.tokens.items(.tag);
40094016
4017 const old_hasher = astgen.src_hasher;
4018 defer astgen.src_hasher = old_hasher;
4019 astgen.src_hasher = std.zig.SrcHasher.init(.{});
4020 // We don't add the full source yet, because we also need the prototype hash!
4021 // The source slice is added towards the *end* of this function.
4022 astgen.src_hasher.update(std.mem.asBytes(&astgen.source_column));
4023
40104024 // missing function name already happened in scanDecls()
40114025 const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail;
40124026
......@@ -4300,11 +4314,21 @@ fn fnDecl(
43004314 .is_extern = true,
43014315 .is_noinline = is_noinline,
43024316 .noalias_bits = noalias_bits,
4317 .proto_hash = undefined, // ignored for `body_gz == null`
43034318 });
43044319 } else func: {
43054320 // as a scope, fn_gz encloses ret_gz, but for instruction list, fn_gz stacks on ret_gz
43064321 fn_gz.instructions_top = ret_gz.instructions.items.len;
43074322
4323 // Construct the prototype hash.
4324 // Leave `astgen.src_hasher` unmodified; this will be used for hashing
4325 // the *whole* function declaration, including its body.
4326 var proto_hasher = astgen.src_hasher;
4327 const proto_node = tree.nodes.items(.data)[decl_node].lhs;
4328 proto_hasher.update(tree.getNodeSource(proto_node));
4329 var proto_hash: std.zig.SrcHash = undefined;
4330 proto_hasher.final(&proto_hash);
4331
43084332 const prev_fn_block = astgen.fn_block;
43094333 const prev_fn_ret_ty = astgen.fn_ret_ty;
43104334 defer {
......@@ -4362,16 +4386,22 @@ fn fnDecl(
43624386 .is_extern = false,
43634387 .is_noinline = is_noinline,
43644388 .noalias_bits = noalias_bits,
4389 .proto_hash = proto_hash,
43654390 });
43664391 };
43674392
4393 // *Now* we can incorporate the full source code into the hasher.
4394 astgen.src_hasher.update(tree.getNodeSource(decl_node));
4395
43684396 // We add this at the end so that its instruction index marks the end range
43694397 // of the top level declaration. addFunc already unstacked fn_gz and ret_gz.
43704398 _ = try decl_gz.addBreak(.break_inline, decl_inst, func_inst);
43714399
4400 var hash: std.zig.SrcHash = undefined;
4401 astgen.src_hasher.final(&hash);
43724402 try setDeclaration(
43734403 decl_inst,
4374 std.zig.hashSrc(tree.getNodeSource(decl_node)),
4404 hash,
43754405 .{ .named = fn_name_token },
43764406 decl_gz.decl_line,
43774407 is_pub,
......@@ -4395,6 +4425,12 @@ fn globalVarDecl(
43954425 const tree = astgen.tree;
43964426 const token_tags = tree.tokens.items(.tag);
43974427
4428 const old_hasher = astgen.src_hasher;
4429 defer astgen.src_hasher = old_hasher;
4430 astgen.src_hasher = std.zig.SrcHasher.init(.{});
4431 astgen.src_hasher.update(tree.getNodeSource(node));
4432 astgen.src_hasher.update(std.mem.asBytes(&astgen.source_column));
4433
43984434 const is_mutable = token_tags[var_decl.ast.mut_token] == .keyword_var;
43994435 // We do this at the beginning so that the instruction index marks the range start
44004436 // of the top level declaration.
......@@ -4534,9 +4570,11 @@ fn globalVarDecl(
45344570 _ = try addrspace_gz.addBreakWithSrcNode(.break_inline, decl_inst, addrspace_inst, node);
45354571 }
45364572
4573 var hash: std.zig.SrcHash = undefined;
4574 astgen.src_hasher.final(&hash);
45374575 try setDeclaration(
45384576 decl_inst,
4539 std.zig.hashSrc(tree.getNodeSource(node)),
4577 hash,
45404578 .{ .named = name_token },
45414579 block_scope.decl_line,
45424580 is_pub,
......@@ -4562,6 +4600,12 @@ fn comptimeDecl(
45624600 const node_datas = tree.nodes.items(.data);
45634601 const body_node = node_datas[node].lhs;
45644602
4603 const old_hasher = astgen.src_hasher;
4604 defer astgen.src_hasher = old_hasher;
4605 astgen.src_hasher = std.zig.SrcHasher.init(.{});
4606 astgen.src_hasher.update(tree.getNodeSource(node));
4607 astgen.src_hasher.update(std.mem.asBytes(&astgen.source_column));
4608
45654609 // Up top so the ZIR instruction index marks the start range of this
45664610 // top-level declaration.
45674611 const decl_inst = try gz.makeDeclaration(node);
......@@ -4584,9 +4628,11 @@ fn comptimeDecl(
45844628 _ = try decl_block.addBreak(.break_inline, decl_inst, .void_value);
45854629 }
45864630
4631 var hash: std.zig.SrcHash = undefined;
4632 astgen.src_hasher.final(&hash);
45874633 try setDeclaration(
45884634 decl_inst,
4589 std.zig.hashSrc(tree.getNodeSource(node)),
4635 hash,
45904636 .@"comptime",
45914637 decl_block.decl_line,
45924638 false,
......@@ -4607,6 +4653,12 @@ fn usingnamespaceDecl(
46074653 const tree = astgen.tree;
46084654 const node_datas = tree.nodes.items(.data);
46094655
4656 const old_hasher = astgen.src_hasher;
4657 defer astgen.src_hasher = old_hasher;
4658 astgen.src_hasher = std.zig.SrcHasher.init(.{});
4659 astgen.src_hasher.update(tree.getNodeSource(node));
4660 astgen.src_hasher.update(std.mem.asBytes(&astgen.source_column));
4661
46104662 const type_expr = node_datas[node].lhs;
46114663 const is_pub = blk: {
46124664 const main_tokens = tree.nodes.items(.main_token);
......@@ -4634,9 +4686,11 @@ fn usingnamespaceDecl(
46344686 const namespace_inst = try typeExpr(&decl_block, &decl_block.base, type_expr);
46354687 _ = try decl_block.addBreak(.break_inline, decl_inst, namespace_inst);
46364688
4689 var hash: std.zig.SrcHash = undefined;
4690 astgen.src_hasher.final(&hash);
46374691 try setDeclaration(
46384692 decl_inst,
4639 std.zig.hashSrc(tree.getNodeSource(node)),
4693 hash,
46404694 .@"usingnamespace",
46414695 decl_block.decl_line,
46424696 is_pub,
......@@ -4658,6 +4712,12 @@ fn testDecl(
46584712 const node_datas = tree.nodes.items(.data);
46594713 const body_node = node_datas[node].rhs;
46604714
4715 const old_hasher = astgen.src_hasher;
4716 defer astgen.src_hasher = old_hasher;
4717 astgen.src_hasher = std.zig.SrcHasher.init(.{});
4718 astgen.src_hasher.update(tree.getNodeSource(node));
4719 astgen.src_hasher.update(std.mem.asBytes(&astgen.source_column));
4720
46614721 // Up top so the ZIR instruction index marks the start range of this
46624722 // top-level declaration.
46634723 const decl_inst = try gz.makeDeclaration(node);
......@@ -4819,13 +4879,18 @@ fn testDecl(
48194879 .is_extern = false,
48204880 .is_noinline = false,
48214881 .noalias_bits = 0,
4882
4883 // Tests don't have a prototype that needs hashing
4884 .proto_hash = .{0} ** 16,
48224885 });
48234886
48244887 _ = try decl_block.addBreak(.break_inline, decl_inst, func_inst);
48254888
4889 var hash: std.zig.SrcHash = undefined;
4890 astgen.src_hasher.final(&hash);
48264891 try setDeclaration(
48274892 decl_inst,
4828 std.zig.hashSrc(tree.getNodeSource(node)),
4893 hash,
48294894 test_name,
48304895 decl_block.decl_line,
48314896 false,
......@@ -4983,10 +5048,12 @@ fn structDeclInner(
49835048 }
49845049 };
49855050
4986 var fields_hasher = std.zig.SrcHasher.init(.{});
4987 fields_hasher.update(@tagName(layout));
5051 const old_hasher = astgen.src_hasher;
5052 defer astgen.src_hasher = old_hasher;
5053 astgen.src_hasher = std.zig.SrcHasher.init(.{});
5054 astgen.src_hasher.update(@tagName(layout));
49885055 if (backing_int_node != 0) {
4989 fields_hasher.update(tree.getNodeSource(backing_int_node));
5056 astgen.src_hasher.update(tree.getNodeSource(backing_int_node));
49905057 }
49915058
49925059 var sfba = std.heap.stackFallback(256, astgen.arena);
......@@ -5009,7 +5076,7 @@ fn structDeclInner(
50095076 .field => |field| field,
50105077 };
50115078
5012 fields_hasher.update(tree.getNodeSource(member_node));
5079 astgen.src_hasher.update(tree.getNodeSource(member_node));
50135080
50145081 if (!is_tuple) {
50155082 const field_name = try astgen.identAsString(member.ast.main_token);
......@@ -5139,7 +5206,7 @@ fn structDeclInner(
51395206 }
51405207
51415208 var fields_hash: std.zig.SrcHash = undefined;
5142 fields_hasher.final(&fields_hash);
5209 astgen.src_hasher.final(&fields_hash);
51435210
51445211 try gz.setStruct(decl_inst, .{
51455212 .src_node = node,
......@@ -5240,11 +5307,13 @@ fn unionDeclInner(
52405307 var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size);
52415308 defer wip_members.deinit();
52425309
5243 var fields_hasher = std.zig.SrcHasher.init(.{});
5244 fields_hasher.update(@tagName(layout));
5245 fields_hasher.update(&.{@intFromBool(auto_enum_tok != null)});
5310 const old_hasher = astgen.src_hasher;
5311 defer astgen.src_hasher = old_hasher;
5312 astgen.src_hasher = std.zig.SrcHasher.init(.{});
5313 astgen.src_hasher.update(@tagName(layout));
5314 astgen.src_hasher.update(&.{@intFromBool(auto_enum_tok != null)});
52465315 if (arg_node != 0) {
5247 fields_hasher.update(astgen.tree.getNodeSource(arg_node));
5316 astgen.src_hasher.update(astgen.tree.getNodeSource(arg_node));
52485317 }
52495318
52505319 var sfba = std.heap.stackFallback(256, astgen.arena);
......@@ -5261,7 +5330,7 @@ fn unionDeclInner(
52615330 .decl => continue,
52625331 .field => |field| field,
52635332 };
5264 fields_hasher.update(astgen.tree.getNodeSource(member_node));
5333 astgen.src_hasher.update(astgen.tree.getNodeSource(member_node));
52655334 member.convertToNonTupleLike(astgen.tree.nodes);
52665335 if (member.ast.tuple_like) {
52675336 return astgen.failTok(member.ast.main_token, "union field missing name", .{});
......@@ -5364,7 +5433,7 @@ fn unionDeclInner(
53645433 }
53655434
53665435 var fields_hash: std.zig.SrcHash = undefined;
5367 fields_hasher.final(&fields_hash);
5436 astgen.src_hasher.final(&fields_hash);
53685437
53695438 if (!block_scope.isEmpty()) {
53705439 _ = try block_scope.addBreak(.break_inline, decl_inst, .void_value);
......@@ -5578,11 +5647,13 @@ fn containerDecl(
55785647 var wip_members = try WipMembers.init(gpa, &astgen.scratch, @intCast(counts.decls), @intCast(counts.total_fields), bits_per_field, max_field_size);
55795648 defer wip_members.deinit();
55805649
5581 var fields_hasher = std.zig.SrcHasher.init(.{});
5650 const old_hasher = astgen.src_hasher;
5651 defer astgen.src_hasher = old_hasher;
5652 astgen.src_hasher = std.zig.SrcHasher.init(.{});
55825653 if (container_decl.ast.arg != 0) {
5583 fields_hasher.update(tree.getNodeSource(container_decl.ast.arg));
5654 astgen.src_hasher.update(tree.getNodeSource(container_decl.ast.arg));
55845655 }
5585 fields_hasher.update(&.{@intFromBool(nonexhaustive)});
5656 astgen.src_hasher.update(&.{@intFromBool(nonexhaustive)});
55865657
55875658 var sfba = std.heap.stackFallback(256, astgen.arena);
55885659 const sfba_allocator = sfba.get();
......@@ -5596,7 +5667,7 @@ fn containerDecl(
55965667 for (container_decl.ast.members) |member_node| {
55975668 if (member_node == counts.nonexhaustive_node)
55985669 continue;
5599 fields_hasher.update(tree.getNodeSource(member_node));
5670 astgen.src_hasher.update(tree.getNodeSource(member_node));
56005671 var member = switch (try containerMember(&block_scope, &namespace.base, &wip_members, member_node)) {
56015672 .decl => continue,
56025673 .field => |field| field,
......@@ -5676,7 +5747,7 @@ fn containerDecl(
56765747 }
56775748
56785749 var fields_hash: std.zig.SrcHash = undefined;
5679 fields_hasher.final(&fields_hash);
5750 astgen.src_hasher.final(&fields_hash);
56805751
56815752 const body = block_scope.instructionsSlice();
56825753 const body_len = astgen.countBodyLenAfterFixups(body);
......@@ -8478,6 +8549,10 @@ fn tunnelThroughClosure(
84788549 });
84798550 }
84808551
8552 // Incorporate the capture index into the source hash, so that changes in
8553 // the order of captures cause suitable re-analysis.
8554 astgen.src_hasher.update(std.mem.asBytes(&cur_capture_index));
8555
84818556 // Add an instruction to get the value from the closure.
84828557 return gz.addExtendedNodeSmall(.closure_get, inner_ref_node, cur_capture_index);
84838558}
......@@ -9306,6 +9381,13 @@ fn builtinCall(
93069381 },
93079382
93089383 .src => {
9384 // Incorporate the source location into the source hash, so that
9385 // changes in the source location of `@src()` result in re-analysis.
9386 astgen.src_hasher.update(
9387 std.mem.asBytes(&astgen.source_line) ++
9388 std.mem.asBytes(&astgen.source_column),
9389 );
9390
93099391 const token_starts = tree.tokens.items(.start);
93109392 const node_start = token_starts[tree.firstToken(node)];
93119393 astgen.advanceSourceCursor(node_start);
......@@ -12122,6 +12204,9 @@ const GenZir = struct {
1212212204 is_test: bool,
1212312205 is_extern: bool,
1212412206 is_noinline: bool,
12207
12208 /// Ignored if `body_gz == null`.
12209 proto_hash: std.zig.SrcHash,
1212512210 }) !Zir.Inst.Ref {
1212612211 assert(args.src_node != 0);
1212712212 const astgen = gz.astgen;
......@@ -12150,15 +12235,7 @@ const GenZir = struct {
1215012235
1215112236 const columns = args.lbrace_column | (rbrace_column << 16);
1215212237
12153 const proto_hash: std.zig.SrcHash = switch (node_tags[fn_decl]) {
12154 .fn_decl => sig_hash: {
12155 const proto_node = node_datas[fn_decl].lhs;
12156 break :sig_hash std.zig.hashSrc(tree.getNodeSource(proto_node));
12157 },
12158 .test_decl => std.zig.hashSrc(""), // tests don't have a prototype
12159 else => unreachable,
12160 };
12161 const proto_hash_arr: [4]u32 = @bitCast(proto_hash);
12238 const proto_hash_arr: [4]u32 = @bitCast(args.proto_hash);
1216212239
1216312240 src_locs_and_hash_buffer = .{
1216412241 args.lbrace_line,