authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-04-26 17:36:28-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-04-26 17:36:28-07:00
log91c317bb9aa906684104db3d73442ab1198a83f4
tree6d475619655b3b0b6d2914e9b11ad79d21b351b7
parent646eb1fa934c413f29505f4742c67e61314acd90

AstGen: improved handling of declarations

* Every decl provides a 16 byte source hash which can be used to detect if the source code for any particular decl has changed. * Include comptime decls, test decls, and usingnamespace decls in the decls list of namespaces. - Tests are encoded as extended functions with is_test bit set.

3 files changed, 170 insertions(+), 64 deletions(-)

src/AstGen.zig+123-46
......@@ -35,10 +35,10 @@ string_bytes: ArrayListUnmanaged(u8) = .{},
3535arena: *Allocator,
3636string_table: std.StringHashMapUnmanaged(u32) = .{},
3737compile_errors: ArrayListUnmanaged(Zir.Inst.CompileErrors.Item) = .{},
38/// String table indexes, keeps track of all `@import` operands.
39imports: std.AutoArrayHashMapUnmanaged(u32, void) = .{},
4038/// The topmost block of the current function.
4139fn_block: ?*GenZir = null,
40/// String table indexes, keeps track of all `@import` operands.
41imports: std.AutoArrayHashMapUnmanaged(u32, void) = .{},
4242
4343pub fn addExtra(astgen: *AstGen, extra: anytype) Allocator.Error!u32 {
4444 const fields = std.meta.fields(@TypeOf(extra));
......@@ -1078,6 +1078,7 @@ pub fn fnProtoExpr(
10781078 .lib_name = 0,
10791079 .is_var_args = is_var_args,
10801080 .is_inferred_error = false,
1081 .is_test = false,
10811082 });
10821083 return rvalue(gz, scope, rl, result, fn_proto.ast.proto_node);
10831084}
......@@ -2610,6 +2611,26 @@ const WipDecls = struct {
26102611 const bits_per_field = 4;
26112612 const fields_per_u32 = 32 / bits_per_field;
26122613
2614 fn next(
2615 wip_decls: *WipDecls,
2616 gpa: *Allocator,
2617 is_pub: bool,
2618 is_export: bool,
2619 has_align: bool,
2620 has_section: bool,
2621 ) Allocator.Error!void {
2622 if (wip_decls.decl_index % fields_per_u32 == 0 and wip_decls.decl_index != 0) {
2623 try wip_decls.bit_bag.append(gpa, wip_decls.cur_bit_bag);
2624 wip_decls.cur_bit_bag = 0;
2625 }
2626 wip_decls.cur_bit_bag = (wip_decls.cur_bit_bag >> bits_per_field) |
2627 (@as(u32, @boolToInt(is_pub)) << 28) |
2628 (@as(u32, @boolToInt(is_export)) << 29) |
2629 (@as(u32, @boolToInt(has_align)) << 30) |
2630 (@as(u32, @boolToInt(has_section)) << 31);
2631 wip_decls.decl_index += 1;
2632 }
2633
26132634 fn deinit(wip_decls: *WipDecls, gpa: *Allocator) void {
26142635 wip_decls.bit_bag.deinit(gpa);
26152636 wip_decls.payload.deinit(gpa);
......@@ -2652,16 +2673,7 @@ fn fnDecl(
26522673 break :inst try comptimeExpr(&decl_gz, &decl_gz.base, .{ .ty = .const_slice_u8_type }, fn_proto.ast.section_expr);
26532674 };
26542675
2655 if (wip_decls.decl_index % WipDecls.fields_per_u32 == 0 and wip_decls.decl_index != 0) {
2656 try wip_decls.bit_bag.append(gpa, wip_decls.cur_bit_bag);
2657 wip_decls.cur_bit_bag = 0;
2658 }
2659 wip_decls.cur_bit_bag = (wip_decls.cur_bit_bag >> WipDecls.bits_per_field) |
2660 (@as(u32, @boolToInt(is_pub)) << 28) |
2661 (@as(u32, @boolToInt(is_export)) << 29) |
2662 (@as(u32, @boolToInt(align_inst != .none)) << 30) |
2663 (@as(u32, @boolToInt(section_inst != .none)) << 31);
2664 wip_decls.decl_index += 1;
2676 try wip_decls.next(gpa, is_pub, is_export, align_inst != .none, section_inst != .none);
26652677
26662678 // The AST params array does not contain anytype and ... parameters.
26672679 // We must iterate to count how many param types to allocate.
......@@ -2750,6 +2762,7 @@ fn fnDecl(
27502762 .lib_name = lib_name,
27512763 .is_var_args = is_var_args,
27522764 .is_inferred_error = false,
2765 .is_test = false,
27532766 });
27542767 } else func: {
27552768 if (is_var_args) {
......@@ -2821,6 +2834,7 @@ fn fnDecl(
28212834 .lib_name = lib_name,
28222835 .is_var_args = is_var_args,
28232836 .is_inferred_error = is_inferred_error,
2837 .is_test = false,
28242838 });
28252839 };
28262840
......@@ -2833,7 +2847,12 @@ fn fnDecl(
28332847 _ = try decl_gz.addBreak(.break_inline, block_inst, func_inst);
28342848 try decl_gz.setBlockBody(block_inst);
28352849
2836 try wip_decls.payload.ensureUnusedCapacity(gpa, 4);
2850 try wip_decls.payload.ensureUnusedCapacity(gpa, 8);
2851 {
2852 const contents_hash = std.zig.hashSrc(tree.getNodeSource(fn_proto.ast.proto_node));
2853 const casted = @bitCast([4]u32, contents_hash);
2854 wip_decls.payload.appendSliceAssumeCapacity(&casted);
2855 }
28372856 wip_decls.payload.appendAssumeCapacity(fn_name_str_index);
28382857 wip_decls.payload.appendAssumeCapacity(block_inst);
28392858 if (align_inst != .none) {
......@@ -2879,16 +2898,7 @@ fn globalVarDecl(
28792898 const section_inst: Zir.Inst.Ref = if (var_decl.ast.section_node == 0) .none else inst: {
28802899 break :inst try comptimeExpr(&block_scope, &block_scope.base, .{ .ty = .const_slice_u8_type }, var_decl.ast.section_node);
28812900 };
2882 if (wip_decls.decl_index % WipDecls.fields_per_u32 == 0 and wip_decls.decl_index != 0) {
2883 try wip_decls.bit_bag.append(gpa, wip_decls.cur_bit_bag);
2884 wip_decls.cur_bit_bag = 0;
2885 }
2886 wip_decls.cur_bit_bag = (wip_decls.cur_bit_bag >> WipDecls.bits_per_field) |
2887 (@as(u32, @boolToInt(is_pub)) << 28) |
2888 (@as(u32, @boolToInt(is_export)) << 29) |
2889 (@as(u32, @boolToInt(align_inst != .none)) << 30) |
2890 (@as(u32, @boolToInt(section_inst != .none)) << 31);
2891 wip_decls.decl_index += 1;
2901 try wip_decls.next(gpa, is_pub, is_export, align_inst != .none, section_inst != .none);
28922902
28932903 const is_mutable = token_tags[var_decl.ast.mut_token] == .keyword_var;
28942904 const is_threadlocal = if (var_decl.threadlocal_token) |tok| blk: {
......@@ -2950,7 +2960,12 @@ fn globalVarDecl(
29502960 const name_token = var_decl.ast.mut_token + 1;
29512961 const name_str_index = try gz.identAsString(name_token);
29522962
2953 try wip_decls.payload.ensureUnusedCapacity(gpa, 4);
2963 try wip_decls.payload.ensureUnusedCapacity(gpa, 8);
2964 {
2965 const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
2966 const casted = @bitCast([4]u32, contents_hash);
2967 wip_decls.payload.appendSliceAssumeCapacity(&casted);
2968 }
29542969 wip_decls.payload.appendAssumeCapacity(name_str_index);
29552970 wip_decls.payload.appendAssumeCapacity(var_inst);
29562971 if (align_inst != .none) {
......@@ -2965,21 +2980,48 @@ fn comptimeDecl(
29652980 astgen: *AstGen,
29662981 gz: *GenZir,
29672982 scope: *Scope,
2983 wip_decls: *WipDecls,
29682984 node: ast.Node.Index,
29692985) InnerError!void {
2986 const gpa = astgen.gpa;
29702987 const tree = &astgen.file.tree;
29712988 const node_datas = tree.nodes.items(.data);
2972 const block_expr = node_datas[node].lhs;
2973 // TODO probably we want to put these into a block and store a list of them
2974 _ = try expr(gz, scope, .none, block_expr);
2989 const body_node = node_datas[node].lhs;
2990
2991 // Up top so the ZIR instruction index marks the start range of this
2992 // top-level declaration.
2993 const block_inst = try gz.addBlock(.block_inline, node);
2994 try wip_decls.next(gpa, false, false, false, false);
2995
2996 var decl_block: GenZir = .{
2997 .force_comptime = true,
2998 .decl_node_index = node,
2999 .parent = scope,
3000 .astgen = astgen,
3001 };
3002 defer decl_block.instructions.deinit(gpa);
3003
3004 _ = try expr(&decl_block, &decl_block.base, .none, body_node);
3005 try decl_block.setBlockBody(block_inst);
3006
3007 try wip_decls.payload.ensureUnusedCapacity(gpa, 6);
3008 {
3009 const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
3010 const casted = @bitCast([4]u32, contents_hash);
3011 wip_decls.payload.appendSliceAssumeCapacity(&casted);
3012 }
3013 wip_decls.payload.appendAssumeCapacity(0);
3014 wip_decls.payload.appendAssumeCapacity(block_inst);
29753015}
29763016
29773017fn usingnamespaceDecl(
29783018 astgen: *AstGen,
29793019 gz: *GenZir,
29803020 scope: *Scope,
3021 wip_decls: *WipDecls,
29813022 node: ast.Node.Index,
29823023) InnerError!void {
3024 const gpa = astgen.gpa;
29833025 const tree = &astgen.file.tree;
29843026 const node_datas = tree.nodes.items(.data);
29853027
......@@ -2990,14 +3032,38 @@ fn usingnamespaceDecl(
29903032 const main_token = main_tokens[node];
29913033 break :blk (main_token > 0 and token_tags[main_token - 1] == .keyword_pub);
29923034 };
2993 // TODO probably we want to put these into a block and store a list of them
2994 const namespace_inst = try expr(gz, scope, .{ .ty = .type_type }, type_expr);
3035 // Up top so the ZIR instruction index marks the start range of this
3036 // top-level declaration.
3037 const block_inst = try gz.addBlock(.block_inline, node);
3038 try wip_decls.next(gpa, is_pub, true, false, false);
3039
3040 var decl_block: GenZir = .{
3041 .force_comptime = true,
3042 .decl_node_index = node,
3043 .parent = scope,
3044 .astgen = astgen,
3045 };
3046 defer decl_block.instructions.deinit(gpa);
3047
3048 const namespace_inst = try typeExpr(&decl_block, &decl_block.base, type_expr);
3049 _ = try decl_block.addBreak(.break_inline, block_inst, namespace_inst);
3050 try decl_block.setBlockBody(block_inst);
3051
3052 try wip_decls.payload.ensureUnusedCapacity(gpa, 6);
3053 {
3054 const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
3055 const casted = @bitCast([4]u32, contents_hash);
3056 wip_decls.payload.appendSliceAssumeCapacity(&casted);
3057 }
3058 wip_decls.payload.appendAssumeCapacity(0);
3059 wip_decls.payload.appendAssumeCapacity(block_inst);
29953060}
29963061
29973062fn testDecl(
29983063 astgen: *AstGen,
29993064 gz: *GenZir,
30003065 scope: *Scope,
3066 wip_decls: *WipDecls,
30013067 node: ast.Node.Index,
30023068) InnerError!void {
30033069 const gpa = astgen.gpa;
......@@ -3005,10 +3071,16 @@ fn testDecl(
30053071 const node_datas = tree.nodes.items(.data);
30063072 const body_node = node_datas[node].rhs;
30073073
3074 // Up top so the ZIR instruction index marks the start range of this
3075 // top-level declaration.
3076 const block_inst = try gz.addBlock(.block_inline, node);
3077
3078 try wip_decls.next(gpa, false, false, false, false);
3079
30083080 var decl_block: GenZir = .{
30093081 .force_comptime = true,
30103082 .decl_node_index = node,
3011 .parent = &gz.base,
3083 .parent = scope,
30123084 .astgen = astgen,
30133085 };
30143086 defer decl_block.instructions.deinit(gpa);
......@@ -3053,15 +3125,20 @@ fn testDecl(
30533125 .lib_name = 0,
30543126 .is_var_args = false,
30553127 .is_inferred_error = true,
3128 .is_test = true,
30563129 });
30573130
3058 const block_inst = try gz.addBlock(.block_inline, node);
30593131 _ = try decl_block.addBreak(.break_inline, block_inst, func_inst);
30603132 try decl_block.setBlockBody(block_inst);
30613133
3062 // TODO collect these into a test decl list
3063 _ = test_name;
3064 _ = block_inst;
3134 try wip_decls.payload.ensureUnusedCapacity(gpa, 6);
3135 {
3136 const contents_hash = std.zig.hashSrc(tree.getNodeSource(node));
3137 const casted = @bitCast([4]u32, contents_hash);
3138 wip_decls.payload.appendSliceAssumeCapacity(&casted);
3139 }
3140 wip_decls.payload.appendAssumeCapacity(test_name);
3141 wip_decls.payload.appendAssumeCapacity(block_inst);
30653142}
30663143
30673144fn structDeclInner(
......@@ -3179,15 +3256,15 @@ fn structDeclInner(
31793256 },
31803257
31813258 .@"comptime" => {
3182 try astgen.comptimeDecl(gz, scope, member_node);
3259 try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
31833260 continue;
31843261 },
31853262 .@"usingnamespace" => {
3186 try astgen.usingnamespaceDecl(gz, scope, member_node);
3263 try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
31873264 continue;
31883265 },
31893266 .test_decl => {
3190 try astgen.testDecl(gz, scope, member_node);
3267 try astgen.testDecl(gz, scope, &wip_decls, member_node);
31913268 continue;
31923269 },
31933270 else => unreachable,
......@@ -3382,15 +3459,15 @@ fn unionDeclInner(
33823459 },
33833460
33843461 .@"comptime" => {
3385 try astgen.comptimeDecl(gz, scope, member_node);
3462 try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
33863463 continue;
33873464 },
33883465 .@"usingnamespace" => {
3389 try astgen.usingnamespaceDecl(gz, scope, member_node);
3466 try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
33903467 continue;
33913468 },
33923469 .test_decl => {
3393 try astgen.testDecl(gz, scope, member_node);
3470 try astgen.testDecl(gz, scope, &wip_decls, member_node);
33943471 continue;
33953472 },
33963473 else => unreachable,
......@@ -3731,15 +3808,15 @@ fn containerDecl(
37313808 },
37323809
37333810 .@"comptime" => {
3734 try astgen.comptimeDecl(gz, scope, member_node);
3811 try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
37353812 continue;
37363813 },
37373814 .@"usingnamespace" => {
3738 try astgen.usingnamespaceDecl(gz, scope, member_node);
3815 try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
37393816 continue;
37403817 },
37413818 .test_decl => {
3742 try astgen.testDecl(gz, scope, member_node);
3819 try astgen.testDecl(gz, scope, &wip_decls, member_node);
37433820 continue;
37443821 },
37453822 else => unreachable,
......@@ -3896,15 +3973,15 @@ fn containerDecl(
38963973 },
38973974
38983975 .@"comptime" => {
3899 try astgen.comptimeDecl(gz, scope, member_node);
3976 try astgen.comptimeDecl(gz, scope, &wip_decls, member_node);
39003977 continue;
39013978 },
39023979 .@"usingnamespace" => {
3903 try astgen.usingnamespaceDecl(gz, scope, member_node);
3980 try astgen.usingnamespaceDecl(gz, scope, &wip_decls, member_node);
39043981 continue;
39053982 },
39063983 .test_decl => {
3907 try astgen.testDecl(gz, scope, member_node);
3984 try astgen.testDecl(gz, scope, &wip_decls, member_node);
39083985 continue;
39093986 },
39103987 else => unreachable,
src/Module.zig+3-1
......@@ -1311,6 +1311,7 @@ pub const Scope = struct {
13111311 lib_name: u32,
13121312 is_var_args: bool,
13131313 is_inferred_error: bool,
1314 is_test: bool,
13141315 }) !Zir.Inst.Ref {
13151316 assert(args.src_node != 0);
13161317 assert(args.ret_ty != .none);
......@@ -1320,7 +1321,7 @@ pub const Scope = struct {
13201321 try gz.instructions.ensureUnusedCapacity(gpa, 1);
13211322 try astgen.instructions.ensureUnusedCapacity(gpa, 1);
13221323
1323 if (args.cc != .none or args.lib_name != 0 or args.is_var_args) {
1324 if (args.cc != .none or args.lib_name != 0 or args.is_var_args or args.is_test) {
13241325 try astgen.extra.ensureUnusedCapacity(
13251326 gpa,
13261327 @typeInfo(Zir.Inst.ExtendedFunc).Struct.fields.len +
......@@ -1353,6 +1354,7 @@ pub const Scope = struct {
13531354 .is_inferred_error = args.is_inferred_error,
13541355 .has_lib_name = args.lib_name != 0,
13551356 .has_cc = args.cc != .none,
1357 .is_test = args.is_test,
13561358 }),
13571359 .operand = payload_index,
13581360 } },
src/Zir.zig+44-17
......@@ -2201,7 +2201,8 @@ pub const Inst = struct {
22012201 is_inferred_error: bool,
22022202 has_lib_name: bool,
22032203 has_cc: bool,
2204 _: u12 = undefined,
2204 is_test: bool,
2205 _: u11 = undefined,
22052206 };
22062207 };
22072208
......@@ -2375,7 +2376,10 @@ pub const Inst = struct {
23752376 /// 0b0X00: whether corresponding decl has an align expression
23762377 /// 0bX000: whether corresponding decl has a linksection expression
23772378 /// 1. decl: { // for every decls_len
2379 /// src_hash: [4]u32, // hash of source bytes
23782380 /// name: u32, // null terminated string index
2381 /// - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
2382 /// - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
23792383 /// value: Index,
23802384 /// align: Ref, // if corresponding bit is set
23812385 /// link_section: Ref, // if corresponding bit is set
......@@ -2405,7 +2409,10 @@ pub const Inst = struct {
24052409 /// 0b0X00: whether corresponding decl has an align expression
24062410 /// 0bX000: whether corresponding decl has a linksection expression
24072411 /// 1. decl: { // for every decls_len
2412 /// src_hash: [4]u32, // hash of source bytes
24082413 /// name: u32, // null terminated string index
2414 /// - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
2415 /// - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
24092416 /// value: Index,
24102417 /// align: Ref, // if corresponding bit is set
24112418 /// link_section: Ref, // if corresponding bit is set
......@@ -2433,7 +2440,10 @@ pub const Inst = struct {
24332440 /// 0b0X00: whether corresponding decl has an align expression
24342441 /// 0bX000: whether corresponding decl has a linksection expression
24352442 /// 1. decl: { // for every decls_len
2443 /// src_hash: [4]u32, // hash of source bytes
24362444 /// name: u32, // null terminated string index
2445 /// - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
2446 /// - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
24372447 /// value: Index,
24382448 /// align: Ref, // if corresponding bit is set
24392449 /// link_section: Ref, // if corresponding bit is set
......@@ -2471,8 +2481,12 @@ pub const Inst = struct {
24712481 /// 0b0X00: whether corresponding decl has an align expression
24722482 /// 0bX000: whether corresponding decl has a linksection expression
24732483 /// 1. decl: { // for every decls_len
2484 /// src_hash: [4]u32, // hash of source bytes
24742485 /// name: u32, // null terminated string index
2486 /// - can be 0 for test decls. always 0 for comptime and usingnamespace decls.
2487 /// - if name == 0 `is_exported` determines which one: 0=comptime,1=usingnamespace
24752488 /// value: Index,
2489 /// - one of: block_inline, block_inline_var
24762490 /// align: Ref, // if corresponding bit is set
24772491 /// link_section: Ref, // if corresponding bit is set
24782492 /// }
......@@ -3553,7 +3567,10 @@ const Writer = struct {
35533567 const has_section = @truncate(u1, cur_bit_bag) != 0;
35543568 cur_bit_bag >>= 1;
35553569
3556 const decl_name = self.code.nullTerminatedString(self.code.extra[extra_index]);
3570 const hash_u32s = self.code.extra[extra_index..][0..4];
3571 extra_index += 4;
3572 const decl_name_index = self.code.extra[extra_index];
3573 const decl_name = self.code.nullTerminatedString(decl_name_index);
35573574 extra_index += 1;
35583575 const decl_index = self.code.extra[extra_index];
35593576 extra_index += 1;
......@@ -3568,24 +3585,33 @@ const Writer = struct {
35683585 break :inst inst;
35693586 };
35703587
3571 const tag = self.code.instructions.items(.tag)[decl_index];
35723588 const pub_str = if (is_pub) "pub " else "";
3573 const export_str = if (is_exported) "export " else "";
3589 const hash_bytes = @bitCast([16]u8, hash_u32s.*);
35743590 try stream.writeByteNTimes(' ', self.indent);
3575 try stream.print("{s}{s}{}", .{
3576 pub_str, export_str, std.zig.fmtId(decl_name),
3577 });
3578 if (align_inst != .none) {
3579 try stream.writeAll(" align(");
3580 try self.writeInstRef(stream, align_inst);
3581 try stream.writeAll(")");
3582 }
3583 if (section_inst != .none) {
3584 try stream.writeAll(" linksection(");
3585 try self.writeInstRef(stream, section_inst);
3586 try stream.writeAll(")");
3591 if (decl_name_index == 0) {
3592 const name = if (is_exported) "usingnamespace" else "comptime";
3593 try stream.writeAll(pub_str);
3594 try stream.writeAll(name);
3595 } else {
3596 const export_str = if (is_exported) "export " else "";
3597 try stream.print("{s}{s}{}", .{
3598 pub_str, export_str, std.zig.fmtId(decl_name),
3599 });
3600 if (align_inst != .none) {
3601 try stream.writeAll(" align(");
3602 try self.writeInstRef(stream, align_inst);
3603 try stream.writeAll(")");
3604 }
3605 if (section_inst != .none) {
3606 try stream.writeAll(" linksection(");
3607 try self.writeInstRef(stream, section_inst);
3608 try stream.writeAll(")");
3609 }
35873610 }
3588 try stream.print(": %{d} = {s}(", .{ decl_index, @tagName(tag) });
3611 const tag = self.code.instructions.items(.tag)[decl_index];
3612 try stream.print(" hash({}): %{d} = {s}(", .{
3613 std.fmt.fmtSliceHexLower(&hash_bytes), decl_index, @tagName(tag),
3614 });
35893615
35903616 const decl_block_inst_data = self.code.instructions.items(.data)[decl_index].pl_node;
35913617 const sub_decl_node_off = decl_block_inst_data.src_node;
......@@ -3939,6 +3965,7 @@ const Writer = struct {
39393965 extra_index += 1;
39403966 try stream.print("lib_name=\"{}\", ", .{std.zig.fmtEscapes(lib_name)});
39413967 }
3968 try self.writeFlag(stream, "test, ", small.is_test);
39423969 const cc: Inst.Ref = if (!small.has_cc) .none else blk: {
39433970 const cc = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]);
39443971 extra_index += 1;