authorgravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2022-01-20 16:08:50+01:00
committergravatar for kappaloris@gmail.comLoris Cro <kappaloris@gmail.com> 2022-01-21 22:00:56+01:00
log3010ccfca5f65a8ee0d70559a38d39c2c56f7367
tree1d268df600e14cb37fdd2b5f601bf33390b3ef49
parentc9ae24503dc8da2e59f46619695bf4eb863fb3ac

astgen saves decl doc comments in zir

The field is saved in `extra` unconditionally for each decl.

4 files changed, 77 insertions(+), 5 deletions(-)

src/AstGen.zig+62-3
...@@ -3080,9 +3080,9 @@ const WipMembers = struct {...@@ -3080,9 +3080,9 @@ const WipMembers = struct {
3080 /// struct, union, enum, and opaque decls all use same 4 bits per decl3080 /// struct, union, enum, and opaque decls all use same 4 bits per decl
3081 const bits_per_decl = 4;3081 const bits_per_decl = 4;
3082 const decls_per_u32 = 32 / bits_per_decl;3082 const decls_per_u32 = 32 / bits_per_decl;
3083 /// struct, union, enum, and opaque decls all have maximum size of 10 u32 slots3083 /// struct, union, enum, and opaque decls all have maximum size of 11 u32 slots
3084 /// (4 for src_hash + line + name + value + align + link_section + address_space)3084 /// (4 for src_hash + line + name + value + doc_comment + align + link_section + address_space )
3085 const max_decl_size = 10;3085 const max_decl_size = 11;
30863086
3087 pub fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self {3087 pub fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self {
3088 const payload_top = @intCast(u32, payload.items.len);3088 const payload_top = @intCast(u32, payload.items.len);
...@@ -3193,6 +3193,7 @@ fn fnDecl(...@@ -3193,6 +3193,7 @@ fn fnDecl(
3193 // missing function name already happened in scanDecls()3193 // missing function name already happened in scanDecls()
3194 const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail;3194 const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail;
3195 const fn_name_str_index = try astgen.identAsString(fn_name_token);3195 const fn_name_str_index = try astgen.identAsString(fn_name_token);
3196 const doc_comment_index = try docCommentAsString(astgen, fn_name_token - 1);
31963197
3197 // We insert this at the beginning so that its instruction index marks the3198 // We insert this at the beginning so that its instruction index marks the
3198 // start of the top level declaration.3199 // start of the top level declaration.
...@@ -3445,6 +3446,7 @@ fn fnDecl(...@@ -3445,6 +3446,7 @@ fn fnDecl(
3445 }3446 }
3446 wip_members.appendToDecl(fn_name_str_index);3447 wip_members.appendToDecl(fn_name_str_index);
3447 wip_members.appendToDecl(block_inst);3448 wip_members.appendToDecl(block_inst);
3449 wip_members.appendToDecl(doc_comment_index);
3448 if (align_inst != .none) {3450 if (align_inst != .none) {
3449 wip_members.appendToDecl(@enumToInt(align_inst));3451 wip_members.appendToDecl(@enumToInt(align_inst));
3450 }3452 }
...@@ -3472,6 +3474,7 @@ fn globalVarDecl(...@@ -3472,6 +3474,7 @@ fn globalVarDecl(
34723474
3473 const name_token = var_decl.ast.mut_token + 1;3475 const name_token = var_decl.ast.mut_token + 1;
3474 const name_str_index = try astgen.identAsString(name_token);3476 const name_str_index = try astgen.identAsString(name_token);
3477 const doc_comment_index = try docCommentAsString(astgen, var_decl.ast.mut_token);
34753478
3476 var block_scope: GenZir = .{3479 var block_scope: GenZir = .{
3477 .parent = scope,3480 .parent = scope,
...@@ -3594,6 +3597,7 @@ fn globalVarDecl(...@@ -3594,6 +3597,7 @@ fn globalVarDecl(
3594 }3597 }
3595 wip_members.appendToDecl(name_str_index);3598 wip_members.appendToDecl(name_str_index);
3596 wip_members.appendToDecl(block_inst);3599 wip_members.appendToDecl(block_inst);
3600 wip_members.appendToDecl(doc_comment_index); // doc_comment wip
3597 if (align_inst != .none) {3601 if (align_inst != .none) {
3598 wip_members.appendToDecl(@enumToInt(align_inst));3602 wip_members.appendToDecl(@enumToInt(align_inst));
3599 }3603 }
...@@ -3648,6 +3652,7 @@ fn comptimeDecl(...@@ -3648,6 +3652,7 @@ fn comptimeDecl(
3648 }3652 }
3649 wip_members.appendToDecl(0);3653 wip_members.appendToDecl(0);
3650 wip_members.appendToDecl(block_inst);3654 wip_members.appendToDecl(block_inst);
3655 wip_members.appendToDecl(0); // no doc comments on comptime decls
3651}3656}
36523657
3653fn usingnamespaceDecl(3658fn usingnamespaceDecl(
...@@ -3699,6 +3704,7 @@ fn usingnamespaceDecl(...@@ -3699,6 +3704,7 @@ fn usingnamespaceDecl(
3699 }3704 }
3700 wip_members.appendToDecl(0);3705 wip_members.appendToDecl(0);
3701 wip_members.appendToDecl(block_inst);3706 wip_members.appendToDecl(block_inst);
3707 wip_members.appendToDecl(0); // no doc comments on usingnamespace decls
3702}3708}
37033709
3704fn testDecl(3710fn testDecl(
...@@ -3802,6 +3808,7 @@ fn testDecl(...@@ -3802,6 +3808,7 @@ fn testDecl(
3802 }3808 }
3803 wip_members.appendToDecl(test_name);3809 wip_members.appendToDecl(test_name);
3804 wip_members.appendToDecl(block_inst);3810 wip_members.appendToDecl(block_inst);
3811 wip_members.appendToDecl(0); // no doc comments on test decls
3805}3812}
38063813
3807fn structDeclInner(3814fn structDeclInner(
...@@ -8784,6 +8791,58 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 {...@@ -8784,6 +8791,58 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 {
8784 }8791 }
8785}8792}
87868793
8794/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
8795/// `end_token` must point at the first token after the last doc coment line.
8796/// Returns 0 if no doc comment is present.
8797fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 {
8798 const gpa = astgen.gpa;
8799 const string_bytes = &astgen.string_bytes;
8800 const str_index = @intCast(u32, string_bytes.items.len);
8801 const token_tags = astgen.tree.tokens.items(.tag);
8802 const token_starts = astgen.tree.tokens.items(.start);
8803
8804 if (end_token == 0) return 0;
8805 const start_token: u32 = blk: {
8806 var tok = end_token - 1;
8807 while (token_tags[tok] == .doc_comment) {
8808 if (tok == 0) break;
8809 tok -= 1;
8810 } else {
8811 tok += 1;
8812 }
8813 break :blk tok;
8814 };
8815 if (start_token == end_token) return 0;
8816
8817 const total_bytes = token_starts[end_token] - token_starts[start_token];
8818 try string_bytes.ensureUnusedCapacity(gpa, total_bytes);
8819
8820 var current_token = start_token;
8821 while (current_token < end_token) : (current_token += 1) {
8822 const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
8823 string_bytes.appendSliceAssumeCapacity(tok_bytes);
8824 if (current_token != end_token - 1) {
8825 string_bytes.appendAssumeCapacity('\n');
8826 }
8827 }
8828
8829 const key = string_bytes.items[str_index..];
8830 const gop = try astgen.string_table.getOrPutContextAdapted(gpa, @as([]const u8, key), StringIndexAdapter{
8831 .bytes = string_bytes,
8832 }, StringIndexContext{
8833 .bytes = string_bytes,
8834 });
8835
8836 if (gop.found_existing) {
8837 string_bytes.shrinkRetainingCapacity(str_index);
8838 return gop.key_ptr.*;
8839 } else {
8840 gop.key_ptr.* = str_index;
8841 try string_bytes.append(gpa, 0);
8842 return str_index;
8843 }
8844}
8845
8787const IndexSlice = struct { index: u32, len: u32 };8846const IndexSlice = struct { index: u32, len: u32 };
87888847
8789fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {8848fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice {
src/Module.zig+1-1
...@@ -4011,7 +4011,7 @@ pub fn scanNamespace(...@@ -4011,7 +4011,7 @@ pub fn scanNamespace(
4011 cur_bit_bag >>= 4;4011 cur_bit_bag >>= 4;
40124012
4013 const decl_sub_index = extra_index;4013 const decl_sub_index = extra_index;
4014 extra_index += 7; // src_hash(4) + line(1) + name(1) + value(1)4014 extra_index += 8; // src_hash(4) + line(1) + name(1) + value(1) + doc_comment(1)
4015 extra_index += @truncate(u1, flags >> 2); // Align4015 extra_index += @truncate(u1, flags >> 2); // Align
4016 extra_index += @as(u2, @truncate(u1, flags >> 3)) * 2; // Link section or address space, consists of 2 Refs4016 extra_index += @as(u2, @truncate(u1, flags >> 3)) * 2; // Link section or address space, consists of 2 Refs
40174017
src/Zir.zig+1-1
...@@ -3001,7 +3001,7 @@ pub const DeclIterator = struct {...@@ -3001,7 +3001,7 @@ pub const DeclIterator = struct {
3001 const sub_index = @intCast(u32, it.extra_index);3001 const sub_index = @intCast(u32, it.extra_index);
3002 it.extra_index += 5; // src_hash(4) + line(1)3002 it.extra_index += 5; // src_hash(4) + line(1)
3003 const name = it.zir.nullTerminatedString(it.zir.extra[it.extra_index]);3003 const name = it.zir.nullTerminatedString(it.zir.extra[it.extra_index]);
3004 it.extra_index += 2; // name(1) + value(1)3004 it.extra_index += 3; // name(1) + value(1) + doc_comment(1)
3005 it.extra_index += @truncate(u1, flags >> 2);3005 it.extra_index += @truncate(u1, flags >> 2);
3006 it.extra_index += @truncate(u1, flags >> 3);3006 it.extra_index += @truncate(u1, flags >> 3);
30073007
src/print_zir.zig+13
...@@ -1398,6 +1398,9 @@ const Writer = struct {...@@ -1398,6 +1398,9 @@ const Writer = struct {
1398 extra_index += 1;1398 extra_index += 1;
1399 const decl_index = self.code.extra[extra_index];1399 const decl_index = self.code.extra[extra_index];
1400 extra_index += 1;1400 extra_index += 1;
1401 const doc_comment_index = self.code.extra[extra_index];
1402 extra_index += 1;
1403
1401 const align_inst: Zir.Inst.Ref = if (!has_align) .none else inst: {1404 const align_inst: Zir.Inst.Ref = if (!has_align) .none else inst: {
1402 const inst = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]);1405 const inst = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]);
1403 extra_index += 1;1406 extra_index += 1;
...@@ -1431,6 +1434,16 @@ const Writer = struct {...@@ -1431,6 +1434,16 @@ const Writer = struct {
1431 raw_decl_name;1434 raw_decl_name;
1432 const test_str = if (raw_decl_name.len == 0) "test " else "";1435 const test_str = if (raw_decl_name.len == 0) "test " else "";
1433 const export_str = if (is_exported) "export " else "";1436 const export_str = if (is_exported) "export " else "";
1437
1438 if (doc_comment_index != 0) {
1439 const doc_comment = self.code.nullTerminatedString(doc_comment_index);
1440 var it = std.mem.tokenize(u8, doc_comment, "\n");
1441 while (it.next()) |doc_line| {
1442 try stream.print("///{s}\n", .{doc_line});
1443 try stream.writeByteNTimes(' ', self.indent);
1444 }
1445 }
1446
1434 try stream.print("[{d}] {s}{s}{s}{}", .{1447 try stream.print("[{d}] {s}{s}{s}{}", .{
1435 sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name),1448 sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name),
1436 });1449 });