authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-23 16:21:26-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-01-23 16:24:46-07:00
log8a697262099d64b2ff222dc9271493fe3b5f2b76
tree78ea36adce4a5fa5fa8739e38f408cb4f305c6db
parent8c23b81e8640df490e2168b9a623bbb57a7d353e

AstGen: doc comment fixups

* AstGen: use Ast.zig helper methods to avoid copy pasting token counting logic - take advantage of the `first_doc_comment` field we already have for param AST nodes * Add missing ZIR docs

3 files changed, 69 insertions(+), 41 deletions(-)

lib/std/zig/Ast.zig+18
......@@ -2120,6 +2120,14 @@ pub const full = struct {
21202120 section_node: Node.Index,
21212121 init_node: Node.Index,
21222122 };
2123
2124 pub fn firstToken(var_decl: VarDecl) TokenIndex {
2125 return var_decl.visib_token orelse
2126 var_decl.extern_export_token orelse
2127 var_decl.threadlocal_token orelse
2128 var_decl.comptime_token orelse
2129 var_decl.ast.mut_token;
2130 }
21232131 };
21242132
21252133 pub const If = struct {
......@@ -2168,6 +2176,10 @@ pub const full = struct {
21682176 value_expr: Node.Index,
21692177 align_expr: Node.Index,
21702178 };
2179
2180 pub fn firstToken(cf: ContainerField) TokenIndex {
2181 return cf.comptime_token orelse cf.ast.name_token;
2182 }
21712183 };
21722184
21732185 pub const FnProto = struct {
......@@ -2197,6 +2209,12 @@ pub const full = struct {
21972209 type_expr: Node.Index,
21982210 };
21992211
2212 pub fn firstToken(fn_proto: FnProto) TokenIndex {
2213 return fn_proto.visib_token orelse
2214 fn_proto.extern_export_inline_token orelse
2215 fn_proto.ast.fn_token;
2216 }
2217
22002218 /// Abstracts over the fact that anytype and ... are not included
22012219 /// in the params slice, since they are simple identifiers and
22022220 /// not sub-expressions.
src/AstGen.zig+45-39
......@@ -1171,7 +1171,7 @@ fn fnProtoExpr(
11711171 const main_tokens = tree.nodes.items(.main_token);
11721172 const name_token = param.name_token orelse main_tokens[param_type_node];
11731173 const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
1174 const param_inst = try block_scope.addParam(&param_gz, tag, name_token, param_name);
1174 const param_inst = try block_scope.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
11751175 assert(param_inst_expected == param_inst);
11761176 }
11771177 }
......@@ -3237,11 +3237,7 @@ fn fnDecl(
32373237 break :blk token_tags[maybe_inline_token] == .keyword_inline;
32383238 };
32393239
3240 const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1 -
3241 @boolToInt(is_pub) -
3242 @boolToInt(is_export) -
3243 @boolToInt(is_extern) -
3244 @boolToInt(has_inline_keyword)); // TODO subtract noinline too
3240 const doc_comment_index = try astgen.docCommentAsString(fn_proto.firstToken());
32453241
32463242 const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0;
32473243 wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace);
......@@ -3301,7 +3297,7 @@ fn fnDecl(
33013297 const main_tokens = tree.nodes.items(.main_token);
33023298 const name_token = param.name_token orelse main_tokens[param_type_node];
33033299 const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param;
3304 const param_inst = try decl_gz.addParam(&param_gz, tag, name_token, param_name);
3300 const param_inst = try decl_gz.addParam(&param_gz, tag, name_token, param_name, param.first_doc_comment);
33053301 assert(param_inst_expected == param_inst);
33063302 break :param indexToRef(param_inst);
33073303 };
......@@ -3527,11 +3523,7 @@ fn globalVarDecl(
35273523 break :blk lib_name_str.index;
35283524 } else 0;
35293525
3530 const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token -
3531 @boolToInt(is_pub) -
3532 @boolToInt(is_export) -
3533 @boolToInt(lib_name != 0) -
3534 @boolToInt(is_threadlocal));
3526 const doc_comment_index = try astgen.docCommentAsString(var_decl.firstToken());
35353527
35363528 assert(var_decl.comptime_token == null); // handled by parser
35373529
......@@ -3899,7 +3891,7 @@ fn structDeclInner(
38993891 try typeExpr(&block_scope, &namespace.base, member.ast.type_expr);
39003892 wip_members.appendToField(@enumToInt(field_type));
39013893
3902 const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token);
3894 const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
39033895 wip_members.appendToField(doc_comment_index);
39043896
39053897 known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr);
......@@ -4016,7 +4008,7 @@ fn unionDeclInner(
40164008 const field_name = try astgen.identAsString(member.ast.name_token);
40174009 wip_members.appendToField(field_name);
40184010
4019 const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token);
4011 const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
40204012 wip_members.appendToField(doc_comment_index);
40214013
40224014 const have_type = member.ast.type_expr != 0;
......@@ -4300,7 +4292,7 @@ fn containerDecl(
43004292 const field_name = try astgen.identAsString(member.ast.name_token);
43014293 wip_members.appendToField(field_name);
43024294
4303 const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token);
4295 const doc_comment_index = try astgen.docCommentAsString(member.firstToken());
43044296 wip_members.appendToField(doc_comment_index);
43054297
43064298 const have_value = member.ast.value_expr != 0;
......@@ -4533,9 +4525,6 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir
45334525 switch (token_tags[tok_i]) {
45344526 .doc_comment, .comma => {},
45354527 .identifier => {
4536 // TODO: maybe consider not using `docCommentAsString`
4537 // since we're already visiting the first doc comment
4538 // token.
45394528 try astgen.extra.ensureUnusedCapacity(gpa, 2);
45404529 const str_index = try astgen.identAsString(tok_i);
45414530 astgen.extra.appendAssumeCapacity(str_index);
......@@ -8817,38 +8806,52 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 {
88178806 }
88188807}
88198808
8820/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
8809/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`.
88218810/// `end_token` must point at the first token after the last doc coment line.
88228811/// Returns 0 if no doc comment is present.
88238812fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 {
8813 if (end_token == 0) return @as(u32, 0);
8814
8815 const token_tags = astgen.tree.tokens.items(.tag);
8816
8817 var tok = end_token - 1;
8818 while (token_tags[tok] == .doc_comment) {
8819 if (tok == 0) break;
8820 tok -= 1;
8821 } else {
8822 tok += 1;
8823 }
8824 return docCommentAsStringFromFirst(astgen, end_token, tok);
8825}
8826
8827/// end_token must be > the index of the last doc comment.
8828fn docCommentAsStringFromFirst(
8829 astgen: *AstGen,
8830 end_token: Ast.TokenIndex,
8831 start_token: Ast.TokenIndex,
8832) !u32 {
8833 if (start_token == end_token) return 0;
8834
88248835 const gpa = astgen.gpa;
88258836 const string_bytes = &astgen.string_bytes;
88268837 const str_index = @intCast(u32, string_bytes.items.len);
8827 const token_tags = astgen.tree.tokens.items(.tag);
88288838 const token_starts = astgen.tree.tokens.items(.start);
8829
8830 if (end_token == 0) return 0;
8831 const start_token: u32 = blk: {
8832 var tok = end_token - 1;
8833 while (token_tags[tok] == .doc_comment) {
8834 if (tok == 0) break;
8835 tok -= 1;
8836 } else {
8837 tok += 1;
8838 }
8839 break :blk tok;
8840 };
8841 if (start_token == end_token) return 0;
8839 const token_tags = astgen.tree.tokens.items(.tag);
88428840
88438841 const total_bytes = token_starts[end_token] - token_starts[start_token];
88448842 try string_bytes.ensureUnusedCapacity(gpa, total_bytes);
88458843
88468844 var current_token = start_token;
88478845 while (current_token < end_token) : (current_token += 1) {
8848 const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
8849 string_bytes.appendSliceAssumeCapacity(tok_bytes);
8850 if (current_token != end_token - 1) {
8851 string_bytes.appendAssumeCapacity('\n');
8846 switch (token_tags[current_token]) {
8847 .doc_comment => {
8848 const tok_bytes = astgen.tree.tokenSlice(current_token)[3..];
8849 string_bytes.appendSliceAssumeCapacity(tok_bytes);
8850 if (current_token != end_token - 1) {
8851 string_bytes.appendAssumeCapacity('\n');
8852 }
8853 },
8854 else => break,
88528855 }
88538856 }
88548857
......@@ -9714,6 +9717,7 @@ const GenZir = struct {
97149717 /// Absolute token index. This function does the conversion to Decl offset.
97159718 abs_tok_index: Ast.TokenIndex,
97169719 name: u32,
9720 first_doc_comment: ?Ast.TokenIndex,
97179721 ) !Zir.Inst.Index {
97189722 const gpa = gz.astgen.gpa;
97199723 const param_body = param_gz.instructionsSlice();
......@@ -9721,8 +9725,10 @@ const GenZir = struct {
97219725 try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len +
97229726 param_body.len);
97239727
9724 const doc_comment_index = try gz.astgen.docCommentAsString(abs_tok_index -
9725 @boolToInt(tag == .param_comptime));
9728 const doc_comment_index = if (first_doc_comment) |first|
9729 try gz.astgen.docCommentAsStringFromFirst(abs_tok_index, first)
9730 else
9731 0;
97269732
97279733 const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{
97289734 .name = name,
src/Zir.zig+6-2
......@@ -2767,7 +2767,11 @@ pub const Inst = struct {
27672767 };
27682768 };
27692769
2770 /// Trailing: field_name: u32 // for every field: null terminated string index
2770 /// Trailing:
2771 /// { // for every fields_len
2772 /// field_name: u32 // null terminated string index
2773 /// doc_comment: u32 // null terminated string index
2774 /// }
27712775 pub const ErrorSetDecl = struct {
27722776 fields_len: u32,
27732777 };
......@@ -2906,7 +2910,7 @@ pub const Inst = struct {
29062910 pub const Param = struct {
29072911 /// Null-terminated string index.
29082912 name: u32,
2909 /// 0 if no doc comment
2913 /// 0 if no doc comment
29102914 doc_comment: u32,
29112915 /// The body contains the type of the parameter.
29122916 body_len: u32,