From 3010ccfca5f65a8ee0d70559a38d39c2c56f7367 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Thu, 20 Jan 2022 16:08:50 +0100 Subject: [PATCH 1/7] astgen saves decl doc comments in zir The field is saved in `extra` unconditionally for each decl. --- src/AstGen.zig | 65 ++++++++++++++++++++++++++++++++++++++++++++--- src/Module.zig | 2 +- src/Zir.zig | 2 +- src/print_zir.zig | 13 ++++++++++ 4 files changed, 77 insertions(+), 5 deletions(-) diff --git a/src/AstGen.zig b/src/AstGen.zig index 750c7f53a9278f2c7cb33da6f0e582ba6b7f6041..eda06d044130d73b893269b56166542c5db98cad 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -3080,9 +3080,9 @@ const WipMembers = struct { /// struct, union, enum, and opaque decls all use same 4 bits per decl const bits_per_decl = 4; const decls_per_u32 = 32 / bits_per_decl; - /// struct, union, enum, and opaque decls all have maximum size of 10 u32 slots - /// (4 for src_hash + line + name + value + align + link_section + address_space) - const max_decl_size = 10; + /// struct, union, enum, and opaque decls all have maximum size of 11 u32 slots + /// (4 for src_hash + line + name + value + doc_comment + align + link_section + address_space ) + const max_decl_size = 11; pub fn init(gpa: Allocator, payload: *ArrayListUnmanaged(u32), decl_count: u32, field_count: u32, comptime bits_per_field: u32, comptime max_field_size: u32) Allocator.Error!Self { const payload_top = @intCast(u32, payload.items.len); @@ -3193,6 +3193,7 @@ fn fnDecl( // missing function name already happened in scanDecls() const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail; const fn_name_str_index = try astgen.identAsString(fn_name_token); + const doc_comment_index = try docCommentAsString(astgen, fn_name_token - 1); // We insert this at the beginning so that its instruction index marks the // start of the top level declaration. @@ -3445,6 +3446,7 @@ fn fnDecl( } wip_members.appendToDecl(fn_name_str_index); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(doc_comment_index); if (align_inst != .none) { wip_members.appendToDecl(@enumToInt(align_inst)); } @@ -3472,6 +3474,7 @@ fn globalVarDecl( const name_token = var_decl.ast.mut_token + 1; const name_str_index = try astgen.identAsString(name_token); + const doc_comment_index = try docCommentAsString(astgen, var_decl.ast.mut_token); var block_scope: GenZir = .{ .parent = scope, @@ -3594,6 +3597,7 @@ fn globalVarDecl( } wip_members.appendToDecl(name_str_index); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(doc_comment_index); // doc_comment wip if (align_inst != .none) { wip_members.appendToDecl(@enumToInt(align_inst)); } @@ -3648,6 +3652,7 @@ fn comptimeDecl( } wip_members.appendToDecl(0); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(0); // no doc comments on comptime decls } fn usingnamespaceDecl( @@ -3699,6 +3704,7 @@ fn usingnamespaceDecl( } wip_members.appendToDecl(0); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(0); // no doc comments on usingnamespace decls } fn testDecl( @@ -3802,6 +3808,7 @@ fn testDecl( } wip_members.appendToDecl(test_name); wip_members.appendToDecl(block_inst); + wip_members.appendToDecl(0); // no doc comments on test decls } fn structDeclInner( @@ -8784,6 +8791,58 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 { } } +/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. +/// `end_token` must point at the first token after the last doc coment line. +/// Returns 0 if no doc comment is present. +fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 { + const gpa = astgen.gpa; + const string_bytes = &astgen.string_bytes; + const str_index = @intCast(u32, string_bytes.items.len); + const token_tags = astgen.tree.tokens.items(.tag); + const token_starts = astgen.tree.tokens.items(.start); + + if (end_token == 0) return 0; + const start_token: u32 = blk: { + var tok = end_token - 1; + while (token_tags[tok] == .doc_comment) { + if (tok == 0) break; + tok -= 1; + } else { + tok += 1; + } + break :blk tok; + }; + if (start_token == end_token) return 0; + + const total_bytes = token_starts[end_token] - token_starts[start_token]; + try string_bytes.ensureUnusedCapacity(gpa, total_bytes); + + var current_token = start_token; + while (current_token < end_token) : (current_token += 1) { + const tok_bytes = astgen.tree.tokenSlice(current_token)[3..]; + string_bytes.appendSliceAssumeCapacity(tok_bytes); + if (current_token != end_token - 1) { + string_bytes.appendAssumeCapacity('\n'); + } + } + + const key = string_bytes.items[str_index..]; + const gop = try astgen.string_table.getOrPutContextAdapted(gpa, @as([]const u8, key), StringIndexAdapter{ + .bytes = string_bytes, + }, StringIndexContext{ + .bytes = string_bytes, + }); + + if (gop.found_existing) { + string_bytes.shrinkRetainingCapacity(str_index); + return gop.key_ptr.*; + } else { + gop.key_ptr.* = str_index; + try string_bytes.append(gpa, 0); + return str_index; + } +} + const IndexSlice = struct { index: u32, len: u32 }; fn strLitAsString(astgen: *AstGen, str_lit_token: Ast.TokenIndex) !IndexSlice { diff --git a/src/Module.zig b/src/Module.zig index 1cb890b886365b27559d65e2f11137b445bac896..ecfc15a319509f8af19465fa1de7f0004aaf6c70 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4011,7 +4011,7 @@ pub fn scanNamespace( cur_bit_bag >>= 4; const decl_sub_index = extra_index; - extra_index += 7; // src_hash(4) + line(1) + name(1) + value(1) + extra_index += 8; // src_hash(4) + line(1) + name(1) + value(1) + doc_comment(1) extra_index += @truncate(u1, flags >> 2); // Align extra_index += @as(u2, @truncate(u1, flags >> 3)) * 2; // Link section or address space, consists of 2 Refs diff --git a/src/Zir.zig b/src/Zir.zig index 96f85b73df4256f2fda471ee4bf55377f361c5c4..74f85aa95ad40d1677f2e51b8733b1bd4d516d89 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -3001,7 +3001,7 @@ pub const DeclIterator = struct { const sub_index = @intCast(u32, it.extra_index); it.extra_index += 5; // src_hash(4) + line(1) const name = it.zir.nullTerminatedString(it.zir.extra[it.extra_index]); - it.extra_index += 2; // name(1) + value(1) + it.extra_index += 3; // name(1) + value(1) + doc_comment(1) it.extra_index += @truncate(u1, flags >> 2); it.extra_index += @truncate(u1, flags >> 3); diff --git a/src/print_zir.zig b/src/print_zir.zig index 6f580abab57432284c5a8c2148a514bb9268d3c2..7a82d84bbd502b3c34b1b6880dee59a949e82a71 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -1398,6 +1398,9 @@ const Writer = struct { extra_index += 1; const decl_index = self.code.extra[extra_index]; extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + const align_inst: Zir.Inst.Ref = if (!has_align) .none else inst: { const inst = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]); extra_index += 1; @@ -1431,6 +1434,16 @@ const Writer = struct { raw_decl_name; const test_str = if (raw_decl_name.len == 0) "test " else ""; const export_str = if (is_exported) "export " else ""; + + if (doc_comment_index != 0) { + const doc_comment = self.code.nullTerminatedString(doc_comment_index); + var it = std.mem.tokenize(u8, doc_comment, "\n"); + while (it.next()) |doc_line| { + try stream.print("///{s}\n", .{doc_line}); + try stream.writeByteNTimes(' ', self.indent); + } + } + try stream.print("[{d}] {s}{s}{s}{}", .{ sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name), }); -- 2.54.0 From 98fddd1c54171b080eb254527203b6f4e4e2f6ca Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Thu, 20 Jan 2022 19:59:40 +0100 Subject: [PATCH 2/7] add field doc comments to zir Doc comment information is stored in `extra` unconditionally for each field. This commmit covers Structs, Enums, Unions, and ErrSets. --- src/AstGen.zig | 27 +++++++++++++++++++++------ src/Sema.zig | 18 +++++++++++++++--- src/Zir.zig | 7 +++++++ src/print_zir.zig | 47 ++++++++++++++++++++++++++++++++++++----------- 4 files changed, 79 insertions(+), 20 deletions(-) diff --git a/src/AstGen.zig b/src/AstGen.zig index eda06d044130d73b893269b56166542c5db98cad..142806e74cca338d372a6d49aafb502dc67f6fdf 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -3193,7 +3193,7 @@ fn fnDecl( // missing function name already happened in scanDecls() const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail; const fn_name_str_index = try astgen.identAsString(fn_name_token); - const doc_comment_index = try docCommentAsString(astgen, fn_name_token - 1); + const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1); // We insert this at the beginning so that its instruction index marks the // start of the top level declaration. @@ -3474,7 +3474,7 @@ fn globalVarDecl( const name_token = var_decl.ast.mut_token + 1; const name_str_index = try astgen.identAsString(name_token); - const doc_comment_index = try docCommentAsString(astgen, var_decl.ast.mut_token); + const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token); var block_scope: GenZir = .{ .parent = scope, @@ -3864,7 +3864,7 @@ fn structDeclInner( const field_count = @intCast(u32, container_decl.ast.members.len - decl_count); const bits_per_field = 4; - const max_field_size = 4; + const max_field_size = 5; var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size); defer wip_members.deinit(); @@ -3888,6 +3888,9 @@ fn structDeclInner( try typeExpr(&block_scope, &namespace.base, member.ast.type_expr); wip_members.appendToField(@enumToInt(field_type)); + const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + wip_members.appendToField(doc_comment_index); + known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr); const have_align = member.ast.align_expr != 0; @@ -3986,7 +3989,7 @@ fn unionDeclInner( .none; const bits_per_field = 4; - const max_field_size = 4; + const max_field_size = 5; var wip_members = try WipMembers.init(gpa, &astgen.scratch, decl_count, field_count, bits_per_field, max_field_size); defer wip_members.deinit(); @@ -4002,6 +4005,9 @@ fn unionDeclInner( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); + const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + wip_members.appendToField(doc_comment_index); + const have_type = member.ast.type_expr != 0; const have_align = member.ast.align_expr != 0; const have_value = member.ast.value_expr != 0; @@ -4265,7 +4271,7 @@ fn containerDecl( .none; const bits_per_field = 1; - const max_field_size = 2; + const max_field_size = 3; var wip_members = try WipMembers.init(gpa, &astgen.scratch, @intCast(u32, counts.decls), @intCast(u32, counts.total_fields), bits_per_field, max_field_size); defer wip_members.deinit(); @@ -4283,6 +4289,9 @@ fn containerDecl( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); + const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + wip_members.appendToField(doc_comment_index); + const have_value = member.ast.value_expr != 0; wip_members.nextField(bits_per_field, .{have_value}); @@ -4513,8 +4522,14 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir switch (token_tags[tok_i]) { .doc_comment, .comma => {}, .identifier => { + // TODO: maybe consider not using `docCommentAsString` + // since we're already visiting the first doc comment + // token. + try astgen.extra.ensureUnusedCapacity(gpa, 2); const str_index = try astgen.identAsString(tok_i); - try astgen.extra.append(gpa, str_index); + astgen.extra.appendAssumeCapacity(str_index); + const doc_comment_index = try astgen.docCommentAsString(tok_i); + astgen.extra.appendAssumeCapacity(doc_comment_index); fields_len += 1; }, .r_brace => break, diff --git a/src/Sema.zig b/src/Sema.zig index d3802a1784d7b713331aeabbeef56f753e58c07b..6a2085cf3f808c3a1ea36413135c9ff2024f8442 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -1916,6 +1916,9 @@ fn zirEnumDecl( const field_name_zir = sema.code.nullTerminatedString(sema.code.extra[extra_index]); extra_index += 1; + // doc comment + extra_index += 1; + // This string needs to outlive the ZIR code. const field_name = try new_decl_arena_allocator.dupe(u8, field_name_zir); @@ -2103,7 +2106,6 @@ fn zirErrorSetDecl( const inst_data = sema.code.instructions.items(.data)[inst].pl_node; const src = inst_data.src(); const extra = sema.code.extraData(Zir.Inst.ErrorSetDecl, inst_data.payload_index); - const fields = sema.code.extra[extra.end..][0..extra.data.fields_len]; var new_decl_arena = std.heap.ArenaAllocator.init(gpa); errdefer new_decl_arena.deinit(); @@ -2121,8 +2123,12 @@ fn zirErrorSetDecl( errdefer sema.mod.abortAnonDecl(new_decl); var names = Module.ErrorSet.NameMap{}; - try names.ensureUnusedCapacity(new_decl_arena_allocator, fields.len); - for (fields) |str_index| { + try names.ensureUnusedCapacity(new_decl_arena_allocator, extra.data.fields_len); + + var extra_index = @intCast(u32, extra.end); + const extra_index_end = extra_index + (extra.data.fields_len * 2); + while (extra_index < extra_index_end) : (extra_index += 2) { // +2 to skip over doc_string + const str_index = sema.code.extra[extra_index]; const name = try new_decl_arena_allocator.dupe(u8, sema.code.nullTerminatedString(str_index)); // TODO: This check should be performed in AstGen instead. @@ -16313,6 +16319,9 @@ fn semaStructFields( const field_type_ref = @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); extra_index += 1; + // doc_comment + extra_index += 1; + // This string needs to outlive the ZIR code. const field_name = try decl_arena_allocator.dupe(u8, field_name_zir); const field_ty: Type = if (field_type_ref == .none) @@ -16502,6 +16511,9 @@ fn semaUnionFields(mod: *Module, union_obj: *Module.Union) CompileError!void { const field_name_zir = zir.nullTerminatedString(zir.extra[extra_index]); extra_index += 1; + // doc_comment + extra_index += 1; + const field_type_ref: Zir.Inst.Ref = if (has_type) blk: { const field_type_ref = @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); extra_index += 1; diff --git a/src/Zir.zig b/src/Zir.zig index 74f85aa95ad40d1677f2e51b8733b1bd4d516d89..b50a63163f8baf24ae16742479b67655dcfe6def 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -2571,6 +2571,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc comment /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, @@ -2588,6 +2589,7 @@ pub const Inst = struct { /// field_name: u32, /// field_type: Ref, /// - if none, means `anytype`. + /// doc_comment: u32, // 0 if no doc comment /// align: Ref, // if corresponding bit is set /// default_value: Ref, // if corresponding bit is set /// } @@ -2638,6 +2640,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc_comment /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, @@ -2649,6 +2652,7 @@ pub const Inst = struct { /// - the bit is whether corresponding field has an value expression /// 9. fields: { // for every fields_len /// field_name: u32, + /// doc_comment: u32, // 0 if no doc_comment /// value: Ref, // if corresponding bit is set /// } pub const EnumDecl = struct { @@ -2686,6 +2690,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc comment /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, @@ -2701,6 +2706,7 @@ pub const Inst = struct { /// 0bX000: unused /// 9. fields: { // for every fields_len /// field_name: u32, // null terminated string index + /// doc_comment: u32, // 0 if no doc comment /// field_type: Ref, // if corresponding bit is set /// - if none, means `anytype`. /// align: Ref, // if corresponding bit is set @@ -2745,6 +2751,7 @@ pub const Inst = struct { /// - if there is a 0 byte at the position `name` indexes, it indicates /// this is a test decl, and the name starts at `name+1`. /// value: Index, + /// doc_comment: u32, // 0 if no doc comment, /// align: Ref, // if corresponding bit is set /// link_section_or_address_space: { // if corresponding bit is set. /// link_section: Ref, diff --git a/src/print_zir.zig b/src/print_zir.zig index 7a82d84bbd502b3c34b1b6880dee59a949e82a71..9fd6b612310dd5836fbe9b8d3f3af6a371cc8066 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -1207,6 +1207,10 @@ const Writer = struct { extra_index += 1; const field_type = @intToEnum(Zir.Inst.Ref, self.code.extra[extra_index]); extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + + try self.writeDocComment(stream, doc_comment_index); try stream.writeByteNTimes(' ', self.indent); try self.writeFlag(stream, "comptime ", is_comptime); @@ -1332,6 +1336,10 @@ const Writer = struct { const field_name = self.code.nullTerminatedString(self.code.extra[extra_index]); extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + + try self.writeDocComment(stream, doc_comment_index); try stream.writeByteNTimes(' ', self.indent); try stream.print("{}", .{std.zig.fmtId(field_name)}); @@ -1419,12 +1427,13 @@ const Writer = struct { const pub_str = if (is_pub) "pub " else ""; const hash_bytes = @bitCast([16]u8, hash_u32s.*); - try stream.writeByteNTimes(' ', self.indent); if (decl_name_index == 0) { + try stream.writeByteNTimes(' ', self.indent); const name = if (is_exported) "usingnamespace" else "comptime"; try stream.writeAll(pub_str); try stream.writeAll(name); } else if (decl_name_index == 1) { + try stream.writeByteNTimes(' ', self.indent); try stream.writeAll("test"); } else { const raw_decl_name = self.code.nullTerminatedString(decl_name_index); @@ -1435,15 +1444,9 @@ const Writer = struct { const test_str = if (raw_decl_name.len == 0) "test " else ""; const export_str = if (is_exported) "export " else ""; - if (doc_comment_index != 0) { - const doc_comment = self.code.nullTerminatedString(doc_comment_index); - var it = std.mem.tokenize(u8, doc_comment, "\n"); - while (it.next()) |doc_line| { - try stream.print("///{s}\n", .{doc_line}); - try stream.writeByteNTimes(' ', self.indent); - } - } + try self.writeDocComment(stream, doc_comment_index); + try stream.writeByteNTimes(' ', self.indent); try stream.print("[{d}] {s}{s}{s}{}", .{ sub_index, pub_str, test_str, export_str, std.zig.fmtId(decl_name), }); @@ -1569,6 +1572,11 @@ const Writer = struct { const field_name = self.code.nullTerminatedString(self.code.extra[extra_index]); extra_index += 1; + const doc_comment_index = self.code.extra[extra_index]; + extra_index += 1; + + try self.writeDocComment(stream, doc_comment_index); + try stream.writeByteNTimes(' ', self.indent); try stream.print("{}", .{std.zig.fmtId(field_name)}); @@ -1632,17 +1640,23 @@ const Writer = struct { ) !void { const inst_data = self.code.instructions.items(.data)[inst].pl_node; const extra = self.code.extraData(Zir.Inst.ErrorSetDecl, inst_data.payload_index); - const fields = self.code.extra[extra.end..][0..extra.data.fields_len]; try stream.print("{s}, ", .{@tagName(name_strategy)}); try stream.writeAll("{\n"); self.indent += 2; - for (fields) |str_index| { + + var extra_index = @intCast(u32, extra.end); + const extra_index_end = extra_index + (extra.data.fields_len * 2); + while (extra_index < extra_index_end) : (extra_index += 2) { + const str_index = self.code.extra[extra_index]; const name = self.code.nullTerminatedString(str_index); + const doc_comment_index = self.code.extra[extra_index + 1]; + try self.writeDocComment(stream, doc_comment_index); try stream.writeByteNTimes(' ', self.indent); try stream.print("{},\n", .{std.zig.fmtId(name)}); } + self.indent -= 2; try stream.writeByteNTimes(' ', self.indent); try stream.writeAll("}) "); @@ -2134,6 +2148,17 @@ const Writer = struct { } } + fn writeDocComment(self: *Writer, stream: anytype, doc_comment_index: u32) !void { + if (doc_comment_index != 0) { + const doc_comment = self.code.nullTerminatedString(doc_comment_index); + var it = std.mem.tokenize(u8, doc_comment, "\n"); + while (it.next()) |doc_line| { + try stream.writeByteNTimes(' ', self.indent); + try stream.print("///{s}\n", .{doc_line}); + } + } + } + fn writeBody(self: *Writer, stream: anytype, body: []const Zir.Inst.Index) !void { for (body) |inst| { try stream.writeByteNTimes(' ', self.indent); -- 2.54.0 From 1f56ff8343ef07d2d04d4a596b3f03f61230a753 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 21 Jan 2022 17:42:48 +0100 Subject: [PATCH 3/7] add support for more decl attributes in doc comment zir The previous commit that implemented doc comment zir support for decls did not properly account for all the possible attribute keyword combinations (threadlocal, extern, and such). --- src/AstGen.zig | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/AstGen.zig b/src/AstGen.zig index 142806e74cca338d372a6d49aafb502dc67f6fdf..c6b4e6e866b472223dc201381b531f304c92832b 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -3193,7 +3193,6 @@ fn fnDecl( // missing function name already happened in scanDecls() const fn_name_token = fn_proto.name_token orelse return error.AnalysisFail; const fn_name_str_index = try astgen.identAsString(fn_name_token); - const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1); // We insert this at the beginning so that its instruction index marks the // start of the top level declaration. @@ -3237,6 +3236,13 @@ fn fnDecl( const maybe_inline_token = fn_proto.extern_export_inline_token orelse break :blk false; break :blk token_tags[maybe_inline_token] == .keyword_inline; }; + + const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1 - + @boolToInt(is_pub) - + @boolToInt(is_export) - + @boolToInt(is_extern) - + @boolToInt(has_inline_keyword)); // TODO subtract noinline too + const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0; wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace); @@ -3474,7 +3480,6 @@ fn globalVarDecl( const name_token = var_decl.ast.mut_token + 1; const name_str_index = try astgen.identAsString(name_token); - const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token); var block_scope: GenZir = .{ .parent = scope, @@ -3522,6 +3527,12 @@ fn globalVarDecl( break :blk lib_name_str.index; } else 0; + const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token - + @boolToInt(is_pub) - + @boolToInt(is_export) - + @boolToInt(lib_name != 0) - + @boolToInt(is_threadlocal)); + assert(var_decl.comptime_token == null); // handled by parser const var_inst: Zir.Inst.Ref = if (var_decl.ast.init_node != 0) vi: { -- 2.54.0 From ba55e32ef2d8ed5875bff8524fbbb05c6d8afd40 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 21 Jan 2022 18:52:42 +0100 Subject: [PATCH 4/7] add function param doc comment info in zir --- src/AstGen.zig | 4 ++++ src/Zir.zig | 2 ++ src/print_zir.zig | 6 ++++++ 3 files changed, 12 insertions(+) diff --git a/src/AstGen.zig b/src/AstGen.zig index c6b4e6e866b472223dc201381b531f304c92832b..5a05209f4cdf5b506020803396d4f0140a627e50 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -9721,8 +9721,12 @@ const GenZir = struct { try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len + param_body.len); + const doc_comment_index = try gz.astgen.docCommentAsString(abs_tok_index - + @boolToInt(tag == .param_comptime)); + const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{ .name = name, + .doc_comment = doc_comment_index, .body_len = @intCast(u32, param_body.len), }); gz.astgen.extra.appendSliceAssumeCapacity(param_body); diff --git a/src/Zir.zig b/src/Zir.zig index b50a63163f8baf24ae16742479b67655dcfe6def..05c4f2c5fb53838f60689509ebb4c6c7f17785e6 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -2906,6 +2906,8 @@ pub const Inst = struct { pub const Param = struct { /// Null-terminated string index. name: u32, + /// 0 if no doc comment + doc_comment: u32, /// The body contains the type of the parameter. body_len: u32, }; diff --git a/src/print_zir.zig b/src/print_zir.zig index 9fd6b612310dd5836fbe9b8d3f3af6a371cc8066..7ce459568b1f845ce71aa3bbcd596f5ddfa87cee 100644 --- a/src/print_zir.zig +++ b/src/print_zir.zig @@ -785,6 +785,12 @@ const Writer = struct { try stream.print("\"{}\", ", .{ std.zig.fmtEscapes(self.code.nullTerminatedString(extra.data.name)), }); + + if (extra.data.doc_comment != 0) { + try stream.writeAll("\n"); + try self.writeDocComment(stream, extra.data.doc_comment); + try stream.writeByteNTimes(' ', self.indent); + } try self.writeBracedBody(stream, body); try stream.writeAll(") "); try self.writeSrc(stream, inst_data.src()); -- 2.54.0 From d819663543bd2f19bcabb86978adf06931a2dd8b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 21 Jan 2022 14:42:58 -0700 Subject: [PATCH 5/7] disable some broken stuff for stage2 llvm backend on aarch64 --- lib/std/special/compiler_rt.zig | 4 +++- test/behavior/cast_llvm.zig | 4 +++- test/behavior/inttoptr.zig | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/std/special/compiler_rt.zig b/lib/std/special/compiler_rt.zig index 0cd7ca2418b9549b35274c8b42a1aa591295c5bd..24bca128de9693a3b2421c798b5fa0a9e28cc22c 100644 --- a/lib/std/special/compiler_rt.zig +++ b/lib/std/special/compiler_rt.zig @@ -26,7 +26,9 @@ comptime { if (builtin.zig_backend == .stage1) { _ = @import("compiler_rt/atomics.zig"); } - _ = @import("compiler_rt/clear_cache.zig").clear_cache; + if (builtin.zig_backend != .stage2_llvm) { // TODO + _ = @import("compiler_rt/clear_cache.zig").clear_cache; + } const __extenddftf2 = @import("compiler_rt/extendXfYf2.zig").__extenddftf2; @export(__extenddftf2, .{ .name = "__extenddftf2", .linkage = linkage }); diff --git a/test/behavior/cast_llvm.zig b/test/behavior/cast_llvm.zig index 57413df920f50afd1a87f2c176bbb2a869b574bc..1b27d10d593c0462594c4a92ccc3f98756b2e303 100644 --- a/test/behavior/cast_llvm.zig +++ b/test/behavior/cast_llvm.zig @@ -1,8 +1,9 @@ +const builtin = @import("builtin"); const std = @import("std"); const expect = std.testing.expect; const mem = std.mem; const maxInt = std.math.maxInt; -const native_endian = @import("builtin").target.cpu.arch.endian(); +const native_endian = builtin.target.cpu.arch.endian(); test "pointer reinterpret const float to int" { // The hex representation is 0x3fe3333333333303. @@ -46,6 +47,7 @@ fn incrementVoidPtrArray(array: ?*anyopaque, len: usize) void { } test "compile time int to ptr of function" { + if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) return error.SkipZigTest; // TODO try foobar(FUNCTION_CONSTANT); } diff --git a/test/behavior/inttoptr.zig b/test/behavior/inttoptr.zig index 7c07a63b7cefd37d86e5dfad2edc47a04fc41c58..ec26a09699164fac1dada1aa099cca4ba43fd943 100644 --- a/test/behavior/inttoptr.zig +++ b/test/behavior/inttoptr.zig @@ -1,4 +1,7 @@ +const builtin = @import("builtin"); + test "casting random address to function pointer" { + if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) return error.SkipZigTest; // TODO randomAddressToFunction(); comptime randomAddressToFunction(); } -- 2.54.0 From 8c23b81e8640df490e2168b9a623bbb57a7d353e Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Fri, 21 Jan 2022 22:46:12 +0100 Subject: [PATCH 6/7] fix incorrect zir offset loading in sema --- src/Module.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Module.zig b/src/Module.zig index ecfc15a319509f8af19465fa1de7f0004aaf6c70..3892b35aeab65ad242ce5015ed6e3a5611861394 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -558,14 +558,14 @@ pub const Decl = struct { if (!decl.has_align) return .none; assert(decl.zir_decl_index != 0); const zir = decl.getFileScope().zir; - return @intToEnum(Zir.Inst.Ref, zir.extra[decl.zir_decl_index + 7]); + return @intToEnum(Zir.Inst.Ref, zir.extra[decl.zir_decl_index + 8]); } pub fn zirLinksectionRef(decl: Decl) Zir.Inst.Ref { if (!decl.has_linksection_or_addrspace) return .none; assert(decl.zir_decl_index != 0); const zir = decl.getFileScope().zir; - const extra_index = decl.zir_decl_index + 7 + @boolToInt(decl.has_align); + const extra_index = decl.zir_decl_index + 8 + @boolToInt(decl.has_align); return @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); } @@ -573,7 +573,7 @@ pub const Decl = struct { if (!decl.has_linksection_or_addrspace) return .none; assert(decl.zir_decl_index != 0); const zir = decl.getFileScope().zir; - const extra_index = decl.zir_decl_index + 7 + @boolToInt(decl.has_align) + 1; + const extra_index = decl.zir_decl_index + 8 + @boolToInt(decl.has_align) + 1; return @intToEnum(Zir.Inst.Ref, zir.extra[extra_index]); } -- 2.54.0 From 8a697262099d64b2ff222dc9271493fe3b5f2b76 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 23 Jan 2022 16:21:26 -0700 Subject: [PATCH 7/7] AstGen: doc comment fixups * AstGen: use Ast.zig helper methods to avoid copy pasting token counting logic - take advantage of the `first_doc_comment` field we already have for param AST nodes * Add missing ZIR docs --- lib/std/zig/Ast.zig | 18 ++++++++++ src/AstGen.zig | 86 ++++++++++++++++++++++++--------------------- src/Zir.zig | 8 +++-- 3 files changed, 70 insertions(+), 42 deletions(-) diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig index 65772c87a8bad879a4a3fff7d2d33008cf7858e2..96007d564f286747d7f373e434568d88c01c5119 100644 --- a/lib/std/zig/Ast.zig +++ b/lib/std/zig/Ast.zig @@ -2120,6 +2120,14 @@ pub const full = struct { section_node: Node.Index, init_node: Node.Index, }; + + pub fn firstToken(var_decl: VarDecl) TokenIndex { + return var_decl.visib_token orelse + var_decl.extern_export_token orelse + var_decl.threadlocal_token orelse + var_decl.comptime_token orelse + var_decl.ast.mut_token; + } }; pub const If = struct { @@ -2168,6 +2176,10 @@ pub const full = struct { value_expr: Node.Index, align_expr: Node.Index, }; + + pub fn firstToken(cf: ContainerField) TokenIndex { + return cf.comptime_token orelse cf.ast.name_token; + } }; pub const FnProto = struct { @@ -2197,6 +2209,12 @@ pub const full = struct { type_expr: Node.Index, }; + pub fn firstToken(fn_proto: FnProto) TokenIndex { + return fn_proto.visib_token orelse + fn_proto.extern_export_inline_token orelse + fn_proto.ast.fn_token; + } + /// Abstracts over the fact that anytype and ... are not included /// in the params slice, since they are simple identifiers and /// not sub-expressions. diff --git a/src/AstGen.zig b/src/AstGen.zig index 5a05209f4cdf5b506020803396d4f0140a627e50..114cd5f505ca55ffa9b7adfccfe91f17cbd50e70 100644 --- a/src/AstGen.zig +++ b/src/AstGen.zig @@ -1171,7 +1171,7 @@ fn fnProtoExpr( const main_tokens = tree.nodes.items(.main_token); const name_token = param.name_token orelse main_tokens[param_type_node]; const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param; - const param_inst = try block_scope.addParam(¶m_gz, tag, name_token, param_name); + const param_inst = try block_scope.addParam(¶m_gz, tag, name_token, param_name, param.first_doc_comment); assert(param_inst_expected == param_inst); } } @@ -3237,11 +3237,7 @@ fn fnDecl( break :blk token_tags[maybe_inline_token] == .keyword_inline; }; - const doc_comment_index = try astgen.docCommentAsString(fn_name_token - 1 - - @boolToInt(is_pub) - - @boolToInt(is_export) - - @boolToInt(is_extern) - - @boolToInt(has_inline_keyword)); // TODO subtract noinline too + const doc_comment_index = try astgen.docCommentAsString(fn_proto.firstToken()); const has_section_or_addrspace = fn_proto.ast.section_expr != 0 or fn_proto.ast.addrspace_expr != 0; wip_members.nextDecl(is_pub, is_export, fn_proto.ast.align_expr != 0, has_section_or_addrspace); @@ -3301,7 +3297,7 @@ fn fnDecl( const main_tokens = tree.nodes.items(.main_token); const name_token = param.name_token orelse main_tokens[param_type_node]; const tag: Zir.Inst.Tag = if (is_comptime) .param_comptime else .param; - const param_inst = try decl_gz.addParam(¶m_gz, tag, name_token, param_name); + const param_inst = try decl_gz.addParam(¶m_gz, tag, name_token, param_name, param.first_doc_comment); assert(param_inst_expected == param_inst); break :param indexToRef(param_inst); }; @@ -3527,11 +3523,7 @@ fn globalVarDecl( break :blk lib_name_str.index; } else 0; - const doc_comment_index = try astgen.docCommentAsString(var_decl.ast.mut_token - - @boolToInt(is_pub) - - @boolToInt(is_export) - - @boolToInt(lib_name != 0) - - @boolToInt(is_threadlocal)); + const doc_comment_index = try astgen.docCommentAsString(var_decl.firstToken()); assert(var_decl.comptime_token == null); // handled by parser @@ -3899,7 +3891,7 @@ fn structDeclInner( try typeExpr(&block_scope, &namespace.base, member.ast.type_expr); wip_members.appendToField(@enumToInt(field_type)); - const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + const doc_comment_index = try astgen.docCommentAsString(member.firstToken()); wip_members.appendToField(doc_comment_index); known_has_bits = known_has_bits or nodeImpliesRuntimeBits(tree, member.ast.type_expr); @@ -4016,7 +4008,7 @@ fn unionDeclInner( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); - const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + const doc_comment_index = try astgen.docCommentAsString(member.firstToken()); wip_members.appendToField(doc_comment_index); const have_type = member.ast.type_expr != 0; @@ -4300,7 +4292,7 @@ fn containerDecl( const field_name = try astgen.identAsString(member.ast.name_token); wip_members.appendToField(field_name); - const doc_comment_index = try astgen.docCommentAsString(member.ast.name_token); + const doc_comment_index = try astgen.docCommentAsString(member.firstToken()); wip_members.appendToField(doc_comment_index); const have_value = member.ast.value_expr != 0; @@ -4533,9 +4525,6 @@ fn errorSetDecl(gz: *GenZir, rl: ResultLoc, node: Ast.Node.Index) InnerError!Zir switch (token_tags[tok_i]) { .doc_comment, .comma => {}, .identifier => { - // TODO: maybe consider not using `docCommentAsString` - // since we're already visiting the first doc comment - // token. try astgen.extra.ensureUnusedCapacity(gpa, 2); const str_index = try astgen.identAsString(tok_i); astgen.extra.appendAssumeCapacity(str_index); @@ -8817,38 +8806,52 @@ fn identAsString(astgen: *AstGen, ident_token: Ast.TokenIndex) !u32 { } } -/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. +/// Adds a doc comment block to `string_bytes` by walking backwards from `end_token`. /// `end_token` must point at the first token after the last doc coment line. /// Returns 0 if no doc comment is present. fn docCommentAsString(astgen: *AstGen, end_token: Ast.TokenIndex) !u32 { - const gpa = astgen.gpa; - const string_bytes = &astgen.string_bytes; - const str_index = @intCast(u32, string_bytes.items.len); + if (end_token == 0) return @as(u32, 0); + const token_tags = astgen.tree.tokens.items(.tag); - const token_starts = astgen.tree.tokens.items(.start); - if (end_token == 0) return 0; - const start_token: u32 = blk: { - var tok = end_token - 1; - while (token_tags[tok] == .doc_comment) { - if (tok == 0) break; - tok -= 1; - } else { - tok += 1; - } - break :blk tok; - }; + var tok = end_token - 1; + while (token_tags[tok] == .doc_comment) { + if (tok == 0) break; + tok -= 1; + } else { + tok += 1; + } + return docCommentAsStringFromFirst(astgen, end_token, tok); +} + +/// end_token must be > the index of the last doc comment. +fn docCommentAsStringFromFirst( + astgen: *AstGen, + end_token: Ast.TokenIndex, + start_token: Ast.TokenIndex, +) !u32 { if (start_token == end_token) return 0; + const gpa = astgen.gpa; + const string_bytes = &astgen.string_bytes; + const str_index = @intCast(u32, string_bytes.items.len); + const token_starts = astgen.tree.tokens.items(.start); + const token_tags = astgen.tree.tokens.items(.tag); + const total_bytes = token_starts[end_token] - token_starts[start_token]; try string_bytes.ensureUnusedCapacity(gpa, total_bytes); var current_token = start_token; while (current_token < end_token) : (current_token += 1) { - const tok_bytes = astgen.tree.tokenSlice(current_token)[3..]; - string_bytes.appendSliceAssumeCapacity(tok_bytes); - if (current_token != end_token - 1) { - string_bytes.appendAssumeCapacity('\n'); + switch (token_tags[current_token]) { + .doc_comment => { + const tok_bytes = astgen.tree.tokenSlice(current_token)[3..]; + string_bytes.appendSliceAssumeCapacity(tok_bytes); + if (current_token != end_token - 1) { + string_bytes.appendAssumeCapacity('\n'); + } + }, + else => break, } } @@ -9714,6 +9717,7 @@ const GenZir = struct { /// Absolute token index. This function does the conversion to Decl offset. abs_tok_index: Ast.TokenIndex, name: u32, + first_doc_comment: ?Ast.TokenIndex, ) !Zir.Inst.Index { const gpa = gz.astgen.gpa; const param_body = param_gz.instructionsSlice(); @@ -9721,8 +9725,10 @@ const GenZir = struct { try gz.astgen.extra.ensureUnusedCapacity(gpa, @typeInfo(Zir.Inst.Param).Struct.fields.len + param_body.len); - const doc_comment_index = try gz.astgen.docCommentAsString(abs_tok_index - - @boolToInt(tag == .param_comptime)); + const doc_comment_index = if (first_doc_comment) |first| + try gz.astgen.docCommentAsStringFromFirst(abs_tok_index, first) + else + 0; const payload_index = gz.astgen.addExtraAssumeCapacity(Zir.Inst.Param{ .name = name, diff --git a/src/Zir.zig b/src/Zir.zig index 05c4f2c5fb53838f60689509ebb4c6c7f17785e6..f38cb2d5f590c62f52a40da7567055a55031c82e 100644 --- a/src/Zir.zig +++ b/src/Zir.zig @@ -2767,7 +2767,11 @@ pub const Inst = struct { }; }; - /// Trailing: field_name: u32 // for every field: null terminated string index + /// Trailing: + /// { // for every fields_len + /// field_name: u32 // null terminated string index + /// doc_comment: u32 // null terminated string index + /// } pub const ErrorSetDecl = struct { fields_len: u32, }; @@ -2906,7 +2910,7 @@ pub const Inst = struct { pub const Param = struct { /// Null-terminated string index. name: u32, - /// 0 if no doc comment + /// 0 if no doc comment doc_comment: u32, /// The body contains the type of the parameter. body_len: u32, -- 2.54.0