authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-22 00:28:07-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-22 00:28:59-04:00
log295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c
tree96ca941e76281f9868057db1b496856adc13e256
parent1dac9e71b525cb14eb358eaf0bc45b1554f5bf55

stage2 parser: don't append doc comments to the list

The DocComment AST node now only points to the first doc comment token. API users are expected to iterate over the following tokens directly. After this commit there are no more linked lists in use in the self-hosted AST API. Performance impact is negligible. Memory usage slightly reduced.

3 files changed, 60 insertions(+), 54 deletions(-)

lib/std/zig/ast.zig+8-9
...@@ -1,7 +1,6 @@...@@ -1,7 +1,6 @@
1const std = @import("../std.zig");1const std = @import("../std.zig");
2const assert = std.debug.assert;2const assert = std.debug.assert;
3const testing = std.testing;3const testing = std.testing;
4const LinkedList = std.SinglyLinkedList;
5const mem = std.mem;4const mem = std.mem;
6const Token = std.zig.Token;5const Token = std.zig.Token;
76
...@@ -3013,9 +3012,10 @@ pub const Node = struct {...@@ -3013,9 +3012,10 @@ pub const Node = struct {
30133012
3014 pub const DocComment = struct {3013 pub const DocComment = struct {
3015 base: Node = Node{ .id = .DocComment },3014 base: Node = Node{ .id = .DocComment },
3016 lines: LineList,3015 /// Points to the first doc comment token. API users are expected to iterate over the
30173016 /// tokens array, looking for more doc comments, ignoring line comments, and stopping
3018 pub const LineList = LinkedList(TokenIndex);3017 /// at the first other token.
3018 first_line: TokenIndex,
30193019
3020 pub fn iterate(self: *const DocComment) Node.Iterator {3020 pub fn iterate(self: *const DocComment) Node.Iterator {
3021 return .{ .parent_node = &self.base, .index = 0 };3021 return .{ .parent_node = &self.base, .index = 0 };
...@@ -3026,14 +3026,13 @@ pub const Node = struct {...@@ -3026,14 +3026,13 @@ pub const Node = struct {
3026 }3026 }
30273027
3028 pub fn firstToken(self: *const DocComment) TokenIndex {3028 pub fn firstToken(self: *const DocComment) TokenIndex {
3029 return self.lines.first.?.data;3029 return self.first_line;
3030 }3030 }
30313031
3032 /// Returns the first doc comment line. Be careful, this may not be the desired behavior,
3033 /// which would require the tokens array.
3032 pub fn lastToken(self: *const DocComment) TokenIndex {3034 pub fn lastToken(self: *const DocComment) TokenIndex {
3033 var node = self.lines.first.?;3035 return self.first_line;
3034 while (true) {
3035 node = node.next orelse return node.data;
3036 }
3037 }3036 }
3038 };3037 };
30393038
lib/std/zig/parse.zig+15-30
...@@ -58,6 +58,8 @@ const Parser = struct {...@@ -58,6 +58,8 @@ const Parser = struct {
58 arena: std.heap.ArenaAllocator,58 arena: std.heap.ArenaAllocator,
59 gpa: *Allocator,59 gpa: *Allocator,
60 source: []const u8,60 source: []const u8,
61 /// TODO: Optimization idea: have this be several arrays of the token fields rather
62 /// than an array of structs.
61 tokens: []const Token,63 tokens: []const Token,
62 tok_i: TokenIndex,64 tok_i: TokenIndex,
63 errors: std.ArrayListUnmanaged(AstError),65 errors: std.ArrayListUnmanaged(AstError),
...@@ -367,20 +369,13 @@ const Parser = struct {...@@ -367,20 +369,13 @@ const Parser = struct {
367369
368 /// Eat a multiline container doc comment370 /// Eat a multiline container doc comment
369 fn parseContainerDocComments(p: *Parser) !?*Node {371 fn parseContainerDocComments(p: *Parser) !?*Node {
370 var lines = Node.DocComment.LineList{};372 if (p.eatToken(.ContainerDocComment)) |first_line| {
371 var lines_it: *?*Node.DocComment.LineList.Node = &lines.first;373 while (p.eatToken(.ContainerDocComment)) |_| {}
372374 const node = try p.arena.allocator.create(Node.DocComment);
373 while (p.eatToken(.ContainerDocComment)) |line| {375 node.* = .{ .first_line = first_line };
374 lines_it = try p.llpush(TokenIndex, lines_it, line);376 return &node.base;
375 }377 }
376378 return null;
377 if (lines.first == null) return null;
378
379 const node = try p.arena.allocator.create(Node.DocComment);
380 node.* = .{
381 .lines = lines,
382 };
383 return &node.base;
384 }379 }
385380
386 /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block381 /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block
...@@ -3210,20 +3205,13 @@ const Parser = struct {...@@ -3210,20 +3205,13 @@ const Parser = struct {
32103205
3211 /// Eat a multiline doc comment3206 /// Eat a multiline doc comment
3212 fn parseDocComment(p: *Parser) !?*Node.DocComment {3207 fn parseDocComment(p: *Parser) !?*Node.DocComment {
3213 var lines = Node.DocComment.LineList{};3208 if (p.eatToken(.DocComment)) |first_line| {
3214 var lines_it = &lines.first;3209 while (p.eatToken(.DocComment)) |_| {}
32153210 const node = try p.arena.allocator.create(Node.DocComment);
3216 while (p.eatToken(.DocComment)) |line| {3211 node.* = .{ .first_line = first_line };
3217 lines_it = try p.llpush(TokenIndex, lines_it, line);3212 return node;
3218 }3213 }
32193214 return null;
3220 if (lines.first == null) return null;
3221
3222 const node = try p.arena.allocator.create(Node.DocComment);
3223 node.* = .{
3224 .lines = lines,
3225 };
3226 return node;
3227 }3215 }
32283216
3229 fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {3217 fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool {
...@@ -3234,11 +3222,8 @@ const Parser = struct {...@@ -3234,11 +3222,8 @@ const Parser = struct {
3234 fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment {3222 fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment {
3235 const comment_token = p.eatToken(.DocComment) orelse return null;3223 const comment_token = p.eatToken(.DocComment) orelse return null;
3236 if (p.tokensOnSameLine(after_token, comment_token)) {3224 if (p.tokensOnSameLine(after_token, comment_token)) {
3237 var lines = Node.DocComment.LineList{};
3238 _ = try p.llpush(TokenIndex, &lines.first, comment_token);
3239
3240 const node = try p.arena.allocator.create(Node.DocComment);3225 const node = try p.arena.allocator.create(Node.DocComment);
3241 node.* = .{ .lines = lines };3226 node.* = .{ .first_line = comment_token };
3242 return node;3227 return node;
3243 }3228 }
3244 p.putBackToken(comment_token);3229 p.putBackToken(comment_token);
lib/std/zig/render.zig+37-15
...@@ -327,11 +327,18 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,...@@ -327,11 +327,18 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree,
327327
328 .DocComment => {328 .DocComment => {
329 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);329 const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl);
330 var it = comment.lines.first;330 const kind = tree.tokens[comment.first_line].id;
331 while (it) |node| : (it = node.next) {331 try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline);
332 try renderToken(tree, stream, node.data, indent, start_col, .Newline);332 var tok_i = comment.first_line + 1;
333 if (node.next != null) {333 while (true) : (tok_i += 1) {
334 const tok_id = tree.tokens[tok_i].id;
335 if (tok_id == kind) {
334 try stream.writeByteNTimes(' ', indent);336 try stream.writeByteNTimes(' ', indent);
337 try renderToken(tree, stream, tok_i, indent, start_col, .Newline);
338 } else if (tok_id == .LineComment) {
339 continue;
340 } else {
341 break;
335 }342 }
336 }343 }
337 },344 },
...@@ -2428,17 +2435,32 @@ fn renderDocComments(...@@ -2428,17 +2435,32 @@ fn renderDocComments(
2428 start_col: *usize,2435 start_col: *usize,
2429) (@TypeOf(stream).Error || Error)!void {2436) (@TypeOf(stream).Error || Error)!void {
2430 const comment = node.doc_comments orelse return;2437 const comment = node.doc_comments orelse return;
2431 var it = comment.lines.first;2438 return renderDocCommentsToken(tree, stream, comment, node.firstToken(), indent, start_col);
2432 const first_token = node.firstToken();2439}
2433 while (it) |line_token_index_node| : (it = line_token_index_node.next) {2440
2434 const line_token_index = line_token_index_node.data;2441fn renderDocCommentsToken(
2435 if (line_token_index < first_token) {2442 tree: *ast.Tree,
2436 try renderToken(tree, stream, line_token_index, indent, start_col, Space.Newline);2443 stream: var,
2437 try stream.writeByteNTimes(' ', indent);2444 comment: *ast.Node.DocComment,
2438 } else {2445 first_token: ast.TokenIndex,
2439 try renderToken(tree, stream, line_token_index, indent, start_col, Space.NoComment);2446 indent: usize,
2440 try stream.writeAll("\n");2447 start_col: *usize,
2441 try stream.writeByteNTimes(' ', indent);2448) (@TypeOf(stream).Error || Error)!void {
2449 var tok_i = comment.first_line;
2450 while (true) : (tok_i += 1) {
2451 switch (tree.tokens[tok_i].id) {
2452 .DocComment, .ContainerDocComment => {
2453 if (comment.first_line < first_token) {
2454 try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline);
2455 try stream.writeByteNTimes(' ', indent);
2456 } else {
2457 try renderToken(tree, stream, tok_i, indent, start_col, Space.NoComment);
2458 try stream.writeAll("\n");
2459 try stream.writeByteNTimes(' ', indent);
2460 }
2461 },
2462 .LineComment => continue,
2463 else => break,
2442 }2464 }
2443 }2465 }
2444}2466}