authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-14 23:39:20-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-15 12:30:29-05:00
log5f5880979e8c5a7cfa4efdabad6358ceb89cc0e7
treea35163af2182869eb56dd1fbc031caf39be4d4d9
parentcc26148ba776f713bb81b5ac06fc646eb323e6dc

zig fmt supports simple line comments

3 files changed, 95 insertions(+), 24 deletions(-)

std/zig/ast.zig+22
......@@ -6,6 +6,7 @@ const mem = std.mem;
66
77pub const Node = struct {
88 id: Id,
9 comment: ?&NodeLineComment,
910
1011 pub const Id = enum {
1112 Root,
......@@ -20,6 +21,7 @@ pub const Node = struct {
2021 FloatLiteral,
2122 StringLiteral,
2223 BuiltinCall,
24 LineComment,
2325 };
2426
2527 pub fn iterate(base: &Node, index: usize) ?&Node {
......@@ -36,6 +38,7 @@ pub const Node = struct {
3638 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
3739 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
3840 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
41 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).iterate(index),
3942 };
4043 }
4144
......@@ -53,6 +56,7 @@ pub const Node = struct {
5356 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
5457 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
5558 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
59 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).firstToken(),
5660 };
5761 }
5862
......@@ -70,6 +74,7 @@ pub const Node = struct {
7074 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
7175 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
7276 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
77 Id.LineComment => @fieldParentPtr(NodeLineComment, "base", base).lastToken(),
7378 };
7479 }
7580};
......@@ -454,3 +459,20 @@ pub const NodeStringLiteral = struct {
454459 return self.token;
455460 }
456461};
462
463pub const NodeLineComment = struct {
464 base: Node,
465 lines: ArrayList(Token),
466
467 pub fn iterate(self: &NodeLineComment, index: usize) ?&Node {
468 return null;
469 }
470
471 pub fn firstToken(self: &NodeLineComment) Token {
472 return self.lines.at(0);
473 }
474
475 pub fn lastToken(self: &NodeLineComment) Token {
476 return self.lines.at(self.lines.len - 1);
477 }
478};
std/zig/parser.zig+70-13
......@@ -18,6 +18,7 @@ pub const Parser = struct {
1818 put_back_tokens: [2]Token,
1919 put_back_count: usize,
2020 source_file_name: []const u8,
21 pending_line_comment_node: ?&ast.NodeLineComment,
2122
2223 pub const Tree = struct {
2324 root_node: &ast.NodeRoot,
......@@ -43,6 +44,7 @@ pub const Parser = struct {
4344 .put_back_count = 0,
4445 .source_file_name = source_file_name,
4546 .utility_bytes = []align(utility_bytes_align) u8{},
47 .pending_line_comment_node = null,
4648 };
4749 }
4850
......@@ -131,6 +133,33 @@ pub const Parser = struct {
131133 // warn("\n");
132134 //}
133135
136 // look for line comments
137 while (true) {
138 const token = self.getNextToken();
139 if (token.id == Token.Id.LineComment) {
140 const node = blk: {
141 if (self.pending_line_comment_node) |comment_node| {
142 break :blk comment_node;
143 } else {
144 const comment_node = try arena.create(ast.NodeLineComment);
145 *comment_node = ast.NodeLineComment {
146 .base = ast.Node {
147 .id = ast.Node.Id.LineComment,
148 .comment = null,
149 },
150 .lines = ArrayList(Token).init(arena),
151 };
152 self.pending_line_comment_node = comment_node;
153 break :blk comment_node;
154 }
155 };
156 try node.lines.append(token);
157 continue;
158 }
159 self.putBackToken(token);
160 break;
161 }
162
134163 // This gives us 1 free append that can't fail
135164 const state = stack.pop();
136165
......@@ -329,7 +358,7 @@ pub const Parser = struct {
329358 Token.Id.Builtin => {
330359 const node = try arena.create(ast.NodeBuiltinCall);
331360 *node = ast.NodeBuiltinCall {
332 .base = ast.Node {.id = ast.Node.Id.BuiltinCall},
361 .base = self.initNode(ast.Node.Id.BuiltinCall),
333362 .builtin_token = token,
334363 .params = ArrayList(&ast.Node).init(arena),
335364 .rparen_token = undefined,
......@@ -350,7 +379,7 @@ pub const Parser = struct {
350379 Token.Id.StringLiteral => {
351380 const node = try arena.create(ast.NodeStringLiteral);
352381 *node = ast.NodeStringLiteral {
353 .base = ast.Node {.id = ast.Node.Id.StringLiteral},
382 .base = self.initNode(ast.Node.Id.StringLiteral),
354383 .token = token,
355384 };
356385 try stack.append(State {
......@@ -359,6 +388,7 @@ pub const Parser = struct {
359388 try stack.append(State.AfterOperand);
360389 continue;
361390 },
391
362392 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
363393 }
364394 },
......@@ -660,11 +690,19 @@ pub const Parser = struct {
660690 }
661691 }
662692
693 fn initNode(self: &Parser, id: ast.Node.Id) ast.Node {
694 if (self.pending_line_comment_node) |comment_node| {
695 self.pending_line_comment_node = null;
696 return ast.Node {.id = id, .comment = comment_node};
697 }
698 return ast.Node {.id = id, .comment = null };
699 }
700
663701 fn createRoot(self: &Parser, arena: &mem.Allocator) !&ast.NodeRoot {
664702 const node = try arena.create(ast.NodeRoot);
665703
666704 *node = ast.NodeRoot {
667 .base = ast.Node {.id = ast.Node.Id.Root},
705 .base = self.initNode(ast.Node.Id.Root),
668706 .decls = ArrayList(&ast.Node).init(arena),
669707 // initialized when we get the eof token
670708 .eof_token = undefined,
......@@ -678,7 +716,7 @@ pub const Parser = struct {
678716 const node = try arena.create(ast.NodeVarDecl);
679717
680718 *node = ast.NodeVarDecl {
681 .base = ast.Node {.id = ast.Node.Id.VarDecl},
719 .base = self.initNode(ast.Node.Id.VarDecl),
682720 .visib_token = *visib_token,
683721 .mut_token = *mut_token,
684722 .comptime_token = *comptime_token,
......@@ -701,7 +739,7 @@ pub const Parser = struct {
701739 const node = try arena.create(ast.NodeFnProto);
702740
703741 *node = ast.NodeFnProto {
704 .base = ast.Node {.id = ast.Node.Id.FnProto},
742 .base = self.initNode(ast.Node.Id.FnProto),
705743 .visib_token = *visib_token,
706744 .name_token = null,
707745 .fn_token = *fn_token,
......@@ -722,7 +760,7 @@ pub const Parser = struct {
722760 const node = try arena.create(ast.NodeParamDecl);
723761
724762 *node = ast.NodeParamDecl {
725 .base = ast.Node {.id = ast.Node.Id.ParamDecl},
763 .base = self.initNode(ast.Node.Id.ParamDecl),
726764 .comptime_token = null,
727765 .noalias_token = null,
728766 .name_token = null,
......@@ -736,7 +774,7 @@ pub const Parser = struct {
736774 const node = try arena.create(ast.NodeBlock);
737775
738776 *node = ast.NodeBlock {
739 .base = ast.Node {.id = ast.Node.Id.Block},
777 .base = self.initNode(ast.Node.Id.Block),
740778 .begin_token = *begin_token,
741779 .end_token = undefined,
742780 .statements = ArrayList(&ast.Node).init(arena),
......@@ -748,7 +786,7 @@ pub const Parser = struct {
748786 const node = try arena.create(ast.NodeInfixOp);
749787
750788 *node = ast.NodeInfixOp {
751 .base = ast.Node {.id = ast.Node.Id.InfixOp},
789 .base = self.initNode(ast.Node.Id.InfixOp),
752790 .op_token = *op_token,
753791 .lhs = undefined,
754792 .op = *op,
......@@ -761,7 +799,7 @@ pub const Parser = struct {
761799 const node = try arena.create(ast.NodePrefixOp);
762800
763801 *node = ast.NodePrefixOp {
764 .base = ast.Node {.id = ast.Node.Id.PrefixOp},
802 .base = self.initNode(ast.Node.Id.PrefixOp),
765803 .op_token = *op_token,
766804 .op = *op,
767805 .rhs = undefined,
......@@ -773,7 +811,7 @@ pub const Parser = struct {
773811 const node = try arena.create(ast.NodeIdentifier);
774812
775813 *node = ast.NodeIdentifier {
776 .base = ast.Node {.id = ast.Node.Id.Identifier},
814 .base = self.initNode(ast.Node.Id.Identifier),
777815 .name_token = *name_token,
778816 };
779817 return node;
......@@ -783,7 +821,7 @@ pub const Parser = struct {
783821 const node = try arena.create(ast.NodeIntegerLiteral);
784822
785823 *node = ast.NodeIntegerLiteral {
786 .base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
824 .base = self.initNode(ast.Node.Id.IntegerLiteral),
787825 .token = *token,
788826 };
789827 return node;
......@@ -793,7 +831,7 @@ pub const Parser = struct {
793831 const node = try arena.create(ast.NodeFloatLiteral);
794832
795833 *node = ast.NodeFloatLiteral {
796 .base = ast.Node {.id = ast.Node.Id.FloatLiteral},
834 .base = self.initNode(ast.Node.Id.FloatLiteral),
797835 .token = *token,
798836 };
799837 return node;
......@@ -1158,9 +1196,11 @@ pub const Parser = struct {
11581196 }
11591197 }
11601198 },
1199 ast.Node.Id.FnProto => @panic("TODO fn proto in an expression"),
1200 ast.Node.Id.LineComment => @panic("TODO render line comment in an expression"),
1201
11611202 ast.Node.Id.Root,
11621203 ast.Node.Id.VarDecl,
1163 ast.Node.Id.FnProto,
11641204 ast.Node.Id.ParamDecl => unreachable,
11651205 },
11661206 RenderState.FnProtoRParen => |fn_proto| {
......@@ -1187,6 +1227,12 @@ pub const Parser = struct {
11871227 }
11881228 },
11891229 RenderState.Statement => |base| {
1230 if (base.comment) |comment| {
1231 for (comment.lines.toSliceConst()) |line_token| {
1232 try stream.print("{}\n", self.tokenizer.getTokenSlice(line_token));
1233 try stream.writeByteNTimes(' ', indent);
1234 }
1235 }
11901236 switch (base.id) {
11911237 ast.Node.Id.VarDecl => {
11921238 const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
......@@ -1279,6 +1325,17 @@ fn testCanonical(source: []const u8) !void {
12791325}
12801326
12811327test "zig fmt" {
1328 try testCanonical(
1329 \\const std = @import("std");
1330 \\
1331 \\pub fn main() !void {
1332 \\ // If this program is run without stdout attached, exit with an error.
1333 \\ // another comment
1334 \\ var stdout_file = try std.io.getStdOut;
1335 \\}
1336 \\
1337 );
1338
12821339 try testCanonical(
12831340 \\const std = @import("std");
12841341 \\
std/zig/tokenizer.zig+3-11
......@@ -99,6 +99,7 @@ pub const Token = struct {
9999 AmpersandEqual,
100100 IntegerLiteral,
101101 FloatLiteral,
102 LineComment,
102103 Keyword_align,
103104 Keyword_and,
104105 Keyword_asm,
......@@ -470,7 +471,7 @@ pub const Tokenizer = struct {
470471
471472 State.Slash => switch (c) {
472473 '/' => {
473 result.id = undefined;
474 result.id = Token.Id.LineComment;
474475 state = State.LineComment;
475476 },
476477 else => {
......@@ -479,16 +480,7 @@ pub const Tokenizer = struct {
479480 },
480481 },
481482 State.LineComment => switch (c) {
482 '\n' => {
483 state = State.Start;
484 result = Token {
485 .id = Token.Id.Eof,
486 .start = self.index + 1,
487 .column = 0,
488 .line = self.line + 1,
489 .end = undefined,
490 };
491 },
483 '\n' => break,
492484 else => self.checkLiteralCharacter(),
493485 },
494486 State.Zero => switch (c) {