authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-14 23:00:53-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-14 23:00:53-05:00
logca597e2bfb3c39aecdf3dea2718e84deb749ed07
tree569a7e0b30ddcb5f4be1e14cb01dcfcf1e1951e6
parent9fa35adbd4772f55e3e43dd7cc69823415661153

std.zig.parser understands try. zig fmt respects a double line break.

6 files changed, 296 insertions(+), 40 deletions(-)

src/tokenizer.cpp-2
......@@ -125,7 +125,6 @@ static const struct ZigKeyword zig_keywords[] = {
125125 {"false", TokenIdKeywordFalse},
126126 {"fn", TokenIdKeywordFn},
127127 {"for", TokenIdKeywordFor},
128 {"goto", TokenIdKeywordGoto},
129128 {"if", TokenIdKeywordIf},
130129 {"inline", TokenIdKeywordInline},
131130 {"nakedcc", TokenIdKeywordNakedCC},
......@@ -1542,7 +1541,6 @@ const char * token_name(TokenId id) {
15421541 case TokenIdKeywordFalse: return "false";
15431542 case TokenIdKeywordFn: return "fn";
15441543 case TokenIdKeywordFor: return "for";
1545 case TokenIdKeywordGoto: return "goto";
15461544 case TokenIdKeywordIf: return "if";
15471545 case TokenIdKeywordInline: return "inline";
15481546 case TokenIdKeywordNakedCC: return "nakedcc";
src/tokenizer.hpp-1
......@@ -66,7 +66,6 @@ enum TokenId {
6666 TokenIdKeywordFalse,
6767 TokenIdKeywordFn,
6868 TokenIdKeywordFor,
69 TokenIdKeywordGoto,
7069 TokenIdKeywordIf,
7170 TokenIdKeywordInline,
7271 TokenIdKeywordNakedCC,
std/debug/index.zig+2-2
......@@ -47,7 +47,7 @@ pub fn getSelfDebugInfo() !&ElfStackTrace {
4747pub fn dumpCurrentStackTrace() void {
4848 const stderr = getStderrStream() catch return;
4949 const debug_info = getSelfDebugInfo() catch |err| {
50 stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return;
50 stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return;
5151 return;
5252 };
5353 defer debug_info.close();
......@@ -61,7 +61,7 @@ pub fn dumpCurrentStackTrace() void {
6161pub fn dumpStackTrace(stack_trace: &const builtin.StackTrace) void {
6262 const stderr = getStderrStream() catch return;
6363 const debug_info = getSelfDebugInfo() catch |err| {
64 stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return;
64 stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return;
6565 return;
6666 };
6767 defer debug_info.close();
std/zig/ast.zig+160-4
......@@ -38,11 +38,46 @@ pub const Node = struct {
3838 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
3939 };
4040 }
41
42 pub fn firstToken(base: &Node) Token {
43 return switch (base.id) {
44 Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(),
45 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(),
46 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(),
47 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(),
48 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(),
49 Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(),
50 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),
51 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
52 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
53 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
54 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
55 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
56 };
57 }
58
59 pub fn lastToken(base: &Node) Token {
60 return switch (base.id) {
61 Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(),
62 Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(),
63 Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(),
64 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(),
65 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(),
66 Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(),
67 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),
68 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
69 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
70 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
71 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
72 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
73 };
74 }
4175};
4276
4377pub const NodeRoot = struct {
4478 base: Node,
4579 decls: ArrayList(&Node),
80 eof_token: Token,
4681
4782 pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
4883 if (index < self.decls.len) {
......@@ -50,6 +85,14 @@ pub const NodeRoot = struct {
5085 }
5186 return null;
5287 }
88
89 pub fn firstToken(self: &NodeRoot) Token {
90 return if (self.decls.len == 0) self.eof_token else self.decls.at(0).firstToken();
91 }
92
93 pub fn lastToken(self: &NodeRoot) Token {
94 return if (self.decls.len == 0) self.eof_token else self.decls.at(self.decls.len - 1).lastToken();
95 }
5396};
5497
5598pub const NodeVarDecl = struct {
......@@ -64,6 +107,7 @@ pub const NodeVarDecl = struct {
64107 type_node: ?&Node,
65108 align_node: ?&Node,
66109 init_node: ?&Node,
110 semicolon_token: Token,
67111
68112 pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
69113 var i = index;
......@@ -85,6 +129,18 @@ pub const NodeVarDecl = struct {
85129
86130 return null;
87131 }
132
133 pub fn firstToken(self: &NodeVarDecl) Token {
134 if (self.visib_token) |visib_token| return visib_token;
135 if (self.comptime_token) |comptime_token| return comptime_token;
136 if (self.extern_token) |extern_token| return extern_token;
137 assert(self.lib_name == null);
138 return self.mut_token;
139 }
140
141 pub fn lastToken(self: &NodeVarDecl) Token {
142 return self.semicolon_token;
143 }
88144};
89145
90146pub const NodeIdentifier = struct {
......@@ -94,6 +150,14 @@ pub const NodeIdentifier = struct {
94150 pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
95151 return null;
96152 }
153
154 pub fn firstToken(self: &NodeIdentifier) Token {
155 return self.name_token;
156 }
157
158 pub fn lastToken(self: &NodeIdentifier) Token {
159 return self.name_token;
160 }
97161};
98162
99163pub const NodeFnProto = struct {
......@@ -113,7 +177,7 @@ pub const NodeFnProto = struct {
113177
114178 pub const ReturnType = union(enum) {
115179 Explicit: &Node,
116 Infer,
180 Infer: Token,
117181 InferErrorSet: &Node,
118182 };
119183
......@@ -153,6 +217,25 @@ pub const NodeFnProto = struct {
153217
154218 return null;
155219 }
220
221 pub fn firstToken(self: &NodeFnProto) Token {
222 if (self.visib_token) |visib_token| return visib_token;
223 if (self.extern_token) |extern_token| return extern_token;
224 assert(self.lib_name == null);
225 if (self.inline_token) |inline_token| return inline_token;
226 if (self.cc_token) |cc_token| return cc_token;
227 return self.fn_token;
228 }
229
230 pub fn lastToken(self: &NodeFnProto) Token {
231 if (self.body_node) |body_node| return body_node.lastToken();
232 switch (self.return_type) {
233 // TODO allow this and next prong to share bodies since the types are the same
234 ReturnType.Explicit => |node| return node.lastToken(),
235 ReturnType.InferErrorSet => |node| return node.lastToken(),
236 ReturnType.Infer => |token| return token,
237 }
238 }
156239};
157240
158241pub const NodeParamDecl = struct {
......@@ -171,6 +254,18 @@ pub const NodeParamDecl = struct {
171254
172255 return null;
173256 }
257
258 pub fn firstToken(self: &NodeParamDecl) Token {
259 if (self.comptime_token) |comptime_token| return comptime_token;
260 if (self.noalias_token) |noalias_token| return noalias_token;
261 if (self.name_token) |name_token| return name_token;
262 return self.type_node.firstToken();
263 }
264
265 pub fn lastToken(self: &NodeParamDecl) Token {
266 if (self.var_args_token) |var_args_token| return var_args_token;
267 return self.type_node.lastToken();
268 }
174269};
175270
176271pub const NodeBlock = struct {
......@@ -187,6 +282,14 @@ pub const NodeBlock = struct {
187282
188283 return null;
189284 }
285
286 pub fn firstToken(self: &NodeBlock) Token {
287 return self.begin_token;
288 }
289
290 pub fn lastToken(self: &NodeBlock) Token {
291 return self.end_token;
292 }
190293};
191294
192295pub const NodeInfixOp = struct {
......@@ -199,6 +302,7 @@ pub const NodeInfixOp = struct {
199302 const InfixOp = enum {
200303 EqualEqual,
201304 BangEqual,
305 Period,
202306 };
203307
204308 pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
......@@ -208,8 +312,9 @@ pub const NodeInfixOp = struct {
208312 i -= 1;
209313
210314 switch (self.op) {
211 InfixOp.EqualEqual => {},
212 InfixOp.BangEqual => {},
315 InfixOp.EqualEqual,
316 InfixOp.BangEqual,
317 InfixOp.Period => {},
213318 }
214319
215320 if (i < 1) return self.rhs;
......@@ -217,6 +322,14 @@ pub const NodeInfixOp = struct {
217322
218323 return null;
219324 }
325
326 pub fn firstToken(self: &NodeInfixOp) Token {
327 return self.lhs.firstToken();
328 }
329
330 pub fn lastToken(self: &NodeInfixOp) Token {
331 return self.rhs.lastToken();
332 }
220333};
221334
222335pub const NodePrefixOp = struct {
......@@ -227,6 +340,7 @@ pub const NodePrefixOp = struct {
227340
228341 const PrefixOp = union(enum) {
229342 Return,
343 Try,
230344 AddrOf: AddrOfInfo,
231345 };
232346 const AddrOfInfo = struct {
......@@ -241,7 +355,8 @@ pub const NodePrefixOp = struct {
241355 var i = index;
242356
243357 switch (self.op) {
244 PrefixOp.Return => {},
358 PrefixOp.Return,
359 PrefixOp.Try => {},
245360 PrefixOp.AddrOf => |addr_of_info| {
246361 if (addr_of_info.align_expr) |align_expr| {
247362 if (i < 1) return align_expr;
......@@ -255,6 +370,14 @@ pub const NodePrefixOp = struct {
255370
256371 return null;
257372 }
373
374 pub fn firstToken(self: &NodePrefixOp) Token {
375 return self.op_token;
376 }
377
378 pub fn lastToken(self: &NodePrefixOp) Token {
379 return self.rhs.lastToken();
380 }
258381};
259382
260383pub const NodeIntegerLiteral = struct {
......@@ -264,6 +387,14 @@ pub const NodeIntegerLiteral = struct {
264387 pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
265388 return null;
266389 }
390
391 pub fn firstToken(self: &NodeIntegerLiteral) Token {
392 return self.token;
393 }
394
395 pub fn lastToken(self: &NodeIntegerLiteral) Token {
396 return self.token;
397 }
267398};
268399
269400pub const NodeFloatLiteral = struct {
......@@ -273,12 +404,21 @@ pub const NodeFloatLiteral = struct {
273404 pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
274405 return null;
275406 }
407
408 pub fn firstToken(self: &NodeFloatLiteral) Token {
409 return self.token;
410 }
411
412 pub fn lastToken(self: &NodeFloatLiteral) Token {
413 return self.token;
414 }
276415};
277416
278417pub const NodeBuiltinCall = struct {
279418 base: Node,
280419 builtin_token: Token,
281420 params: ArrayList(&Node),
421 rparen_token: Token,
282422
283423 pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node {
284424 var i = index;
......@@ -288,6 +428,14 @@ pub const NodeBuiltinCall = struct {
288428
289429 return null;
290430 }
431
432 pub fn firstToken(self: &NodeBuiltinCall) Token {
433 return self.builtin_token;
434 }
435
436 pub fn lastToken(self: &NodeBuiltinCall) Token {
437 return self.rparen_token;
438 }
291439};
292440
293441pub const NodeStringLiteral = struct {
......@@ -297,4 +445,12 @@ pub const NodeStringLiteral = struct {
297445 pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node {
298446 return null;
299447 }
448
449 pub fn firstToken(self: &NodeStringLiteral) Token {
450 return self.token;
451 }
452
453 pub fn lastToken(self: &NodeStringLiteral) Token {
454 return self.token;
455 }
300456};
std/zig/parser.zig+96-15
......@@ -69,6 +69,11 @@ pub const Parser = struct {
6969 }
7070 };
7171
72 const ExpectTokenSave = struct {
73 id: Token.Id,
74 ptr: &Token,
75 };
76
7277 const State = union(enum) {
7378 TopLevel,
7479 TopLevelExtern: ?Token,
......@@ -85,6 +90,7 @@ pub const Parser = struct {
8590 VarDeclAlign: &ast.NodeVarDecl,
8691 VarDeclEq: &ast.NodeVarDecl,
8792 ExpectToken: @TagType(Token.Id),
93 ExpectTokenSave: ExpectTokenSave,
8894 FnProto: &ast.NodeFnProto,
8995 FnProtoAlign: &ast.NodeFnProto,
9096 FnProtoReturnType: &ast.NodeFnProto,
......@@ -136,7 +142,10 @@ pub const Parser = struct {
136142 stack.append(State { .TopLevelExtern = token }) catch unreachable;
137143 continue;
138144 },
139 Token.Id.Eof => return Tree {.root_node = root_node, .arena_allocator = arena_allocator},
145 Token.Id.Eof => {
146 root_node.eof_token = token;
147 return Tree {.root_node = root_node, .arena_allocator = arena_allocator};
148 },
140149 else => {
141150 self.putBackToken(token);
142151 stack.append(State { .TopLevelExtern = null }) catch unreachable;
......@@ -231,13 +240,19 @@ pub const Parser = struct {
231240 const token = self.getNextToken();
232241 if (token.id == Token.Id.Equal) {
233242 var_decl.eq_token = token;
234 stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
243 stack.append(State {
244 .ExpectTokenSave = ExpectTokenSave {
245 .id = Token.Id.Semicolon,
246 .ptr = &var_decl.semicolon_token,
247 },
248 }) catch unreachable;
235249 try stack.append(State {
236250 .Expression = DestPtr {.NullableField = &var_decl.init_node},
237251 });
238252 continue;
239253 }
240254 if (token.id == Token.Id.Semicolon) {
255 var_decl.semicolon_token = token;
241256 continue;
242257 }
243258 return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
......@@ -247,6 +262,11 @@ pub const Parser = struct {
247262 continue;
248263 },
249264
265 State.ExpectTokenSave => |expect_token_save| {
266 *expect_token_save.ptr = try self.eatToken(expect_token_save.id);
267 continue;
268 },
269
250270 State.Expression => |dest_ptr| {
251271 // save the dest_ptr for later
252272 stack.append(state) catch unreachable;
......@@ -264,6 +284,12 @@ pub const Parser = struct {
264284 try stack.append(State.ExpectOperand);
265285 continue;
266286 },
287 Token.Id.Keyword_try => {
288 try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token,
289 ast.NodePrefixOp.PrefixOp.Try) });
290 try stack.append(State.ExpectOperand);
291 continue;
292 },
267293 Token.Id.Ampersand => {
268294 const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{
269295 .AddrOf = ast.NodePrefixOp.AddrOfInfo {
......@@ -306,13 +332,19 @@ pub const Parser = struct {
306332 .base = ast.Node {.id = ast.Node.Id.BuiltinCall},
307333 .builtin_token = token,
308334 .params = ArrayList(&ast.Node).init(arena),
335 .rparen_token = undefined,
309336 };
310337 try stack.append(State {
311338 .Operand = &node.base
312339 });
313340 try stack.append(State.AfterOperand);
314341 try stack.append(State {.ExprListItemOrEnd = &node.params });
315 try stack.append(State {.ExpectToken = Token.Id.LParen });
342 try stack.append(State {
343 .ExpectTokenSave = ExpectTokenSave {
344 .id = Token.Id.LParen,
345 .ptr = &node.rparen_token,
346 },
347 });
316348 continue;
317349 },
318350 Token.Id.StringLiteral => {
......@@ -351,6 +383,13 @@ pub const Parser = struct {
351383 try stack.append(State.ExpectOperand);
352384 continue;
353385 },
386 Token.Id.Period => {
387 try stack.append(State {
388 .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period)
389 });
390 try stack.append(State.ExpectOperand);
391 continue;
392 },
354393 else => {
355394 // no postfix/infix operator after this operand.
356395 self.putBackToken(token);
......@@ -476,7 +515,7 @@ pub const Parser = struct {
476515 const token = self.getNextToken();
477516 switch (token.id) {
478517 Token.Id.Keyword_var => {
479 fn_proto.return_type = ast.NodeFnProto.ReturnType.Infer;
518 fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token };
480519 },
481520 Token.Id.Bang => {
482521 fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined };
......@@ -627,6 +666,8 @@ pub const Parser = struct {
627666 *node = ast.NodeRoot {
628667 .base = ast.Node {.id = ast.Node.Id.Root},
629668 .decls = ArrayList(&ast.Node).init(arena),
669 // initialized when we get the eof token
670 .eof_token = undefined,
630671 };
631672 return node;
632673 }
......@@ -649,6 +690,7 @@ pub const Parser = struct {
649690 // initialized later
650691 .name_token = undefined,
651692 .eq_token = undefined,
693 .semicolon_token = undefined,
652694 };
653695 return node;
654696 }
......@@ -789,11 +831,11 @@ pub const Parser = struct {
789831
790832 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) {
791833 const loc = self.tokenizer.getTokenLocation(token);
792 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
834 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args);
793835 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
794836 {
795837 var i: usize = 0;
796 while (i < loc.column) : (i += 1) {
838 while (i < token.column) : (i += 1) {
797839 warn(" ");
798840 }
799841 }
......@@ -885,11 +927,26 @@ pub const Parser = struct {
885927 defer self.deinitUtilityArrayList(stack);
886928
887929 {
930 try stack.append(RenderState { .Text = "\n"});
931
888932 var i = root_node.decls.len;
889933 while (i != 0) {
890934 i -= 1;
891935 const decl = root_node.decls.items[i];
892936 try stack.append(RenderState {.TopLevelDecl = decl});
937 if (i != 0) {
938 try stack.append(RenderState {
939 .Text = blk: {
940 const prev_node = root_node.decls.at(i - 1);
941 const prev_line_index = prev_node.lastToken().line;
942 const this_line_index = decl.firstToken().line;
943 if (this_line_index - prev_line_index >= 2) {
944 break :blk "\n\n";
945 }
946 break :blk "\n";
947 },
948 });
949 }
893950 }
894951 }
895952
......@@ -919,7 +976,6 @@ pub const Parser = struct {
919976
920977 try stream.print("(");
921978
922 try stack.append(RenderState { .Text = "\n" });
923979 if (fn_proto.body_node == null) {
924980 try stack.append(RenderState { .Text = ";" });
925981 }
......@@ -937,7 +993,6 @@ pub const Parser = struct {
937993 },
938994 ast.Node.Id.VarDecl => {
939995 const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
940 try stack.append(RenderState { .Text = "\n"});
941996 try stack.append(RenderState { .VarDecl = var_decl});
942997
943998 },
......@@ -1019,7 +1074,19 @@ pub const Parser = struct {
10191074 try stack.append(RenderState { .Statement = statement_node});
10201075 try stack.append(RenderState.PrintIndent);
10211076 try stack.append(RenderState { .Indent = indent + indent_delta});
1022 try stack.append(RenderState { .Text = "\n" });
1077 try stack.append(RenderState {
1078 .Text = blk: {
1079 if (i != 0) {
1080 const prev_statement_node = block.statements.items[i - 1];
1081 const prev_line_index = prev_statement_node.lastToken().line;
1082 const this_line_index = statement_node.firstToken().line;
1083 if (this_line_index - prev_line_index >= 2) {
1084 break :blk "\n\n";
1085 }
1086 }
1087 break :blk "\n";
1088 },
1089 });
10231090 }
10241091 }
10251092 },
......@@ -1033,7 +1100,9 @@ pub const Parser = struct {
10331100 ast.NodeInfixOp.InfixOp.BangEqual => {
10341101 try stack.append(RenderState { .Text = " != "});
10351102 },
1036 else => unreachable,
1103 ast.NodeInfixOp.InfixOp.Period => {
1104 try stack.append(RenderState { .Text = "."});
1105 },
10371106 }
10381107 try stack.append(RenderState { .Expression = prefix_op_node.lhs });
10391108 },
......@@ -1044,6 +1113,9 @@ pub const Parser = struct {
10441113 ast.NodePrefixOp.PrefixOp.Return => {
10451114 try stream.write("return ");
10461115 },
1116 ast.NodePrefixOp.PrefixOp.Try => {
1117 try stream.write("try ");
1118 },
10471119 ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| {
10481120 try stream.write("&");
10491121 if (addr_of_info.volatile_token != null) {
......@@ -1058,7 +1130,6 @@ pub const Parser = struct {
10581130 try stack.append(RenderState { .Expression = align_expr});
10591131 }
10601132 },
1061 else => unreachable,
10621133 }
10631134 },
10641135 ast.Node.Id.IntegerLiteral => {
......@@ -1153,10 +1224,7 @@ pub const Parser = struct {
11531224var fixed_buffer_mem: [100 * 1024]u8 = undefined;
11541225
11551226fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
1156 var padded_source: [0x100]u8 = undefined;
1157 std.mem.copy(u8, padded_source[0..source.len], source);
1158
1159 var tokenizer = Tokenizer.init(padded_source[0..source.len]);
1227 var tokenizer = Tokenizer.init(source);
11601228 var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
11611229 defer parser.deinit();
11621230
......@@ -1211,6 +1279,19 @@ fn testCanonical(source: []const u8) !void {
12111279}
12121280
12131281test "zig fmt" {
1282 try testCanonical(
1283 \\const std = @import("std");
1284 \\
1285 \\pub fn main() !void {
1286 \\ var stdout_file = try std.io.getStdOut;
1287 \\ var stdout_file = try std.io.getStdOut;
1288 \\
1289 \\ var stdout_file = try std.io.getStdOut;
1290 \\ var stdout_file = try std.io.getStdOut;
1291 \\}
1292 \\
1293 );
1294
12141295 try testCanonical(
12151296 \\pub fn main() !void {}
12161297 \\pub fn main() var {}
std/zig/tokenizer.zig+38-16
......@@ -5,6 +5,8 @@ pub const Token = struct {
55 id: Id,
66 start: usize,
77 end: usize,
8 line: usize,
9 column: usize,
810
911 const KeywordId = struct {
1012 bytes: []const u8,
......@@ -16,6 +18,7 @@ pub const Token = struct {
1618 KeywordId{.bytes="and", .id = Id.Keyword_and},
1719 KeywordId{.bytes="asm", .id = Id.Keyword_asm},
1820 KeywordId{.bytes="break", .id = Id.Keyword_break},
21 KeywordId{.bytes="catch", .id = Id.Keyword_catch},
1922 KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
2023 KeywordId{.bytes="const", .id = Id.Keyword_const},
2124 KeywordId{.bytes="continue", .id = Id.Keyword_continue},
......@@ -28,7 +31,6 @@ pub const Token = struct {
2831 KeywordId{.bytes="false", .id = Id.Keyword_false},
2932 KeywordId{.bytes="fn", .id = Id.Keyword_fn},
3033 KeywordId{.bytes="for", .id = Id.Keyword_for},
31 KeywordId{.bytes="goto", .id = Id.Keyword_goto},
3234 KeywordId{.bytes="if", .id = Id.Keyword_if},
3335 KeywordId{.bytes="inline", .id = Id.Keyword_inline},
3436 KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
......@@ -38,12 +40,14 @@ pub const Token = struct {
3840 KeywordId{.bytes="packed", .id = Id.Keyword_packed},
3941 KeywordId{.bytes="pub", .id = Id.Keyword_pub},
4042 KeywordId{.bytes="return", .id = Id.Keyword_return},
43 KeywordId{.bytes="section", .id = Id.Keyword_section},
4144 KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
4245 KeywordId{.bytes="struct", .id = Id.Keyword_struct},
4346 KeywordId{.bytes="switch", .id = Id.Keyword_switch},
4447 KeywordId{.bytes="test", .id = Id.Keyword_test},
4548 KeywordId{.bytes="this", .id = Id.Keyword_this},
4649 KeywordId{.bytes="true", .id = Id.Keyword_true},
50 KeywordId{.bytes="try", .id = Id.Keyword_try},
4751 KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
4852 KeywordId{.bytes="union", .id = Id.Keyword_union},
4953 KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
......@@ -99,6 +103,7 @@ pub const Token = struct {
99103 Keyword_and,
100104 Keyword_asm,
101105 Keyword_break,
106 Keyword_catch,
102107 Keyword_comptime,
103108 Keyword_const,
104109 Keyword_continue,
......@@ -111,7 +116,6 @@ pub const Token = struct {
111116 Keyword_false,
112117 Keyword_fn,
113118 Keyword_for,
114 Keyword_goto,
115119 Keyword_if,
116120 Keyword_inline,
117121 Keyword_nakedcc,
......@@ -121,12 +125,14 @@ pub const Token = struct {
121125 Keyword_packed,
122126 Keyword_pub,
123127 Keyword_return,
128 Keyword_section,
124129 Keyword_stdcallcc,
125130 Keyword_struct,
126131 Keyword_switch,
127132 Keyword_test,
128133 Keyword_this,
129134 Keyword_true,
135 Keyword_try,
130136 Keyword_undefined,
131137 Keyword_union,
132138 Keyword_unreachable,
......@@ -140,21 +146,19 @@ pub const Token = struct {
140146pub const Tokenizer = struct {
141147 buffer: []const u8,
142148 index: usize,
149 line: usize,
150 column: usize,
143151 pending_invalid_token: ?Token,
144152
145 pub const Location = struct {
146 line: usize,
147 column: usize,
153 pub const LineLocation = struct {
148154 line_start: usize,
149155 line_end: usize,
150156 };
151157
152 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {
153 var loc = Location {
154 .line = 0,
155 .column = 0,
158 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation {
159 var loc = LineLocation {
156160 .line_start = 0,
157 .line_end = 0,
161 .line_end = self.buffer.len,
158162 };
159163 for (self.buffer) |c, i| {
160164 if (i == token.start) {
......@@ -163,11 +167,7 @@ pub const Tokenizer = struct {
163167 return loc;
164168 }
165169 if (c == '\n') {
166 loc.line += 1;
167 loc.column = 0;
168170 loc.line_start = i + 1;
169 } else {
170 loc.column += 1;
171171 }
172172 }
173173 return loc;
......@@ -182,6 +182,8 @@ pub const Tokenizer = struct {
182182 return Tokenizer {
183183 .buffer = buffer,
184184 .index = 0,
185 .line = 0,
186 .column = 0,
185187 .pending_invalid_token = null,
186188 };
187189 }
......@@ -222,13 +224,21 @@ pub const Tokenizer = struct {
222224 .id = Token.Id.Eof,
223225 .start = self.index,
224226 .end = undefined,
227 .line = self.line,
228 .column = self.column,
225229 };
226 while (self.index < self.buffer.len) : (self.index += 1) {
230 while (self.index < self.buffer.len) {
227231 const c = self.buffer[self.index];
228232 switch (state) {
229233 State.Start => switch (c) {
230 ' ', '\n' => {
234 ' ' => {
235 result.start = self.index + 1;
236 result.column += 1;
237 },
238 '\n' => {
231239 result.start = self.index + 1;
240 result.line += 1;
241 result.column = 0;
232242 },
233243 'c' => {
234244 state = State.C;
......@@ -474,6 +484,8 @@ pub const Tokenizer = struct {
474484 result = Token {
475485 .id = Token.Id.Eof,
476486 .start = self.index + 1,
487 .column = 0,
488 .line = self.line + 1,
477489 .end = undefined,
478490 };
479491 },
......@@ -543,6 +555,14 @@ pub const Tokenizer = struct {
543555 else => break,
544556 },
545557 }
558
559 self.index += 1;
560 if (c == '\n') {
561 self.line += 1;
562 self.column = 0;
563 } else {
564 self.column += 1;
565 }
546566 } else if (self.index == self.buffer.len) {
547567 switch (state) {
548568 State.Start,
......@@ -622,6 +642,8 @@ pub const Tokenizer = struct {
622642 .id = Token.Id.Invalid,
623643 .start = self.index,
624644 .end = self.index + invalid_length,
645 .line = self.line,
646 .column = self.column,
625647 };
626648 }
627649