authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-07 23:54:35-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-05-07 23:54:35-04:00
logca27ce3bee16ebb611621f15830dd6bf74d65f9f
treeeb3c01eb95da13f282e0c1194d6ec8588713c101
parent0cb65b266aa20015f068e0460c74eb75a0b7f65c

std.zig.parser supports same-line comments on any token

3 files changed, 351 insertions(+), 273 deletions(-)

std/zig/ast.zig+9-3
...@@ -25,7 +25,10 @@ pub const Tree = struct {...@@ -25,7 +25,10 @@ pub const Tree = struct {
25 }25 }
2626
27 pub fn tokenSlice(self: &Tree, token_index: TokenIndex) []const u8 {27 pub fn tokenSlice(self: &Tree, token_index: TokenIndex) []const u8 {
28 const token = self.tokens.at(token_index);28 return self.tokenSlicePtr(self.tokens.at(token_index));
29 }
30
31 pub fn tokenSlicePtr(self: &Tree, token: &const Token) []const u8 {
29 return self.source[token.start..token.end];32 return self.source[token.start..token.end];
30 }33 }
3134
...@@ -36,14 +39,14 @@ pub const Tree = struct {...@@ -36,14 +39,14 @@ pub const Tree = struct {
36 line_end: usize,39 line_end: usize,
37 };40 };
3841
39 pub fn tokenLocation(self: &Tree, start_index: usize, token_index: TokenIndex) Location {42 pub fn tokenLocationPtr(self: &Tree, start_index: usize, token: &const Token) Location {
40 var loc = Location {43 var loc = Location {
41 .line = 0,44 .line = 0,
42 .column = 0,45 .column = 0,
43 .line_start = start_index,46 .line_start = start_index,
44 .line_end = self.source.len,47 .line_end = self.source.len,
45 };48 };
46 const token_start = self.tokens.at(token_index).start;49 const token_start = token.start;
47 for (self.source[start_index..]) |c, i| {50 for (self.source[start_index..]) |c, i| {
48 if (i + start_index == token_start) {51 if (i + start_index == token_start) {
49 loc.line_end = i + start_index;52 loc.line_end = i + start_index;
...@@ -61,6 +64,9 @@ pub const Tree = struct {...@@ -61,6 +64,9 @@ pub const Tree = struct {
61 return loc;64 return loc;
62 }65 }
6366
67 pub fn tokenLocation(self: &Tree, start_index: usize, token_index: TokenIndex) Location {
68 return self.tokenLocationPtr(start_index, self.tokens.at(token_index));
69 }
64};70};
6571
66pub const Error = union(enum) {72pub const Error = union(enum) {
std/zig/parse.zig+288-216
...@@ -54,14 +54,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -54,14 +54,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
5454
55 switch (state) {55 switch (state) {
56 State.TopLevel => {56 State.TopLevel => {
57 while (try eatLineComment(arena, &tok_it)) |line_comment| {57 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
58 try root_node.decls.push(&line_comment.base);58 try root_node.decls.push(&line_comment.base);
59 }59 }
6060
61 const comments = try eatDocComments(arena, &tok_it);61 const comments = try eatDocComments(arena, &tok_it, &tree);
6262
63 const token_index = tok_it.index;63 const token = nextToken(&tok_it, &tree);
64 const token_ptr = ??tok_it.next();64 const token_index = token.index;
65 const token_ptr = token.ptr;
65 switch (token_ptr.id) {66 switch (token_ptr.id) {
66 Token.Id.Keyword_test => {67 Token.Id.Keyword_test => {
67 stack.push(State.TopLevel) catch unreachable;68 stack.push(State.TopLevel) catch unreachable;
...@@ -144,7 +145,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -144,7 +145,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
144 continue;145 continue;
145 },146 },
146 else => {147 else => {
147 _ = tok_it.prev();148 putBackToken(&tok_it, &tree);
148 stack.push(State.TopLevel) catch unreachable;149 stack.push(State.TopLevel) catch unreachable;
149 try stack.push(State {150 try stack.push(State {
150 .TopLevelExtern = TopLevelDeclCtx {151 .TopLevelExtern = TopLevelDeclCtx {
...@@ -160,8 +161,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -160,8 +161,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
160 }161 }
161 },162 },
162 State.TopLevelExtern => |ctx| {163 State.TopLevelExtern => |ctx| {
163 const token_index = tok_it.index;164 const token = nextToken(&tok_it, &tree);
164 const token_ptr = ??tok_it.next();165 const token_index = token.index;
166 const token_ptr = token.ptr;
165 switch (token_ptr.id) {167 switch (token_ptr.id) {
166 Token.Id.Keyword_export, Token.Id.Keyword_inline => {168 Token.Id.Keyword_export, Token.Id.Keyword_inline => {
167 stack.push(State {169 stack.push(State {
...@@ -194,7 +196,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -194,7 +196,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
194 continue;196 continue;
195 },197 },
196 else => {198 else => {
197 _ = tok_it.prev();199 putBackToken(&tok_it, &tree);
198 stack.push(State { .TopLevelDecl = ctx }) catch unreachable;200 stack.push(State { .TopLevelDecl = ctx }) catch unreachable;
199 continue;201 continue;
200 }202 }
...@@ -202,10 +204,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -202,10 +204,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
202 },204 },
203 State.TopLevelLibname => |ctx| {205 State.TopLevelLibname => |ctx| {
204 const lib_name = blk: {206 const lib_name = blk: {
205 const lib_name_token_index = tok_it.index;207 const lib_name_token = nextToken(&tok_it, &tree);
206 const lib_name_token_ptr = ??tok_it.next();208 const lib_name_token_index = lib_name_token.index;
207 break :blk (try parseStringLiteral(arena, &tok_it, lib_name_token_ptr, lib_name_token_index)) ?? {209 const lib_name_token_ptr = lib_name_token.ptr;
208 _ = tok_it.prev();210 break :blk (try parseStringLiteral(arena, &tok_it, lib_name_token_ptr, lib_name_token_index, &tree)) ?? {
211 putBackToken(&tok_it, &tree);
209 break :blk null;212 break :blk null;
210 };213 };
211 };214 };
...@@ -222,8 +225,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -222,8 +225,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
222 continue;225 continue;
223 },226 },
224 State.TopLevelDecl => |ctx| {227 State.TopLevelDecl => |ctx| {
225 const token_index = tok_it.index;228 const token = nextToken(&tok_it, &tree);
226 const token_ptr = ??tok_it.next();229 const token_index = token.index;
230 const token_ptr = token.ptr;
227 switch (token_ptr.id) {231 switch (token_ptr.id) {
228 Token.Id.Keyword_use => {232 Token.Id.Keyword_use => {
229 if (ctx.extern_export_inline_token) |annotated_token| {233 if (ctx.extern_export_inline_token) |annotated_token| {
...@@ -345,7 +349,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -345,7 +349,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
345 }349 }
346 },350 },
347 State.TopLevelExternOrField => |ctx| {351 State.TopLevelExternOrField => |ctx| {
348 if (eatToken(&tok_it, Token.Id.Identifier)) |identifier| {352 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |identifier| {
349 std.debug.assert(ctx.container_decl.kind == ast.Node.ContainerDecl.Kind.Struct);353 std.debug.assert(ctx.container_decl.kind == ast.Node.ContainerDecl.Kind.Struct);
350 const node = try arena.construct(ast.Node.StructField {354 const node = try arena.construct(ast.Node.StructField {
351 .base = ast.Node {355 .base = ast.Node {
...@@ -379,10 +383,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -379,10 +383,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
379 },383 },
380384
381 State.FieldInitValue => |ctx| {385 State.FieldInitValue => |ctx| {
382 const eq_tok_index = tok_it.index;386 const eq_tok = nextToken(&tok_it, &tree);
383 const eq_tok_ptr = ??tok_it.next();387 const eq_tok_index = eq_tok.index;
388 const eq_tok_ptr = eq_tok.ptr;
384 if (eq_tok_ptr.id != Token.Id.Equal) {389 if (eq_tok_ptr.id != Token.Id.Equal) {
385 _ = tok_it.prev();390 putBackToken(&tok_it, &tree);
386 continue;391 continue;
387 }392 }
388 stack.push(State { .Expression = ctx }) catch unreachable;393 stack.push(State { .Expression = ctx }) catch unreachable;
...@@ -390,8 +395,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -390,8 +395,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
390 },395 },
391396
392 State.ContainerKind => |ctx| {397 State.ContainerKind => |ctx| {
393 const token_index = tok_it.index;398 const token = nextToken(&tok_it, &tree);
394 const token_ptr = ??tok_it.next();399 const token_index = token.index;
400 const token_ptr = token.ptr;
395 const node = try createToCtxNode(arena, ctx.opt_ctx, ast.Node.ContainerDecl,401 const node = try createToCtxNode(arena, ctx.opt_ctx, ast.Node.ContainerDecl,
396 ast.Node.ContainerDecl {402 ast.Node.ContainerDecl {
397 .base = undefined,403 .base = undefined,
...@@ -421,7 +427,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -421,7 +427,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
421 },427 },
422428
423 State.ContainerInitArgStart => |container_decl| {429 State.ContainerInitArgStart => |container_decl| {
424 if (eatToken(&tok_it, Token.Id.LParen) == null) {430 if (eatToken(&tok_it, &tree, Token.Id.LParen) == null) {
425 continue;431 continue;
426 }432 }
427433
...@@ -431,24 +437,26 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -431,24 +437,26 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
431 },437 },
432438
433 State.ContainerInitArg => |container_decl| {439 State.ContainerInitArg => |container_decl| {
434 const init_arg_token_index = tok_it.index;440 const init_arg_token = nextToken(&tok_it, &tree);
435 const init_arg_token_ptr = ??tok_it.next();441 const init_arg_token_index = init_arg_token.index;
442 const init_arg_token_ptr = init_arg_token.ptr;
436 switch (init_arg_token_ptr.id) {443 switch (init_arg_token_ptr.id) {
437 Token.Id.Keyword_enum => {444 Token.Id.Keyword_enum => {
438 container_decl.init_arg_expr = ast.Node.ContainerDecl.InitArg {.Enum = null};445 container_decl.init_arg_expr = ast.Node.ContainerDecl.InitArg {.Enum = null};
439 const lparen_tok_index = tok_it.index;446 const lparen_tok = nextToken(&tok_it, &tree);
440 const lparen_tok_ptr = ??tok_it.next();447 const lparen_tok_index = lparen_tok.index;
448 const lparen_tok_ptr = lparen_tok.ptr;
441 if (lparen_tok_ptr.id == Token.Id.LParen) {449 if (lparen_tok_ptr.id == Token.Id.LParen) {
442 try stack.push(State { .ExpectToken = Token.Id.RParen } );450 try stack.push(State { .ExpectToken = Token.Id.RParen } );
443 try stack.push(State { .Expression = OptionalCtx {451 try stack.push(State { .Expression = OptionalCtx {
444 .RequiredNull = &container_decl.init_arg_expr.Enum,452 .RequiredNull = &container_decl.init_arg_expr.Enum,
445 } });453 } });
446 } else {454 } else {
447 _ = tok_it.prev();455 putBackToken(&tok_it, &tree);
448 }456 }
449 },457 },
450 else => {458 else => {
451 _ = tok_it.prev();459 putBackToken(&tok_it, &tree);
452 container_decl.init_arg_expr = ast.Node.ContainerDecl.InitArg { .Type = undefined };460 container_decl.init_arg_expr = ast.Node.ContainerDecl.InitArg { .Type = undefined };
453 stack.push(State { .Expression = OptionalCtx { .Required = &container_decl.init_arg_expr.Type } }) catch unreachable;461 stack.push(State { .Expression = OptionalCtx { .Required = &container_decl.init_arg_expr.Type } }) catch unreachable;
454 },462 },
...@@ -457,13 +465,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -457,13 +465,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
457 },465 },
458466
459 State.ContainerDecl => |container_decl| {467 State.ContainerDecl => |container_decl| {
460 while (try eatLineComment(arena, &tok_it)) |line_comment| {468 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
461 try container_decl.fields_and_decls.push(&line_comment.base);469 try container_decl.fields_and_decls.push(&line_comment.base);
462 }470 }
463471
464 const comments = try eatDocComments(arena, &tok_it);472 const comments = try eatDocComments(arena, &tok_it, &tree);
465 const token_index = tok_it.index;473 const token = nextToken(&tok_it, &tree);
466 const token_ptr = ??tok_it.next();474 const token_index = token.index;
475 const token_ptr = token.ptr;
467 switch (token_ptr.id) {476 switch (token_ptr.id) {
468 Token.Id.Identifier => {477 Token.Id.Identifier => {
469 switch (container_decl.kind) {478 switch (container_decl.kind) {
...@@ -568,7 +577,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -568,7 +577,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
568 continue;577 continue;
569 },578 },
570 else => {579 else => {
571 _ = tok_it.prev();580 putBackToken(&tok_it, &tree);
572 stack.push(State{ .ContainerDecl = container_decl }) catch unreachable;581 stack.push(State{ .ContainerDecl = container_decl }) catch unreachable;
573 try stack.push(State {582 try stack.push(State {
574 .TopLevelExtern = TopLevelDeclCtx {583 .TopLevelExtern = TopLevelDeclCtx {
...@@ -620,8 +629,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -620,8 +629,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
620 State.VarDeclAlign => |var_decl| {629 State.VarDeclAlign => |var_decl| {
621 try stack.push(State { .VarDeclEq = var_decl });630 try stack.push(State { .VarDeclEq = var_decl });
622631
623 const next_token_index = tok_it.index;632 const next_token = nextToken(&tok_it, &tree);
624 const next_token_ptr = ??tok_it.next();633 const next_token_index = next_token.index;
634 const next_token_ptr = next_token.ptr;
625 if (next_token_ptr.id == Token.Id.Keyword_align) {635 if (next_token_ptr.id == Token.Id.Keyword_align) {
626 try stack.push(State { .ExpectToken = Token.Id.RParen });636 try stack.push(State { .ExpectToken = Token.Id.RParen });
627 try stack.push(State { .Expression = OptionalCtx { .RequiredNull = &var_decl.align_node} });637 try stack.push(State { .Expression = OptionalCtx { .RequiredNull = &var_decl.align_node} });
...@@ -629,12 +639,13 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -629,12 +639,13 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
629 continue;639 continue;
630 }640 }
631641
632 _ = tok_it.prev();642 putBackToken(&tok_it, &tree);
633 continue;643 continue;
634 },644 },
635 State.VarDeclEq => |var_decl| {645 State.VarDeclEq => |var_decl| {
636 const token_index = tok_it.index;646 const token = nextToken(&tok_it, &tree);
637 const token_ptr = ??tok_it.next();647 const token_index = token.index;
648 const token_ptr = token.ptr;
638 switch (token_ptr.id) {649 switch (token_ptr.id) {
639 Token.Id.Equal => {650 Token.Id.Equal => {
640 var_decl.eq_token = token_index;651 var_decl.eq_token = token_index;
...@@ -662,8 +673,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -662,8 +673,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
662673
663674
664 State.FnDef => |fn_proto| {675 State.FnDef => |fn_proto| {
665 const token_index = tok_it.index;676 const token = nextToken(&tok_it, &tree);
666 const token_ptr = ??tok_it.next();677 const token_index = token.index;
678 const token_ptr = token.ptr;
667 switch(token_ptr.id) {679 switch(token_ptr.id) {
668 Token.Id.LBrace => {680 Token.Id.LBrace => {
669 const block = try arena.construct(ast.Node.Block {681 const block = try arena.construct(ast.Node.Block {
...@@ -691,7 +703,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -691,7 +703,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
691 try stack.push(State { .ParamDecl = fn_proto });703 try stack.push(State { .ParamDecl = fn_proto });
692 try stack.push(State { .ExpectToken = Token.Id.LParen });704 try stack.push(State { .ExpectToken = Token.Id.LParen });
693705
694 if (eatToken(&tok_it, Token.Id.Identifier)) |name_token| {706 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |name_token| {
695 fn_proto.name_token = name_token;707 fn_proto.name_token = name_token;
696 }708 }
697 continue;709 continue;
...@@ -699,7 +711,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -699,7 +711,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
699 State.FnProtoAlign => |fn_proto| {711 State.FnProtoAlign => |fn_proto| {
700 stack.push(State { .FnProtoReturnType = fn_proto }) catch unreachable;712 stack.push(State { .FnProtoReturnType = fn_proto }) catch unreachable;
701713
702 if (eatToken(&tok_it, Token.Id.Keyword_align)) |align_token| {714 if (eatToken(&tok_it, &tree, Token.Id.Keyword_align)) |align_token| {
703 try stack.push(State { .ExpectToken = Token.Id.RParen });715 try stack.push(State { .ExpectToken = Token.Id.RParen });
704 try stack.push(State { .Expression = OptionalCtx { .RequiredNull = &fn_proto.align_expr } });716 try stack.push(State { .Expression = OptionalCtx { .RequiredNull = &fn_proto.align_expr } });
705 try stack.push(State { .ExpectToken = Token.Id.LParen });717 try stack.push(State { .ExpectToken = Token.Id.LParen });
...@@ -707,8 +719,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -707,8 +719,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
707 continue;719 continue;
708 },720 },
709 State.FnProtoReturnType => |fn_proto| {721 State.FnProtoReturnType => |fn_proto| {
710 const token_index = tok_it.index;722 const token = nextToken(&tok_it, &tree);
711 const token_ptr = ??tok_it.next();723 const token_index = token.index;
724 const token_ptr = token.ptr;
712 switch (token_ptr.id) {725 switch (token_ptr.id) {
713 Token.Id.Bang => {726 Token.Id.Bang => {
714 fn_proto.return_type = ast.Node.FnProto.ReturnType { .InferErrorSet = undefined };727 fn_proto.return_type = ast.Node.FnProto.ReturnType { .InferErrorSet = undefined };
...@@ -732,7 +745,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -732,7 +745,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
732 }745 }
733 }746 }
734747
735 _ = tok_it.prev();748 putBackToken(&tok_it, &tree);
736 fn_proto.return_type = ast.Node.FnProto.ReturnType { .Explicit = undefined };749 fn_proto.return_type = ast.Node.FnProto.ReturnType { .Explicit = undefined };
737 stack.push(State { .TypeExprBegin = OptionalCtx { .Required = &fn_proto.return_type.Explicit }, }) catch unreachable;750 stack.push(State { .TypeExprBegin = OptionalCtx { .Required = &fn_proto.return_type.Explicit }, }) catch unreachable;
738 continue;751 continue;
...@@ -742,7 +755,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -742,7 +755,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
742755
743756
744 State.ParamDecl => |fn_proto| {757 State.ParamDecl => |fn_proto| {
745 if (eatToken(&tok_it, Token.Id.RParen)) |_| {758 if (eatToken(&tok_it, &tree, Token.Id.RParen)) |_| {
746 continue;759 continue;
747 }760 }
748 const param_decl = try arena.construct(ast.Node.ParamDecl {761 const param_decl = try arena.construct(ast.Node.ParamDecl {
...@@ -766,9 +779,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -766,9 +779,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
766 continue;779 continue;
767 },780 },
768 State.ParamDeclAliasOrComptime => |param_decl| {781 State.ParamDeclAliasOrComptime => |param_decl| {
769 if (eatToken(&tok_it, Token.Id.Keyword_comptime)) |comptime_token| {782 if (eatToken(&tok_it, &tree, Token.Id.Keyword_comptime)) |comptime_token| {
770 param_decl.comptime_token = comptime_token;783 param_decl.comptime_token = comptime_token;
771 } else if (eatToken(&tok_it, Token.Id.Keyword_noalias)) |noalias_token| {784 } else if (eatToken(&tok_it, &tree, Token.Id.Keyword_noalias)) |noalias_token| {
772 param_decl.noalias_token = noalias_token;785 param_decl.noalias_token = noalias_token;
773 }786 }
774 continue;787 continue;
...@@ -776,17 +789,17 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -776,17 +789,17 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
776 State.ParamDeclName => |param_decl| {789 State.ParamDeclName => |param_decl| {
777 // TODO: Here, we eat two tokens in one state. This means that we can't have790 // TODO: Here, we eat two tokens in one state. This means that we can't have
778 // comments between these two tokens.791 // comments between these two tokens.
779 if (eatToken(&tok_it, Token.Id.Identifier)) |ident_token| {792 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |ident_token| {
780 if (eatToken(&tok_it, Token.Id.Colon)) |_| {793 if (eatToken(&tok_it, &tree, Token.Id.Colon)) |_| {
781 param_decl.name_token = ident_token;794 param_decl.name_token = ident_token;
782 } else {795 } else {
783 _ = tok_it.prev();796 putBackToken(&tok_it, &tree);
784 }797 }
785 }798 }
786 continue;799 continue;
787 },800 },
788 State.ParamDeclEnd => |ctx| {801 State.ParamDeclEnd => |ctx| {
789 if (eatToken(&tok_it, Token.Id.Ellipsis3)) |ellipsis3| {802 if (eatToken(&tok_it, &tree, Token.Id.Ellipsis3)) |ellipsis3| {
790 ctx.param_decl.var_args_token = ellipsis3;803 ctx.param_decl.var_args_token = ellipsis3;
791 stack.push(State { .ExpectToken = Token.Id.RParen }) catch unreachable;804 stack.push(State { .ExpectToken = Token.Id.RParen }) catch unreachable;
792 continue;805 continue;
...@@ -799,7 +812,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -799,7 +812,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
799 continue;812 continue;
800 },813 },
801 State.ParamDeclComma => |fn_proto| {814 State.ParamDeclComma => |fn_proto| {
802 switch (expectCommaOrEnd(&tok_it, Token.Id.RParen)) {815 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RParen)) {
803 ExpectCommaOrEndResult.end_token => |t| {816 ExpectCommaOrEndResult.end_token => |t| {
804 if (t == null) {817 if (t == null) {
805 stack.push(State { .ParamDecl = fn_proto }) catch unreachable;818 stack.push(State { .ParamDecl = fn_proto }) catch unreachable;
...@@ -814,7 +827,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -814,7 +827,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
814 },827 },
815828
816 State.MaybeLabeledExpression => |ctx| {829 State.MaybeLabeledExpression => |ctx| {
817 if (eatToken(&tok_it, Token.Id.Colon)) |_| {830 if (eatToken(&tok_it, &tree, Token.Id.Colon)) |_| {
818 stack.push(State {831 stack.push(State {
819 .LabeledExpression = LabelCtx {832 .LabeledExpression = LabelCtx {
820 .label = ctx.label,833 .label = ctx.label,
...@@ -828,8 +841,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -828,8 +841,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
828 continue;841 continue;
829 },842 },
830 State.LabeledExpression => |ctx| {843 State.LabeledExpression => |ctx| {
831 const token_index = tok_it.index;844 const token = nextToken(&tok_it, &tree);
832 const token_ptr = ??tok_it.next();845 const token_index = token.index;
846 const token_ptr = token.ptr;
833 switch (token_ptr.id) {847 switch (token_ptr.id) {
834 Token.Id.LBrace => {848 Token.Id.LBrace => {
835 const block = try createToCtxNode(arena, ctx.opt_ctx, ast.Node.Block,849 const block = try createToCtxNode(arena, ctx.opt_ctx, ast.Node.Block,
...@@ -899,14 +913,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -899,14 +913,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
899 return tree;913 return tree;
900 }914 }
901915
902 _ = tok_it.prev();916 putBackToken(&tok_it, &tree);
903 continue;917 continue;
904 },918 },
905 }919 }
906 },920 },
907 State.Inline => |ctx| {921 State.Inline => |ctx| {
908 const token_index = tok_it.index;922 const token = nextToken(&tok_it, &tree);
909 const token_ptr = ??tok_it.next();923 const token_index = token.index;
924 const token_ptr = token.ptr;
910 switch (token_ptr.id) {925 switch (token_ptr.id) {
911 Token.Id.Keyword_while => {926 Token.Id.Keyword_while => {
912 stack.push(State {927 stack.push(State {
...@@ -938,7 +953,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -938,7 +953,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
938 return tree;953 return tree;
939 }954 }
940955
941 _ = tok_it.prev();956 putBackToken(&tok_it, &tree);
942 continue;957 continue;
943 },958 },
944 }959 }
...@@ -995,7 +1010,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -995,7 +1010,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
995 continue;1010 continue;
996 },1011 },
997 State.Else => |dest| {1012 State.Else => |dest| {
998 if (eatToken(&tok_it, Token.Id.Keyword_else)) |else_token| {1013 if (eatToken(&tok_it, &tree, Token.Id.Keyword_else)) |else_token| {
999 const node = try createNode(arena, ast.Node.Else,1014 const node = try createNode(arena, ast.Node.Else,
1000 ast.Node.Else {1015 ast.Node.Else {
1001 .base = undefined,1016 .base = undefined,
...@@ -1016,19 +1031,20 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1016,19 +1031,20 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
10161031
10171032
1018 State.Block => |block| {1033 State.Block => |block| {
1019 const token_index = tok_it.index;1034 const token = nextToken(&tok_it, &tree);
1020 const token_ptr = ??tok_it.next();1035 const token_index = token.index;
1036 const token_ptr = token.ptr;
1021 switch (token_ptr.id) {1037 switch (token_ptr.id) {
1022 Token.Id.RBrace => {1038 Token.Id.RBrace => {
1023 block.rbrace = token_index;1039 block.rbrace = token_index;
1024 continue;1040 continue;
1025 },1041 },
1026 else => {1042 else => {
1027 _ = tok_it.prev();1043 putBackToken(&tok_it, &tree);
1028 stack.push(State { .Block = block }) catch unreachable;1044 stack.push(State { .Block = block }) catch unreachable;
10291045
1030 var any_comments = false;1046 var any_comments = false;
1031 while (try eatLineComment(arena, &tok_it)) |line_comment| {1047 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
1032 try block.statements.push(&line_comment.base);1048 try block.statements.push(&line_comment.base);
1033 any_comments = true;1049 any_comments = true;
1034 }1050 }
...@@ -1040,8 +1056,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1040,8 +1056,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1040 }1056 }
1041 },1057 },
1042 State.Statement => |block| {1058 State.Statement => |block| {
1043 const token_index = tok_it.index;1059 const token = nextToken(&tok_it, &tree);
1044 const token_ptr = ??tok_it.next();1060 const token_index = token.index;
1061 const token_ptr = token.ptr;
1045 switch (token_ptr.id) {1062 switch (token_ptr.id) {
1046 Token.Id.Keyword_comptime => {1063 Token.Id.Keyword_comptime => {
1047 stack.push(State {1064 stack.push(State {
...@@ -1100,7 +1117,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1100,7 +1117,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1100 continue;1117 continue;
1101 },1118 },
1102 else => {1119 else => {
1103 _ = tok_it.prev();1120 putBackToken(&tok_it, &tree);
1104 const statement = try block.statements.addOne();1121 const statement = try block.statements.addOne();
1105 try stack.push(State { .Semicolon = statement });1122 try stack.push(State { .Semicolon = statement });
1106 try stack.push(State { .AssignmentExpressionBegin = OptionalCtx{ .Required = statement } });1123 try stack.push(State { .AssignmentExpressionBegin = OptionalCtx{ .Required = statement } });
...@@ -1109,8 +1126,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1109,8 +1126,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1109 }1126 }
1110 },1127 },
1111 State.ComptimeStatement => |ctx| {1128 State.ComptimeStatement => |ctx| {
1112 const token_index = tok_it.index;1129 const token = nextToken(&tok_it, &tree);
1113 const token_ptr = ??tok_it.next();1130 const token_index = token.index;
1131 const token_ptr = token.ptr;
1114 switch (token_ptr.id) {1132 switch (token_ptr.id) {
1115 Token.Id.Keyword_var, Token.Id.Keyword_const => {1133 Token.Id.Keyword_var, Token.Id.Keyword_const => {
1116 stack.push(State {1134 stack.push(State {
...@@ -1127,8 +1145,8 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1127,8 +1145,8 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1127 continue;1145 continue;
1128 },1146 },
1129 else => {1147 else => {
1130 _ = tok_it.prev();1148 putBackToken(&tok_it, &tree);
1131 _ = tok_it.prev();1149 putBackToken(&tok_it, &tree);
1132 const statement = try ctx.block.statements.addOne();1150 const statement = try ctx.block.statements.addOne();
1133 try stack.push(State { .Semicolon = statement });1151 try stack.push(State { .Semicolon = statement });
1134 try stack.push(State { .Expression = OptionalCtx { .Required = statement } });1152 try stack.push(State { .Expression = OptionalCtx { .Required = statement } });
...@@ -1146,10 +1164,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1146,10 +1164,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1146 },1164 },
11471165
1148 State.AsmOutputItems => |items| {1166 State.AsmOutputItems => |items| {
1149 const lbracket_index = tok_it.index;1167 const lbracket = nextToken(&tok_it, &tree);
1150 const lbracket_ptr = ??tok_it.next();1168 const lbracket_index = lbracket.index;
1169 const lbracket_ptr = lbracket.ptr;
1151 if (lbracket_ptr.id != Token.Id.LBracket) {1170 if (lbracket_ptr.id != Token.Id.LBracket) {
1152 _ = tok_it.prev();1171 putBackToken(&tok_it, &tree);
1153 continue;1172 continue;
1154 }1173 }
11551174
...@@ -1174,8 +1193,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1174,8 +1193,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1174 continue;1193 continue;
1175 },1194 },
1176 State.AsmOutputReturnOrType => |node| {1195 State.AsmOutputReturnOrType => |node| {
1177 const token_index = tok_it.index;1196 const token = nextToken(&tok_it, &tree);
1178 const token_ptr = ??tok_it.next();1197 const token_index = token.index;
1198 const token_ptr = token.ptr;
1179 switch (token_ptr.id) {1199 switch (token_ptr.id) {
1180 Token.Id.Identifier => {1200 Token.Id.Identifier => {
1181 node.kind = ast.Node.AsmOutput.Kind { .Variable = try createLiteral(arena, ast.Node.Identifier, token_index) };1201 node.kind = ast.Node.AsmOutput.Kind { .Variable = try createLiteral(arena, ast.Node.Identifier, token_index) };
...@@ -1197,10 +1217,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1197,10 +1217,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1197 }1217 }
1198 },1218 },
1199 State.AsmInputItems => |items| {1219 State.AsmInputItems => |items| {
1200 const lbracket_index = tok_it.index;1220 const lbracket = nextToken(&tok_it, &tree);
1201 const lbracket_ptr = ??tok_it.next();1221 const lbracket_index = lbracket.index;
1222 const lbracket_ptr = lbracket.ptr;
1202 if (lbracket_ptr.id != Token.Id.LBracket) {1223 if (lbracket_ptr.id != Token.Id.LBracket) {
1203 _ = tok_it.prev();1224 putBackToken(&tok_it, &tree);
1204 continue;1225 continue;
1205 }1226 }
12061227
...@@ -1233,7 +1254,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1233,7 +1254,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
12331254
12341255
1235 State.ExprListItemOrEnd => |list_state| {1256 State.ExprListItemOrEnd => |list_state| {
1236 if (eatToken(&tok_it, list_state.end)) |token_index| {1257 if (eatToken(&tok_it, &tree, list_state.end)) |token_index| {
1237 *list_state.ptr = token_index;1258 *list_state.ptr = token_index;
1238 continue;1259 continue;
1239 }1260 }
...@@ -1243,7 +1264,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1243,7 +1264,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1243 continue;1264 continue;
1244 },1265 },
1245 State.ExprListCommaOrEnd => |list_state| {1266 State.ExprListCommaOrEnd => |list_state| {
1246 switch (expectCommaOrEnd(&tok_it, list_state.end)) {1267 switch (expectCommaOrEnd(&tok_it, &tree, list_state.end)) {
1247 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {1268 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
1248 *list_state.ptr = end;1269 *list_state.ptr = end;
1249 continue;1270 continue;
...@@ -1258,11 +1279,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1258,11 +1279,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1258 }1279 }
1259 },1280 },
1260 State.FieldInitListItemOrEnd => |list_state| {1281 State.FieldInitListItemOrEnd => |list_state| {
1261 while (try eatLineComment(arena, &tok_it)) |line_comment| {1282 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
1262 try list_state.list.push(&line_comment.base);1283 try list_state.list.push(&line_comment.base);
1263 }1284 }
12641285
1265 if (eatToken(&tok_it, Token.Id.RBrace)) |rbrace| {1286 if (eatToken(&tok_it, &tree, Token.Id.RBrace)) |rbrace| {
1266 *list_state.ptr = rbrace;1287 *list_state.ptr = rbrace;
1267 continue;1288 continue;
1268 }1289 }
...@@ -1295,7 +1316,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1295,7 +1316,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1295 continue;1316 continue;
1296 },1317 },
1297 State.FieldInitListCommaOrEnd => |list_state| {1318 State.FieldInitListCommaOrEnd => |list_state| {
1298 switch (expectCommaOrEnd(&tok_it, Token.Id.RBrace)) {1319 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
1299 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {1320 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
1300 *list_state.ptr = end;1321 *list_state.ptr = end;
1301 continue;1322 continue;
...@@ -1310,7 +1331,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1310,7 +1331,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1310 }1331 }
1311 },1332 },
1312 State.FieldListCommaOrEnd => |container_decl| {1333 State.FieldListCommaOrEnd => |container_decl| {
1313 switch (expectCommaOrEnd(&tok_it, Token.Id.RBrace)) {1334 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
1314 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {1335 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
1315 container_decl.rbrace_token = end;1336 container_decl.rbrace_token = end;
1316 continue;1337 continue;
...@@ -1325,11 +1346,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1325,11 +1346,11 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1325 }1346 }
1326 },1347 },
1327 State.ErrorTagListItemOrEnd => |list_state| {1348 State.ErrorTagListItemOrEnd => |list_state| {
1328 while (try eatLineComment(arena, &tok_it)) |line_comment| {1349 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
1329 try list_state.list.push(&line_comment.base);1350 try list_state.list.push(&line_comment.base);
1330 }1351 }
13311352
1332 if (eatToken(&tok_it, Token.Id.RBrace)) |rbrace| {1353 if (eatToken(&tok_it, &tree, Token.Id.RBrace)) |rbrace| {
1333 *list_state.ptr = rbrace;1354 *list_state.ptr = rbrace;
1334 continue;1355 continue;
1335 }1356 }
...@@ -1341,7 +1362,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1341,7 +1362,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1341 continue;1362 continue;
1342 },1363 },
1343 State.ErrorTagListCommaOrEnd => |list_state| {1364 State.ErrorTagListCommaOrEnd => |list_state| {
1344 switch (expectCommaOrEnd(&tok_it, Token.Id.RBrace)) {1365 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RBrace)) {
1345 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {1366 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
1346 *list_state.ptr = end;1367 *list_state.ptr = end;
1347 continue;1368 continue;
...@@ -1356,16 +1377,16 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1356,16 +1377,16 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1356 }1377 }
1357 },1378 },
1358 State.SwitchCaseOrEnd => |list_state| {1379 State.SwitchCaseOrEnd => |list_state| {
1359 while (try eatLineComment(arena, &tok_it)) |line_comment| {1380 while (try eatLineComment(arena, &tok_it, &tree)) |line_comment| {
1360 try list_state.list.push(&line_comment.base);1381 try list_state.list.push(&line_comment.base);
1361 }1382 }
13621383
1363 if (eatToken(&tok_it, Token.Id.RBrace)) |rbrace| {1384 if (eatToken(&tok_it, &tree, Token.Id.RBrace)) |rbrace| {
1364 *list_state.ptr = rbrace;1385 *list_state.ptr = rbrace;
1365 continue;1386 continue;
1366 }1387 }
13671388
1368 const comments = try eatDocComments(arena, &tok_it);1389 const comments = try eatDocComments(arena, &tok_it, &tree);
1369 const node = try arena.construct(ast.Node.SwitchCase {1390 const node = try arena.construct(ast.Node.SwitchCase {
1370 .base = ast.Node {1391 .base = ast.Node {
1371 .id = ast.Node.Id.SwitchCase,1392 .id = ast.Node.Id.SwitchCase,
...@@ -1384,7 +1405,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1384,7 +1405,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1384 },1405 },
13851406
1386 State.SwitchCaseCommaOrEnd => |list_state| {1407 State.SwitchCaseCommaOrEnd => |list_state| {
1387 switch (expectCommaOrEnd(&tok_it, Token.Id.RParen)) {1408 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.RParen)) {
1388 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {1409 ExpectCommaOrEndResult.end_token => |maybe_end| if (maybe_end) |end| {
1389 *list_state.ptr = end;1410 *list_state.ptr = end;
1390 continue;1411 continue;
...@@ -1400,8 +1421,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1400,8 +1421,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1400 },1421 },
14011422
1402 State.SwitchCaseFirstItem => |case_items| {1423 State.SwitchCaseFirstItem => |case_items| {
1403 const token_index = tok_it.index;1424 const token = nextToken(&tok_it, &tree);
1404 const token_ptr = ??tok_it.next();1425 const token_index = token.index;
1426 const token_ptr = token.ptr;
1405 if (token_ptr.id == Token.Id.Keyword_else) {1427 if (token_ptr.id == Token.Id.Keyword_else) {
1406 const else_node = try arena.construct(ast.Node.SwitchElse {1428 const else_node = try arena.construct(ast.Node.SwitchElse {
1407 .base = ast.Node{ .id = ast.Node.Id.SwitchElse},1429 .base = ast.Node{ .id = ast.Node.Id.SwitchElse},
...@@ -1412,7 +1434,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1412,7 +1434,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1412 try stack.push(State { .ExpectToken = Token.Id.EqualAngleBracketRight });1434 try stack.push(State { .ExpectToken = Token.Id.EqualAngleBracketRight });
1413 continue;1435 continue;
1414 } else {1436 } else {
1415 _ = tok_it.prev();1437 putBackToken(&tok_it, &tree);
1416 try stack.push(State { .SwitchCaseItem = case_items });1438 try stack.push(State { .SwitchCaseItem = case_items });
1417 continue;1439 continue;
1418 }1440 }
...@@ -1422,7 +1444,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1422,7 +1444,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1422 try stack.push(State { .RangeExpressionBegin = OptionalCtx { .Required = try case_items.addOne() } });1444 try stack.push(State { .RangeExpressionBegin = OptionalCtx { .Required = try case_items.addOne() } });
1423 },1445 },
1424 State.SwitchCaseItemCommaOrEnd => |case_items| {1446 State.SwitchCaseItemCommaOrEnd => |case_items| {
1425 switch (expectCommaOrEnd(&tok_it, Token.Id.EqualAngleBracketRight)) {1447 switch (expectCommaOrEnd(&tok_it, &tree, Token.Id.EqualAngleBracketRight)) {
1426 ExpectCommaOrEndResult.end_token => |t| {1448 ExpectCommaOrEndResult.end_token => |t| {
1427 if (t == null) {1449 if (t == null) {
1428 stack.push(State { .SwitchCaseItem = case_items }) catch unreachable;1450 stack.push(State { .SwitchCaseItem = case_items }) catch unreachable;
...@@ -1445,7 +1467,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1445,7 +1467,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1445 continue;1467 continue;
1446 },1468 },
1447 State.AsyncAllocator => |async_node| {1469 State.AsyncAllocator => |async_node| {
1448 if (eatToken(&tok_it, Token.Id.AngleBracketLeft) == null) {1470 if (eatToken(&tok_it, &tree, Token.Id.AngleBracketLeft) == null) {
1449 continue;1471 continue;
1450 }1472 }
14511473
...@@ -1491,7 +1513,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1491,7 +1513,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
14911513
14921514
1493 State.ExternType => |ctx| {1515 State.ExternType => |ctx| {
1494 if (eatToken(&tok_it, Token.Id.Keyword_fn)) |fn_token| {1516 if (eatToken(&tok_it, &tree, Token.Id.Keyword_fn)) |fn_token| {
1495 const fn_proto = try arena.construct(ast.Node.FnProto {1517 const fn_proto = try arena.construct(ast.Node.FnProto {
1496 .base = ast.Node {1518 .base = ast.Node {
1497 .id = ast.Node.Id.FnProto,1519 .id = ast.Node.Id.FnProto,
...@@ -1525,8 +1547,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1525,8 +1547,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1525 continue;1547 continue;
1526 },1548 },
1527 State.SliceOrArrayAccess => |node| {1549 State.SliceOrArrayAccess => |node| {
1528 const token_index = tok_it.index;1550 const token = nextToken(&tok_it, &tree);
1529 const token_ptr = ??tok_it.next();1551 const token_index = token.index;
1552 const token_ptr = token.ptr;
1530 switch (token_ptr.id) {1553 switch (token_ptr.id) {
1531 Token.Id.Ellipsis2 => {1554 Token.Id.Ellipsis2 => {
1532 const start = node.op.ArrayAccess;1555 const start = node.op.ArrayAccess;
...@@ -1559,7 +1582,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1559,7 +1582,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1559 }1582 }
1560 },1583 },
1561 State.SliceOrArrayType => |node| {1584 State.SliceOrArrayType => |node| {
1562 if (eatToken(&tok_it, Token.Id.RBracket)) |_| {1585 if (eatToken(&tok_it, &tree, Token.Id.RBracket)) |_| {
1563 node.op = ast.Node.PrefixOp.Op {1586 node.op = ast.Node.PrefixOp.Op {
1564 .SliceType = ast.Node.PrefixOp.AddrOfInfo {1587 .SliceType = ast.Node.PrefixOp.AddrOfInfo {
1565 .align_expr = null,1588 .align_expr = null,
...@@ -1581,8 +1604,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1581,8 +1604,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1581 continue;1604 continue;
1582 },1605 },
1583 State.AddrOfModifiers => |addr_of_info| {1606 State.AddrOfModifiers => |addr_of_info| {
1584 const token_index = tok_it.index;1607 const token = nextToken(&tok_it, &tree);
1585 const token_ptr = ??tok_it.next();1608 const token_index = token.index;
1609 const token_ptr = token.ptr;
1586 switch (token_ptr.id) {1610 switch (token_ptr.id) {
1587 Token.Id.Keyword_align => {1611 Token.Id.Keyword_align => {
1588 stack.push(state) catch unreachable;1612 stack.push(state) catch unreachable;
...@@ -1620,7 +1644,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1620,7 +1644,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1620 continue;1644 continue;
1621 },1645 },
1622 else => {1646 else => {
1623 _ = tok_it.prev();1647 putBackToken(&tok_it, &tree);
1624 continue;1648 continue;
1625 },1649 },
1626 }1650 }
...@@ -1628,8 +1652,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1628,8 +1652,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
16281652
16291653
1630 State.Payload => |opt_ctx| {1654 State.Payload => |opt_ctx| {
1631 const token_index = tok_it.index;1655 const token = nextToken(&tok_it, &tree);
1632 const token_ptr = ??tok_it.next();1656 const token_index = token.index;
1657 const token_ptr = token.ptr;
1633 if (token_ptr.id != Token.Id.Pipe) {1658 if (token_ptr.id != Token.Id.Pipe) {
1634 if (opt_ctx != OptionalCtx.Optional) {1659 if (opt_ctx != OptionalCtx.Optional) {
1635 *(try tree.errors.addOne()) = Error {1660 *(try tree.errors.addOne()) = Error {
...@@ -1641,7 +1666,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1641,7 +1666,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1641 return tree;1666 return tree;
1642 }1667 }
16431668
1644 _ = tok_it.prev();1669 putBackToken(&tok_it, &tree);
1645 continue;1670 continue;
1646 }1671 }
16471672
...@@ -1664,8 +1689,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1664,8 +1689,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1664 continue;1689 continue;
1665 },1690 },
1666 State.PointerPayload => |opt_ctx| {1691 State.PointerPayload => |opt_ctx| {
1667 const token_index = tok_it.index;1692 const token = nextToken(&tok_it, &tree);
1668 const token_ptr = ??tok_it.next();1693 const token_index = token.index;
1694 const token_ptr = token.ptr;
1669 if (token_ptr.id != Token.Id.Pipe) {1695 if (token_ptr.id != Token.Id.Pipe) {
1670 if (opt_ctx != OptionalCtx.Optional) {1696 if (opt_ctx != OptionalCtx.Optional) {
1671 *(try tree.errors.addOne()) = Error {1697 *(try tree.errors.addOne()) = Error {
...@@ -1677,7 +1703,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1677,7 +1703,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1677 return tree;1703 return tree;
1678 }1704 }
16791705
1680 _ = tok_it.prev();1706 putBackToken(&tok_it, &tree);
1681 continue;1707 continue;
1682 }1708 }
16831709
...@@ -1707,8 +1733,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1707,8 +1733,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1707 continue;1733 continue;
1708 },1734 },
1709 State.PointerIndexPayload => |opt_ctx| {1735 State.PointerIndexPayload => |opt_ctx| {
1710 const token_index = tok_it.index;1736 const token = nextToken(&tok_it, &tree);
1711 const token_ptr = ??tok_it.next();1737 const token_index = token.index;
1738 const token_ptr = token.ptr;
1712 if (token_ptr.id != Token.Id.Pipe) {1739 if (token_ptr.id != Token.Id.Pipe) {
1713 if (opt_ctx != OptionalCtx.Optional) {1740 if (opt_ctx != OptionalCtx.Optional) {
1714 *(try tree.errors.addOne()) = Error {1741 *(try tree.errors.addOne()) = Error {
...@@ -1720,7 +1747,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1720,7 +1747,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1720 return tree;1747 return tree;
1721 }1748 }
17221749
1723 _ = tok_it.prev();1750 putBackToken(&tok_it, &tree);
1724 continue;1751 continue;
1725 }1752 }
17261753
...@@ -1755,8 +1782,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1755,8 +1782,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
17551782
17561783
1757 State.Expression => |opt_ctx| {1784 State.Expression => |opt_ctx| {
1758 const token_index = tok_it.index;1785 const token = nextToken(&tok_it, &tree);
1759 const token_ptr = ??tok_it.next();1786 const token_index = token.index;
1787 const token_ptr = token.ptr;
1760 switch (token_ptr.id) {1788 switch (token_ptr.id) {
1761 Token.Id.Keyword_return, Token.Id.Keyword_break, Token.Id.Keyword_continue => {1789 Token.Id.Keyword_return, Token.Id.Keyword_break, Token.Id.Keyword_continue => {
1762 const node = try createToCtxNode(arena, opt_ctx, ast.Node.ControlFlowExpression,1790 const node = try createToCtxNode(arena, opt_ctx, ast.Node.ControlFlowExpression,
...@@ -1808,7 +1836,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1808,7 +1836,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1808 },1836 },
1809 else => {1837 else => {
1810 if (!try parseBlockExpr(&stack, arena, opt_ctx, token_ptr, token_index)) {1838 if (!try parseBlockExpr(&stack, arena, opt_ctx, token_ptr, token_index)) {
1811 _ = tok_it.prev();1839 putBackToken(&tok_it, &tree);
1812 stack.push(State { .UnwrapExpressionBegin = opt_ctx }) catch unreachable;1840 stack.push(State { .UnwrapExpressionBegin = opt_ctx }) catch unreachable;
1813 }1841 }
1814 continue;1842 continue;
...@@ -1823,7 +1851,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1823,7 +1851,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1823 State.RangeExpressionEnd => |opt_ctx| {1851 State.RangeExpressionEnd => |opt_ctx| {
1824 const lhs = opt_ctx.get() ?? continue;1852 const lhs = opt_ctx.get() ?? continue;
18251853
1826 if (eatToken(&tok_it, Token.Id.Ellipsis3)) |ellipsis3| {1854 if (eatToken(&tok_it, &tree, Token.Id.Ellipsis3)) |ellipsis3| {
1827 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,1855 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1828 ast.Node.InfixOp {1856 ast.Node.InfixOp {
1829 .base = undefined,1857 .base = undefined,
...@@ -1846,8 +1874,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1846,8 +1874,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1846 State.AssignmentExpressionEnd => |opt_ctx| {1874 State.AssignmentExpressionEnd => |opt_ctx| {
1847 const lhs = opt_ctx.get() ?? continue;1875 const lhs = opt_ctx.get() ?? continue;
18481876
1849 const token_index = tok_it.index;1877 const token = nextToken(&tok_it, &tree);
1850 const token_ptr = ??tok_it.next();1878 const token_index = token.index;
1879 const token_ptr = token.ptr;
1851 if (tokenIdToAssignment(token_ptr.id)) |ass_id| {1880 if (tokenIdToAssignment(token_ptr.id)) |ass_id| {
1852 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,1881 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1853 ast.Node.InfixOp {1882 ast.Node.InfixOp {
...@@ -1862,7 +1891,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1862,7 +1891,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1862 try stack.push(State { .Expression = OptionalCtx { .Required = &node.rhs } });1891 try stack.push(State { .Expression = OptionalCtx { .Required = &node.rhs } });
1863 continue;1892 continue;
1864 } else {1893 } else {
1865 _ = tok_it.prev();1894 putBackToken(&tok_it, &tree);
1866 continue;1895 continue;
1867 }1896 }
1868 },1897 },
...@@ -1876,8 +1905,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1876,8 +1905,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1876 State.UnwrapExpressionEnd => |opt_ctx| {1905 State.UnwrapExpressionEnd => |opt_ctx| {
1877 const lhs = opt_ctx.get() ?? continue;1906 const lhs = opt_ctx.get() ?? continue;
18781907
1879 const token_index = tok_it.index;1908 const token = nextToken(&tok_it, &tree);
1880 const token_ptr = ??tok_it.next();1909 const token_index = token.index;
1910 const token_ptr = token.ptr;
1881 if (tokenIdToUnwrapExpr(token_ptr.id)) |unwrap_id| {1911 if (tokenIdToUnwrapExpr(token_ptr.id)) |unwrap_id| {
1882 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,1912 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1883 ast.Node.InfixOp {1913 ast.Node.InfixOp {
...@@ -1897,7 +1927,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1897,7 +1927,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1897 }1927 }
1898 continue;1928 continue;
1899 } else {1929 } else {
1900 _ = tok_it.prev();1930 putBackToken(&tok_it, &tree);
1901 continue;1931 continue;
1902 }1932 }
1903 },1933 },
...@@ -1911,7 +1941,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1911,7 +1941,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1911 State.BoolOrExpressionEnd => |opt_ctx| {1941 State.BoolOrExpressionEnd => |opt_ctx| {
1912 const lhs = opt_ctx.get() ?? continue;1942 const lhs = opt_ctx.get() ?? continue;
19131943
1914 if (eatToken(&tok_it, Token.Id.Keyword_or)) |or_token| {1944 if (eatToken(&tok_it, &tree, Token.Id.Keyword_or)) |or_token| {
1915 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,1945 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1916 ast.Node.InfixOp {1946 ast.Node.InfixOp {
1917 .base = undefined,1947 .base = undefined,
...@@ -1936,7 +1966,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1936,7 +1966,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1936 State.BoolAndExpressionEnd => |opt_ctx| {1966 State.BoolAndExpressionEnd => |opt_ctx| {
1937 const lhs = opt_ctx.get() ?? continue;1967 const lhs = opt_ctx.get() ?? continue;
19381968
1939 if (eatToken(&tok_it, Token.Id.Keyword_and)) |and_token| {1969 if (eatToken(&tok_it, &tree, Token.Id.Keyword_and)) |and_token| {
1940 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,1970 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1941 ast.Node.InfixOp {1971 ast.Node.InfixOp {
1942 .base = undefined,1972 .base = undefined,
...@@ -1961,8 +1991,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1961,8 +1991,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1961 State.ComparisonExpressionEnd => |opt_ctx| {1991 State.ComparisonExpressionEnd => |opt_ctx| {
1962 const lhs = opt_ctx.get() ?? continue;1992 const lhs = opt_ctx.get() ?? continue;
19631993
1964 const token_index = tok_it.index;1994 const token = nextToken(&tok_it, &tree);
1965 const token_ptr = ??tok_it.next();1995 const token_index = token.index;
1996 const token_ptr = token.ptr;
1966 if (tokenIdToComparison(token_ptr.id)) |comp_id| {1997 if (tokenIdToComparison(token_ptr.id)) |comp_id| {
1967 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,1998 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1968 ast.Node.InfixOp {1999 ast.Node.InfixOp {
...@@ -1977,7 +2008,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1977,7 +2008,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1977 try stack.push(State { .BinaryOrExpressionBegin = OptionalCtx { .Required = &node.rhs } });2008 try stack.push(State { .BinaryOrExpressionBegin = OptionalCtx { .Required = &node.rhs } });
1978 continue;2009 continue;
1979 } else {2010 } else {
1980 _ = tok_it.prev();2011 putBackToken(&tok_it, &tree);
1981 continue;2012 continue;
1982 }2013 }
1983 },2014 },
...@@ -1991,7 +2022,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -1991,7 +2022,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
1991 State.BinaryOrExpressionEnd => |opt_ctx| {2022 State.BinaryOrExpressionEnd => |opt_ctx| {
1992 const lhs = opt_ctx.get() ?? continue;2023 const lhs = opt_ctx.get() ?? continue;
19932024
1994 if (eatToken(&tok_it, Token.Id.Pipe)) |pipe| {2025 if (eatToken(&tok_it, &tree, Token.Id.Pipe)) |pipe| {
1995 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2026 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
1996 ast.Node.InfixOp {2027 ast.Node.InfixOp {
1997 .base = undefined,2028 .base = undefined,
...@@ -2016,7 +2047,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2016,7 +2047,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2016 State.BinaryXorExpressionEnd => |opt_ctx| {2047 State.BinaryXorExpressionEnd => |opt_ctx| {
2017 const lhs = opt_ctx.get() ?? continue;2048 const lhs = opt_ctx.get() ?? continue;
20182049
2019 if (eatToken(&tok_it, Token.Id.Caret)) |caret| {2050 if (eatToken(&tok_it, &tree, Token.Id.Caret)) |caret| {
2020 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2051 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
2021 ast.Node.InfixOp {2052 ast.Node.InfixOp {
2022 .base = undefined,2053 .base = undefined,
...@@ -2041,7 +2072,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2041,7 +2072,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2041 State.BinaryAndExpressionEnd => |opt_ctx| {2072 State.BinaryAndExpressionEnd => |opt_ctx| {
2042 const lhs = opt_ctx.get() ?? continue;2073 const lhs = opt_ctx.get() ?? continue;
20432074
2044 if (eatToken(&tok_it, Token.Id.Ampersand)) |ampersand| {2075 if (eatToken(&tok_it, &tree, Token.Id.Ampersand)) |ampersand| {
2045 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2076 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
2046 ast.Node.InfixOp {2077 ast.Node.InfixOp {
2047 .base = undefined,2078 .base = undefined,
...@@ -2066,8 +2097,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2066,8 +2097,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2066 State.BitShiftExpressionEnd => |opt_ctx| {2097 State.BitShiftExpressionEnd => |opt_ctx| {
2067 const lhs = opt_ctx.get() ?? continue;2098 const lhs = opt_ctx.get() ?? continue;
20682099
2069 const token_index = tok_it.index;2100 const token = nextToken(&tok_it, &tree);
2070 const token_ptr = ??tok_it.next();2101 const token_index = token.index;
2102 const token_ptr = token.ptr;
2071 if (tokenIdToBitShift(token_ptr.id)) |bitshift_id| {2103 if (tokenIdToBitShift(token_ptr.id)) |bitshift_id| {
2072 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2104 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
2073 ast.Node.InfixOp {2105 ast.Node.InfixOp {
...@@ -2082,7 +2114,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2082,7 +2114,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2082 try stack.push(State { .AdditionExpressionBegin = OptionalCtx { .Required = &node.rhs } });2114 try stack.push(State { .AdditionExpressionBegin = OptionalCtx { .Required = &node.rhs } });
2083 continue;2115 continue;
2084 } else {2116 } else {
2085 _ = tok_it.prev();2117 putBackToken(&tok_it, &tree);
2086 continue;2118 continue;
2087 }2119 }
2088 },2120 },
...@@ -2096,8 +2128,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2096,8 +2128,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2096 State.AdditionExpressionEnd => |opt_ctx| {2128 State.AdditionExpressionEnd => |opt_ctx| {
2097 const lhs = opt_ctx.get() ?? continue;2129 const lhs = opt_ctx.get() ?? continue;
20982130
2099 const token_index = tok_it.index;2131 const token = nextToken(&tok_it, &tree);
2100 const token_ptr = ??tok_it.next();2132 const token_index = token.index;
2133 const token_ptr = token.ptr;
2101 if (tokenIdToAddition(token_ptr.id)) |add_id| {2134 if (tokenIdToAddition(token_ptr.id)) |add_id| {
2102 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2135 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
2103 ast.Node.InfixOp {2136 ast.Node.InfixOp {
...@@ -2112,7 +2145,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2112,7 +2145,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2112 try stack.push(State { .MultiplyExpressionBegin = OptionalCtx { .Required = &node.rhs } });2145 try stack.push(State { .MultiplyExpressionBegin = OptionalCtx { .Required = &node.rhs } });
2113 continue;2146 continue;
2114 } else {2147 } else {
2115 _ = tok_it.prev();2148 putBackToken(&tok_it, &tree);
2116 continue;2149 continue;
2117 }2150 }
2118 },2151 },
...@@ -2126,8 +2159,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2126,8 +2159,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2126 State.MultiplyExpressionEnd => |opt_ctx| {2159 State.MultiplyExpressionEnd => |opt_ctx| {
2127 const lhs = opt_ctx.get() ?? continue;2160 const lhs = opt_ctx.get() ?? continue;
21282161
2129 const token_index = tok_it.index;2162 const token = nextToken(&tok_it, &tree);
2130 const token_ptr = ??tok_it.next();2163 const token_index = token.index;
2164 const token_ptr = token.ptr;
2131 if (tokenIdToMultiply(token_ptr.id)) |mult_id| {2165 if (tokenIdToMultiply(token_ptr.id)) |mult_id| {
2132 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2166 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
2133 ast.Node.InfixOp {2167 ast.Node.InfixOp {
...@@ -2142,7 +2176,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2142,7 +2176,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2142 try stack.push(State { .CurlySuffixExpressionBegin = OptionalCtx { .Required = &node.rhs } });2176 try stack.push(State { .CurlySuffixExpressionBegin = OptionalCtx { .Required = &node.rhs } });
2143 continue;2177 continue;
2144 } else {2178 } else {
2145 _ = tok_it.prev();2179 putBackToken(&tok_it, &tree);
2146 continue;2180 continue;
2147 }2181 }
2148 },2182 },
...@@ -2210,7 +2244,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2210,7 +2244,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2210 State.TypeExprEnd => |opt_ctx| {2244 State.TypeExprEnd => |opt_ctx| {
2211 const lhs = opt_ctx.get() ?? continue;2245 const lhs = opt_ctx.get() ?? continue;
22122246
2213 if (eatToken(&tok_it, Token.Id.Bang)) |bang| {2247 if (eatToken(&tok_it, &tree, Token.Id.Bang)) |bang| {
2214 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,2248 const node = try createToCtxNode(arena, opt_ctx, ast.Node.InfixOp,
2215 ast.Node.InfixOp {2249 ast.Node.InfixOp {
2216 .base = undefined,2250 .base = undefined,
...@@ -2227,8 +2261,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2227,8 +2261,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2227 },2261 },
22282262
2229 State.PrefixOpExpression => |opt_ctx| {2263 State.PrefixOpExpression => |opt_ctx| {
2230 const token_index = tok_it.index;2264 const token = nextToken(&tok_it, &tree);
2231 const token_ptr = ??tok_it.next();2265 const token_index = token.index;
2266 const token_ptr = token.ptr;
2232 if (tokenIdToPrefixOp(token_ptr.id)) |prefix_id| {2267 if (tokenIdToPrefixOp(token_ptr.id)) |prefix_id| {
2233 var node = try createToCtxNode(arena, opt_ctx, ast.Node.PrefixOp,2268 var node = try createToCtxNode(arena, opt_ctx, ast.Node.PrefixOp,
2234 ast.Node.PrefixOp {2269 ast.Node.PrefixOp {
...@@ -2259,14 +2294,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2259,14 +2294,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2259 }2294 }
2260 continue;2295 continue;
2261 } else {2296 } else {
2262 _ = tok_it.prev();2297 putBackToken(&tok_it, &tree);
2263 stack.push(State { .SuffixOpExpressionBegin = opt_ctx }) catch unreachable;2298 stack.push(State { .SuffixOpExpressionBegin = opt_ctx }) catch unreachable;
2264 continue;2299 continue;
2265 }2300 }
2266 },2301 },
22672302
2268 State.SuffixOpExpressionBegin => |opt_ctx| {2303 State.SuffixOpExpressionBegin => |opt_ctx| {
2269 if (eatToken(&tok_it, Token.Id.Keyword_async)) |async_token| {2304 if (eatToken(&tok_it, &tree, Token.Id.Keyword_async)) |async_token| {
2270 const async_node = try createNode(arena, ast.Node.AsyncAttribute,2305 const async_node = try createNode(arena, ast.Node.AsyncAttribute,
2271 ast.Node.AsyncAttribute {2306 ast.Node.AsyncAttribute {
2272 .base = undefined,2307 .base = undefined,
...@@ -2295,8 +2330,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2295,8 +2330,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2295 State.SuffixOpExpressionEnd => |opt_ctx| {2330 State.SuffixOpExpressionEnd => |opt_ctx| {
2296 const lhs = opt_ctx.get() ?? continue;2331 const lhs = opt_ctx.get() ?? continue;
22972332
2298 const token_index = tok_it.index;2333 const token = nextToken(&tok_it, &tree);
2299 const token_ptr = ??tok_it.next();2334 const token_index = token.index;
2335 const token_ptr = token.ptr;
2300 switch (token_ptr.id) {2336 switch (token_ptr.id) {
2301 Token.Id.LParen => {2337 Token.Id.LParen => {
2302 const node = try createToCtxNode(arena, opt_ctx, ast.Node.SuffixOp,2338 const node = try createToCtxNode(arena, opt_ctx, ast.Node.SuffixOp,
...@@ -2353,50 +2389,49 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2353,50 +2389,49 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2353 continue;2389 continue;
2354 },2390 },
2355 else => {2391 else => {
2356 _ = tok_it.prev();2392 putBackToken(&tok_it, &tree);
2357 continue;2393 continue;
2358 },2394 },
2359 }2395 }
2360 },2396 },
23612397
2362 State.PrimaryExpression => |opt_ctx| {2398 State.PrimaryExpression => |opt_ctx| {
2363 const token_index = tok_it.index;2399 const token = nextToken(&tok_it, &tree);
2364 const token_ptr = ??tok_it.next();2400 switch (token.ptr.id) {
2365 switch (token_ptr.id) {
2366 Token.Id.IntegerLiteral => {2401 Token.Id.IntegerLiteral => {
2367 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.StringLiteral, token_index);2402 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.StringLiteral, token.index);
2368 continue;2403 continue;
2369 },2404 },
2370 Token.Id.FloatLiteral => {2405 Token.Id.FloatLiteral => {
2371 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.FloatLiteral, token_index);2406 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.FloatLiteral, token.index);
2372 continue;2407 continue;
2373 },2408 },
2374 Token.Id.CharLiteral => {2409 Token.Id.CharLiteral => {
2375 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.CharLiteral, token_index);2410 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.CharLiteral, token.index);
2376 continue;2411 continue;
2377 },2412 },
2378 Token.Id.Keyword_undefined => {2413 Token.Id.Keyword_undefined => {
2379 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.UndefinedLiteral, token_index);2414 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.UndefinedLiteral, token.index);
2380 continue;2415 continue;
2381 },2416 },
2382 Token.Id.Keyword_true, Token.Id.Keyword_false => {2417 Token.Id.Keyword_true, Token.Id.Keyword_false => {
2383 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.BoolLiteral, token_index);2418 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.BoolLiteral, token.index);
2384 continue;2419 continue;
2385 },2420 },
2386 Token.Id.Keyword_null => {2421 Token.Id.Keyword_null => {
2387 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.NullLiteral, token_index);2422 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.NullLiteral, token.index);
2388 continue;2423 continue;
2389 },2424 },
2390 Token.Id.Keyword_this => {2425 Token.Id.Keyword_this => {
2391 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.ThisLiteral, token_index);2426 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.ThisLiteral, token.index);
2392 continue;2427 continue;
2393 },2428 },
2394 Token.Id.Keyword_var => {2429 Token.Id.Keyword_var => {
2395 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.VarType, token_index);2430 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.VarType, token.index);
2396 continue;2431 continue;
2397 },2432 },
2398 Token.Id.Keyword_unreachable => {2433 Token.Id.Keyword_unreachable => {
2399 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Unreachable, token_index);2434 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Unreachable, token.index);
2400 continue;2435 continue;
2401 },2436 },
2402 Token.Id.Keyword_promise => {2437 Token.Id.Keyword_promise => {
...@@ -2404,14 +2439,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2404,14 +2439,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2404 .base = ast.Node {2439 .base = ast.Node {
2405 .id = ast.Node.Id.PromiseType,2440 .id = ast.Node.Id.PromiseType,
2406 },2441 },
2407 .promise_token = token_index,2442 .promise_token = token.index,
2408 .result = null,2443 .result = null,
2409 });2444 });
2410 opt_ctx.store(&node.base);2445 opt_ctx.store(&node.base);
2411 const next_token_index = tok_it.index;2446 const next_token = nextToken(&tok_it, &tree);
2412 const next_token_ptr = ??tok_it.next();2447 const next_token_index = next_token.index;
2448 const next_token_ptr = next_token.ptr;
2413 if (next_token_ptr.id != Token.Id.Arrow) {2449 if (next_token_ptr.id != Token.Id.Arrow) {
2414 _ = tok_it.prev();2450 putBackToken(&tok_it, &tree);
2415 continue;2451 continue;
2416 }2452 }
2417 node.result = ast.Node.PromiseType.Result {2453 node.result = ast.Node.PromiseType.Result {
...@@ -2423,14 +2459,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2423,14 +2459,14 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2423 continue;2459 continue;
2424 },2460 },
2425 Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => {2461 Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => {
2426 opt_ctx.store((try parseStringLiteral(arena, &tok_it, token_ptr, token_index)) ?? unreachable);2462 opt_ctx.store((try parseStringLiteral(arena, &tok_it, token.ptr, token.index, &tree)) ?? unreachable);
2427 continue;2463 continue;
2428 },2464 },
2429 Token.Id.LParen => {2465 Token.Id.LParen => {
2430 const node = try createToCtxNode(arena, opt_ctx, ast.Node.GroupedExpression,2466 const node = try createToCtxNode(arena, opt_ctx, ast.Node.GroupedExpression,
2431 ast.Node.GroupedExpression {2467 ast.Node.GroupedExpression {
2432 .base = undefined,2468 .base = undefined,
2433 .lparen = token_index,2469 .lparen = token.index,
2434 .expr = undefined,2470 .expr = undefined,
2435 .rparen = undefined,2471 .rparen = undefined,
2436 }2472 }
...@@ -2448,7 +2484,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2448,7 +2484,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2448 const node = try createToCtxNode(arena, opt_ctx, ast.Node.BuiltinCall,2484 const node = try createToCtxNode(arena, opt_ctx, ast.Node.BuiltinCall,
2449 ast.Node.BuiltinCall {2485 ast.Node.BuiltinCall {
2450 .base = undefined,2486 .base = undefined,
2451 .builtin_token = token_index,2487 .builtin_token = token.index,
2452 .params = ast.Node.BuiltinCall.ParamList.init(arena),2488 .params = ast.Node.BuiltinCall.ParamList.init(arena),
2453 .rparen_token = undefined,2489 .rparen_token = undefined,
2454 }2490 }
...@@ -2467,7 +2503,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2467,7 +2503,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2467 const node = try createToCtxNode(arena, opt_ctx, ast.Node.PrefixOp,2503 const node = try createToCtxNode(arena, opt_ctx, ast.Node.PrefixOp,
2468 ast.Node.PrefixOp {2504 ast.Node.PrefixOp {
2469 .base = undefined,2505 .base = undefined,
2470 .op_token = token_index,2506 .op_token = token.index,
2471 .op = undefined,2507 .op = undefined,
2472 .rhs = undefined,2508 .rhs = undefined,
2473 }2509 }
...@@ -2478,7 +2514,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2478,7 +2514,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2478 Token.Id.Keyword_error => {2514 Token.Id.Keyword_error => {
2479 stack.push(State {2515 stack.push(State {
2480 .ErrorTypeOrSetDecl = ErrorTypeOrSetDeclCtx {2516 .ErrorTypeOrSetDecl = ErrorTypeOrSetDeclCtx {
2481 .error_token = token_index,2517 .error_token = token.index,
2482 .opt_ctx = opt_ctx2518 .opt_ctx = opt_ctx
2483 }2519 }
2484 }) catch unreachable;2520 }) catch unreachable;
...@@ -2488,7 +2524,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2488,7 +2524,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2488 stack.push(State {2524 stack.push(State {
2489 .ContainerKind = ContainerKindCtx {2525 .ContainerKind = ContainerKindCtx {
2490 .opt_ctx = opt_ctx,2526 .opt_ctx = opt_ctx,
2491 .ltoken = token_index,2527 .ltoken = token.index,
2492 .layout = ast.Node.ContainerDecl.Layout.Packed,2528 .layout = ast.Node.ContainerDecl.Layout.Packed,
2493 },2529 },
2494 }) catch unreachable;2530 }) catch unreachable;
...@@ -2498,18 +2534,18 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2498,18 +2534,18 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2498 stack.push(State {2534 stack.push(State {
2499 .ExternType = ExternTypeCtx {2535 .ExternType = ExternTypeCtx {
2500 .opt_ctx = opt_ctx,2536 .opt_ctx = opt_ctx,
2501 .extern_token = token_index,2537 .extern_token = token.index,
2502 .comments = null,2538 .comments = null,
2503 },2539 },
2504 }) catch unreachable;2540 }) catch unreachable;
2505 continue;2541 continue;
2506 },2542 },
2507 Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => {2543 Token.Id.Keyword_struct, Token.Id.Keyword_union, Token.Id.Keyword_enum => {
2508 _ = tok_it.prev();2544 putBackToken(&tok_it, &tree);
2509 stack.push(State {2545 stack.push(State {
2510 .ContainerKind = ContainerKindCtx {2546 .ContainerKind = ContainerKindCtx {
2511 .opt_ctx = opt_ctx,2547 .opt_ctx = opt_ctx,
2512 .ltoken = token_index,2548 .ltoken = token.index,
2513 .layout = ast.Node.ContainerDecl.Layout.Auto,2549 .layout = ast.Node.ContainerDecl.Layout.Auto,
2514 },2550 },
2515 }) catch unreachable;2551 }) catch unreachable;
...@@ -2518,7 +2554,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2518,7 +2554,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2518 Token.Id.Identifier => {2554 Token.Id.Identifier => {
2519 stack.push(State {2555 stack.push(State {
2520 .MaybeLabeledExpression = MaybeLabeledExpressionCtx {2556 .MaybeLabeledExpression = MaybeLabeledExpressionCtx {
2521 .label = token_index,2557 .label = token.index,
2522 .opt_ctx = opt_ctx2558 .opt_ctx = opt_ctx
2523 }2559 }
2524 }) catch unreachable;2560 }) catch unreachable;
...@@ -2532,7 +2568,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2532,7 +2568,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2532 .doc_comments = null,2568 .doc_comments = null,
2533 .visib_token = null,2569 .visib_token = null,
2534 .name_token = null,2570 .name_token = null,
2535 .fn_token = token_index,2571 .fn_token = token.index,
2536 .params = ast.Node.FnProto.ParamList.init(arena),2572 .params = ast.Node.FnProto.ParamList.init(arena),
2537 .return_type = undefined,2573 .return_type = undefined,
2538 .var_args_token = null,2574 .var_args_token = null,
...@@ -2560,7 +2596,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2560,7 +2596,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2560 .return_type = undefined,2596 .return_type = undefined,
2561 .var_args_token = null,2597 .var_args_token = null,
2562 .extern_export_inline_token = null,2598 .extern_export_inline_token = null,
2563 .cc_token = token_index,2599 .cc_token = token.index,
2564 .async_attr = null,2600 .async_attr = null,
2565 .body_node = null,2601 .body_node = null,
2566 .lib_name = null,2602 .lib_name = null,
...@@ -2580,7 +2616,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2580,7 +2616,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2580 const node = try createToCtxNode(arena, opt_ctx, ast.Node.Asm,2616 const node = try createToCtxNode(arena, opt_ctx, ast.Node.Asm,
2581 ast.Node.Asm {2617 ast.Node.Asm {
2582 .base = undefined,2618 .base = undefined,
2583 .asm_token = token_index,2619 .asm_token = token.index,
2584 .volatile_token = null,2620 .volatile_token = null,
2585 .template = undefined,2621 .template = undefined,
2586 .outputs = ast.Node.Asm.OutputList.init(arena),2622 .outputs = ast.Node.Asm.OutputList.init(arena),
...@@ -2614,18 +2650,18 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2614,18 +2650,18 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2614 stack.push(State {2650 stack.push(State {
2615 .Inline = InlineCtx {2651 .Inline = InlineCtx {
2616 .label = null,2652 .label = null,
2617 .inline_token = token_index,2653 .inline_token = token.index,
2618 .opt_ctx = opt_ctx,2654 .opt_ctx = opt_ctx,
2619 }2655 }
2620 }) catch unreachable;2656 }) catch unreachable;
2621 continue;2657 continue;
2622 },2658 },
2623 else => {2659 else => {
2624 if (!try parseBlockExpr(&stack, arena, opt_ctx, token_ptr, token_index)) {2660 if (!try parseBlockExpr(&stack, arena, opt_ctx, token.ptr, token.index)) {
2625 _ = tok_it.prev();2661 putBackToken(&tok_it, &tree);
2626 if (opt_ctx != OptionalCtx.Optional) {2662 if (opt_ctx != OptionalCtx.Optional) {
2627 *(try tree.errors.addOne()) = Error {2663 *(try tree.errors.addOne()) = Error {
2628 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token_index },2664 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token.index },
2629 };2665 };
2630 return tree;2666 return tree;
2631 }2667 }
...@@ -2637,7 +2673,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2637,7 +2673,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
26372673
26382674
2639 State.ErrorTypeOrSetDecl => |ctx| {2675 State.ErrorTypeOrSetDecl => |ctx| {
2640 if (eatToken(&tok_it, Token.Id.LBrace) == null) {2676 if (eatToken(&tok_it, &tree, Token.Id.LBrace) == null) {
2641 _ = try createToCtxLiteral(arena, ctx.opt_ctx, ast.Node.ErrorType, ctx.error_token);2677 _ = try createToCtxLiteral(arena, ctx.opt_ctx, ast.Node.ErrorType, ctx.error_token);
2642 continue;2678 continue;
2643 }2679 }
...@@ -2661,11 +2697,12 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2661,11 +2697,12 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2661 continue;2697 continue;
2662 },2698 },
2663 State.StringLiteral => |opt_ctx| {2699 State.StringLiteral => |opt_ctx| {
2664 const token_index = tok_it.index;2700 const token = nextToken(&tok_it, &tree);
2665 const token_ptr = ??tok_it.next();2701 const token_index = token.index;
2702 const token_ptr = token.ptr;
2666 opt_ctx.store(2703 opt_ctx.store(
2667 (try parseStringLiteral(arena, &tok_it, token_ptr, token_index)) ?? {2704 (try parseStringLiteral(arena, &tok_it, token_ptr, token_index, &tree)) ?? {
2668 _ = tok_it.prev();2705 putBackToken(&tok_it, &tree);
2669 if (opt_ctx != OptionalCtx.Optional) {2706 if (opt_ctx != OptionalCtx.Optional) {
2670 *(try tree.errors.addOne()) = Error {2707 *(try tree.errors.addOne()) = Error {
2671 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token_index },2708 .ExpectedPrimaryExpr = Error.ExpectedPrimaryExpr { .token = token_index },
...@@ -2679,14 +2716,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2679,14 +2716,15 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2679 },2716 },
26802717
2681 State.Identifier => |opt_ctx| {2718 State.Identifier => |opt_ctx| {
2682 if (eatToken(&tok_it, Token.Id.Identifier)) |ident_token| {2719 if (eatToken(&tok_it, &tree, Token.Id.Identifier)) |ident_token| {
2683 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Identifier, ident_token);2720 _ = try createToCtxLiteral(arena, opt_ctx, ast.Node.Identifier, ident_token);
2684 continue;2721 continue;
2685 }2722 }
26862723
2687 if (opt_ctx != OptionalCtx.Optional) {2724 if (opt_ctx != OptionalCtx.Optional) {
2688 const token_index = tok_it.index;2725 const token = nextToken(&tok_it, &tree);
2689 const token_ptr = ??tok_it.next();2726 const token_index = token.index;
2727 const token_ptr = token.ptr;
2690 *(try tree.errors.addOne()) = Error {2728 *(try tree.errors.addOne()) = Error {
2691 .ExpectedToken = Error.ExpectedToken {2729 .ExpectedToken = Error.ExpectedToken {
2692 .token = token_index,2730 .token = token_index,
...@@ -2698,9 +2736,10 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2698,9 +2736,10 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2698 },2736 },
26992737
2700 State.ErrorTag => |node_ptr| {2738 State.ErrorTag => |node_ptr| {
2701 const comments = try eatDocComments(arena, &tok_it);2739 const comments = try eatDocComments(arena, &tok_it, &tree);
2702 const ident_token_index = tok_it.index;2740 const ident_token = nextToken(&tok_it, &tree);
2703 const ident_token_ptr = ??tok_it.next();2741 const ident_token_index = ident_token.index;
2742 const ident_token_ptr = ident_token.ptr;
2704 if (ident_token_ptr.id != Token.Id.Identifier) {2743 if (ident_token_ptr.id != Token.Id.Identifier) {
2705 *(try tree.errors.addOne()) = Error {2744 *(try tree.errors.addOne()) = Error {
2706 .ExpectedToken = Error.ExpectedToken {2745 .ExpectedToken = Error.ExpectedToken {
...@@ -2723,8 +2762,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2723,8 +2762,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2723 },2762 },
27242763
2725 State.ExpectToken => |token_id| {2764 State.ExpectToken => |token_id| {
2726 const token_index = tok_it.index;2765 const token = nextToken(&tok_it, &tree);
2727 const token_ptr = ??tok_it.next();2766 const token_index = token.index;
2767 const token_ptr = token.ptr;
2728 if (token_ptr.id != token_id) {2768 if (token_ptr.id != token_id) {
2729 *(try tree.errors.addOne()) = Error {2769 *(try tree.errors.addOne()) = Error {
2730 .ExpectedToken = Error.ExpectedToken {2770 .ExpectedToken = Error.ExpectedToken {
...@@ -2737,8 +2777,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2737,8 +2777,9 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2737 continue;2777 continue;
2738 },2778 },
2739 State.ExpectTokenSave => |expect_token_save| {2779 State.ExpectTokenSave => |expect_token_save| {
2740 const token_index = tok_it.index;2780 const token = nextToken(&tok_it, &tree);
2741 const token_ptr = ??tok_it.next();2781 const token_index = token.index;
2782 const token_ptr = token.ptr;
2742 if (token_ptr.id != expect_token_save.id) {2783 if (token_ptr.id != expect_token_save.id) {
2743 *(try tree.errors.addOne()) = Error {2784 *(try tree.errors.addOne()) = Error {
2744 .ExpectedToken = Error.ExpectedToken {2785 .ExpectedToken = Error.ExpectedToken {
...@@ -2752,7 +2793,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2752,7 +2793,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2752 continue;2793 continue;
2753 },2794 },
2754 State.IfToken => |token_id| {2795 State.IfToken => |token_id| {
2755 if (eatToken(&tok_it, token_id)) |_| {2796 if (eatToken(&tok_it, &tree, token_id)) |_| {
2756 continue;2797 continue;
2757 }2798 }
27582799
...@@ -2760,7 +2801,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2760,7 +2801,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2760 continue;2801 continue;
2761 },2802 },
2762 State.IfTokenSave => |if_token_save| {2803 State.IfTokenSave => |if_token_save| {
2763 if (eatToken(&tok_it, if_token_save.id)) |token_index| {2804 if (eatToken(&tok_it, &tree, if_token_save.id)) |token_index| {
2764 *if_token_save.ptr = token_index;2805 *if_token_save.ptr = token_index;
2765 continue;2806 continue;
2766 }2807 }
...@@ -2769,7 +2810,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {...@@ -2769,7 +2810,7 @@ pub fn parse(allocator: &mem.Allocator, source: []const u8) !ast.Tree {
2769 continue;2810 continue;
2770 },2811 },
2771 State.OptionalTokenSave => |optional_token_save| {2812 State.OptionalTokenSave => |optional_token_save| {
2772 if (eatToken(&tok_it, optional_token_save.id)) |token_index| {2813 if (eatToken(&tok_it, &tree, optional_token_save.id)) |token_index| {
2773 *optional_token_save.ptr = token_index;2814 *optional_token_save.ptr = token_index;
2774 continue;2815 continue;
2775 }2816 }
...@@ -3043,10 +3084,10 @@ const State = union(enum) {...@@ -3043,10 +3084,10 @@ const State = union(enum) {
3043 OptionalTokenSave: OptionalTokenSave,3084 OptionalTokenSave: OptionalTokenSave,
3044};3085};
30453086
3046fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !?&ast.Node.DocComment {3087fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) !?&ast.Node.DocComment {
3047 var result: ?&ast.Node.DocComment = null;3088 var result: ?&ast.Node.DocComment = null;
3048 while (true) {3089 while (true) {
3049 if (eatToken(tok_it, Token.Id.DocComment)) |line_comment| {3090 if (eatToken(tok_it, tree, Token.Id.DocComment)) |line_comment| {
3050 const node = blk: {3091 const node = blk: {
3051 if (result) |comment_node| {3092 if (result) |comment_node| {
3052 break :blk comment_node;3093 break :blk comment_node;
...@@ -3069,8 +3110,8 @@ fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !...@@ -3069,8 +3110,8 @@ fn eatDocComments(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !
3069 return result;3110 return result;
3070}3111}
30713112
3072fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !?&ast.Node.LineComment {3113fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) !?&ast.Node.LineComment {
3073 const token = eatToken(tok_it, Token.Id.LineComment) ?? return null;3114 const token = eatToken(tok_it, tree, Token.Id.LineComment) ?? return null;
3074 return try arena.construct(ast.Node.LineComment {3115 return try arena.construct(ast.Node.LineComment {
3075 .base = ast.Node {3116 .base = ast.Node {
3076 .id = ast.Node.Id.LineComment,3117 .id = ast.Node.Id.LineComment,
...@@ -3080,7 +3121,7 @@ fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !...@@ -3080,7 +3121,7 @@ fn eatLineComment(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator) !
3080}3121}
30813122
3082fn parseStringLiteral(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator,3123fn parseStringLiteral(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterator,
3083 token_ptr: &const Token, token_index: TokenIndex) !?&ast.Node3124 token_ptr: &const Token, token_index: TokenIndex, tree: &ast.Tree) !?&ast.Node
3084{3125{
3085 switch (token_ptr.id) {3126 switch (token_ptr.id) {
3086 Token.Id.StringLiteral => {3127 Token.Id.StringLiteral => {
...@@ -3093,10 +3134,11 @@ fn parseStringLiteral(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterato...@@ -3093,10 +3134,11 @@ fn parseStringLiteral(arena: &mem.Allocator, tok_it: &ast.Tree.TokenList.Iterato
3093 });3134 });
3094 try node.lines.push(token_index);3135 try node.lines.push(token_index);
3095 while (true) {3136 while (true) {
3096 const multiline_str_index = tok_it.index;3137 const multiline_str = nextToken(tok_it, tree);
3097 const multiline_str_ptr = ??tok_it.next();3138 const multiline_str_index = multiline_str.index;
3139 const multiline_str_ptr = multiline_str.ptr;
3098 if (multiline_str_ptr.id != Token.Id.MultilineStringLiteralLine) {3140 if (multiline_str_ptr.id != Token.Id.MultilineStringLiteralLine) {
3099 _ = tok_it.prev();3141 putBackToken(tok_it, tree);
3100 break;3142 break;
3101 }3143 }
31023144
...@@ -3230,9 +3272,10 @@ const ExpectCommaOrEndResult = union(enum) {...@@ -3230,9 +3272,10 @@ const ExpectCommaOrEndResult = union(enum) {
3230 parse_error: Error,3272 parse_error: Error,
3231};3273};
32323274
3233fn expectCommaOrEnd(tok_it: &ast.Tree.TokenList.Iterator, end: @TagType(Token.Id)) ExpectCommaOrEndResult {3275fn expectCommaOrEnd(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree, end: @TagType(Token.Id)) ExpectCommaOrEndResult {
3234 const token_index = tok_it.index;3276 const token = nextToken(tok_it, tree);
3235 const token_ptr = ??tok_it.next();3277 const token_index = token.index;
3278 const token_ptr = token.ptr;
3236 switch (token_ptr.id) {3279 switch (token_ptr.id) {
3237 Token.Id.Comma => return ExpectCommaOrEndResult { .end_token = null},3280 Token.Id.Comma => return ExpectCommaOrEndResult { .end_token = null},
3238 else => {3281 else => {
...@@ -3385,16 +3428,45 @@ fn createToCtxLiteral(arena: &mem.Allocator, opt_ctx: &const OptionalCtx, compti...@@ -3385,16 +3428,45 @@ fn createToCtxLiteral(arena: &mem.Allocator, opt_ctx: &const OptionalCtx, compti
3385 return node;3428 return node;
3386}3429}
33873430
3388fn eatToken(tok_it: &ast.Tree.TokenList.Iterator, id: @TagType(Token.Id)) ?TokenIndex {3431fn eatToken(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree, id: @TagType(Token.Id)) ?TokenIndex {
3389 const token_index = tok_it.index;3432 const token = nextToken(tok_it, tree);
3390 const token_ptr = ??tok_it.next();3433
3391 if (token_ptr.id == id)3434 if (token.ptr.id == id)
3392 return token_index;3435 return token.index;
33933436
3394 _ = tok_it.prev();3437 putBackToken(tok_it, tree);
3395 return null;3438 return null;
3396}3439}
33973440
3441fn nextToken(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) AnnotatedToken {
3442 const result = AnnotatedToken {
3443 .index = tok_it.index,
3444 .ptr = ??tok_it.next(),
3445 };
3446 // possibly skip a following same line token
3447 const token = tok_it.next() ?? return result;
3448 if (token.id != Token.Id.LineComment) {
3449 putBackToken(tok_it, tree);
3450 return result;
3451 }
3452 const loc = tree.tokenLocationPtr(result.ptr.end, token);
3453 if (loc.line != 0) {
3454 putBackToken(tok_it, tree);
3455 }
3456 return result;
3457}
3458
3459fn putBackToken(tok_it: &ast.Tree.TokenList.Iterator, tree: &ast.Tree) void {
3460 const prev_tok = ??tok_it.prev();
3461 if (prev_tok.id == Token.Id.LineComment) {
3462 const minus2_tok = tok_it.prev() ?? return;
3463 const loc = tree.tokenLocationPtr(minus2_tok.end, prev_tok);
3464 if (loc.line != 0) {
3465 _ = tok_it.next();
3466 }
3467 }
3468}
3469
3398const RenderAstFrame = struct {3470const RenderAstFrame = struct {
3399 node: &ast.Node,3471 node: &ast.Node,
3400 indent: usize,3472 indent: usize,
std/zig/parser_test.zig+54-54
...@@ -1,3 +1,48 @@...@@ -1,3 +1,48 @@
1//test "zig fmt: same-line comment after a statement" {
2// try testCanonical(
3// \\test "" {
4// \\ a = b;
5// \\ debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption
6// \\ a = b;
7// \\}
8// \\
9// );
10//}
11//
12//test "zig fmt: same-line comment after var decl in struct" {
13// try testCanonical(
14// \\pub const vfs_cap_data = extern struct {
15// \\ const Data = struct {}; // when on disk.
16// \\};
17// \\
18// );
19//}
20//
21//test "zig fmt: same-line comment after field decl" {
22// try testCanonical(
23// \\pub const dirent = extern struct {
24// \\ d_name: u8,
25// \\ d_name: u8, // comment 1
26// \\ d_name: u8,
27// \\ d_name: u8, // comment 2
28// \\ d_name: u8,
29// \\};
30// \\
31// );
32//}
33//
34//test "zig fmt: same-line comment after switch prong" {
35// try testCanonical(
36// \\test "" {
37// \\ switch (err) {
38// \\ error.PathAlreadyExists => {}, // comment 2
39// \\ else => return err, // comment 1
40// \\ }
41// \\}
42// \\
43// );
44//}
45//
1//test "zig fmt: same-line comment after non-block if expression" {46//test "zig fmt: same-line comment after non-block if expression" {
2// try testCanonical(47// try testCanonical(
3// \\comptime {48// \\comptime {
...@@ -7,6 +52,15 @@...@@ -7,6 +52,15 @@
7// \\52// \\
8// );53// );
9//}54//}
55//
56//test "zig fmt: same-line comment on comptime expression" {
57// try testCanonical(
58// \\test "" {
59// \\ comptime assert(@typeId(T) == builtin.TypeId.Int); // must pass an integer to absInt
60// \\}
61// \\
62// );
63//}
1064
11test "zig fmt: switch with empty body" {65test "zig fmt: switch with empty body" {
12 try testCanonical(66 try testCanonical(
...@@ -17,15 +71,6 @@ test "zig fmt: switch with empty body" {...@@ -17,15 +71,6 @@ test "zig fmt: switch with empty body" {
17 );71 );
18}72}
1973
20//test "zig fmt: same-line comment on comptime expression" {
21// try testCanonical(
22// \\test "" {
23// \\ comptime assert(@typeId(T) == builtin.TypeId.Int); // must pass an integer to absInt
24// \\}
25// \\
26// );
27//}
28
29test "zig fmt: float literal with exponent" {74test "zig fmt: float literal with exponent" {
30 try testCanonical(75 try testCanonical(
31 \\pub const f64_true_min = 4.94065645841246544177e-324;76 \\pub const f64_true_min = 4.94065645841246544177e-324;
...@@ -152,18 +197,6 @@ test "zig fmt: comments before switch prong" {...@@ -152,18 +197,6 @@ test "zig fmt: comments before switch prong" {
152 );197 );
153}198}
154199
155//test "zig fmt: same-line comment after switch prong" {
156// try testCanonical(
157// \\test "" {
158// \\ switch (err) {
159// \\ error.PathAlreadyExists => {}, // comment 2
160// \\ else => return err, // comment 1
161// \\ }
162// \\}
163// \\
164// );
165//}
166
167test "zig fmt: comments before var decl in struct" {200test "zig fmt: comments before var decl in struct" {
168 try testCanonical(201 try testCanonical(
169 \\pub const vfs_cap_data = extern struct {202 \\pub const vfs_cap_data = extern struct {
...@@ -189,28 +222,6 @@ test "zig fmt: comments before var decl in struct" {...@@ -189,28 +222,6 @@ test "zig fmt: comments before var decl in struct" {
189 );222 );
190}223}
191224
192//test "zig fmt: same-line comment after var decl in struct" {
193// try testCanonical(
194// \\pub const vfs_cap_data = extern struct {
195// \\ const Data = struct {}; // when on disk.
196// \\};
197// \\
198// );
199//}
200//
201//test "zig fmt: same-line comment after field decl" {
202// try testCanonical(
203// \\pub const dirent = extern struct {
204// \\ d_name: u8,
205// \\ d_name: u8, // comment 1
206// \\ d_name: u8,
207// \\ d_name: u8, // comment 2
208// \\ d_name: u8,
209// \\};
210// \\
211// );
212//}
213
214test "zig fmt: array literal with 1 item on 1 line" {225test "zig fmt: array literal with 1 item on 1 line" {
215 try testCanonical(226 try testCanonical(
216 \\var s = []const u64{0} ** 25;227 \\var s = []const u64{0} ** 25;
...@@ -218,17 +229,6 @@ test "zig fmt: array literal with 1 item on 1 line" {...@@ -218,17 +229,6 @@ test "zig fmt: array literal with 1 item on 1 line" {
218 );229 );
219}230}
220231
221//test "zig fmt: same-line comment after a statement" {
222// try testCanonical(
223// \\test "" {
224// \\ a = b;
225// \\ debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption
226// \\ a = b;
227// \\}
228// \\
229// );
230//}
231
232test "zig fmt: comments before global variables" {232test "zig fmt: comments before global variables" {
233 try testCanonical(233 try testCanonical(
234 \\/// Foo copies keys and values before they go into the map, and234 \\/// Foo copies keys and values before they go into the map, and