authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 14:20:34+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 14:20:34+02:00
log0b9247fb636a47add80f7c4ec194df436629df91
treea59e3916a05af37d8aacbb1f096250fb92db7eb3
parentb424cd75ab9df69f44fe47c10ded3fd25c2d27bb

std.zig.parser Refactor:

* Slice/Array access is now not parsed in the expr contruction loop * State.ExprListItemOrEnd now takes a token id for the end token

2 files changed, 118 insertions(+), 61 deletions(-)

std/zig/ast.zig+29-1
......@@ -18,6 +18,7 @@ pub const Node = struct {
1818 InfixOp,
1919 PrefixOp,
2020 SuffixOp,
21 FieldInitializer,
2122 IntegerLiteral,
2223 FloatLiteral,
2324 StringLiteral,
......@@ -45,6 +46,7 @@ pub const Node = struct {
4546 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
4647 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
4748 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index),
49 Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index),
4850 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
4951 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
5052 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
......@@ -73,6 +75,7 @@ pub const Node = struct {
7375 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),
7476 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
7577 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(),
78 Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(),
7679 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
7780 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
7881 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
......@@ -101,6 +104,7 @@ pub const Node = struct {
101104 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),
102105 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
103106 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(),
107 Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(),
104108 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
105109 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
106110 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
......@@ -526,6 +530,30 @@ pub const NodePrefixOp = struct {
526530 }
527531};
528532
533pub const NodeFieldInitializer = struct {
534 base: Node,
535 dot_token: Token,
536 name_token: Token,
537 expr: &Node,
538
539 pub fn iterate(self: &NodeFieldInitializer, index: usize) ?&Node {
540 var i = index;
541
542 if (i < 1) return self.expr;
543 i -= 1;
544
545 return null;
546 }
547
548 pub fn firstToken(self: &NodeFieldInitializer) Token {
549 return self.dot_token;
550 }
551
552 pub fn lastToken(self: &NodeFieldInitializer) Token {
553 return self.expr.lastToken();
554 }
555};
556
529557pub const NodeSuffixOp = struct {
530558 base: Node,
531559 lhs: &Node,
......@@ -537,7 +565,7 @@ pub const NodeSuffixOp = struct {
537565 ArrayAccess: &Node,
538566 Slice: SliceRange,
539567 ArrayInitializer: ArrayList(&Node),
540 StructInitializer: ArrayList(&Node),
568 StructInitializer: ArrayList(&NodeFieldInitializer),
541569 };
542570
543571 const CallInfo = struct {
std/zig/parser.zig+89-60
......@@ -77,6 +77,12 @@ pub const Parser = struct {
7777 ptr: &Token,
7878 };
7979
80 const ExprListState = struct {
81 list: &ArrayList(&ast.Node),
82 end: Token.Id,
83 ptr: &Token,
84 };
85
8086 const State = union(enum) {
8187 TopLevel,
8288 TopLevelExtern: ?Token,
......@@ -88,7 +94,7 @@ pub const Parser = struct {
8894 InfixOp: &ast.NodeInfixOp,
8995 PrefixOp: &ast.NodePrefixOp,
9096 SuffixOp: &ast.NodeSuffixOp,
91 SliceOrArrayAccess,
97 SliceOrArrayAccess: &ast.NodeSuffixOp,
9298 AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
9399 TypeExpr: DestPtr,
94100 VarDecl: &ast.NodeVarDecl,
......@@ -104,8 +110,8 @@ pub const Parser = struct {
104110 FnDef: &ast.NodeFnProto,
105111 Block: &ast.NodeBlock,
106112 Statement: &ast.NodeBlock,
107 ExprListItemOrEnd: &ArrayList(&ast.Node),
108 ExprListCommaOrEnd: &ArrayList(&ast.Node),
113 ExprListItemOrEnd: ExprListState,
114 ExprListCommaOrEnd: ExprListState,
109115 };
110116
111117 /// Returns an AST tree, allocated with the parser's allocator.
......@@ -529,13 +535,14 @@ pub const Parser = struct {
529535 .Operand = &node.base
530536 });
531537 try stack.append(State.AfterOperand);
532 try stack.append(State {.ExprListItemOrEnd = &node.params });
533538 try stack.append(State {
534 .ExpectTokenSave = ExpectTokenSave {
535 .id = Token.Id.LParen,
539 .ExprListItemOrEnd = ExprListState {
540 .list = &node.params,
541 .end = Token.Id.RParen,
536542 .ptr = &node.rparen_token,
537 },
543 }
538544 });
545 try stack.append(State { .ExpectToken = Token.Id.LParen, });
539546 continue;
540547 },
541548 Token.Id.StringLiteral => {
......@@ -587,6 +594,46 @@ pub const Parser = struct {
587594 }
588595 },
589596
597 State.SliceOrArrayAccess => |node| {
598 var token = self.getNextToken();
599
600 switch (token.id) {
601 Token.Id.Ellipsis2 => {
602 const start = node.op.ArrayAccess;
603 node.op = ast.NodeSuffixOp.SuffixOp {
604 .Slice = ast.NodeSuffixOp.SliceRange {
605 .start = start,
606 .end = undefined,
607 }
608 };
609 try stack.append(State { .SuffixOp = node });
610 try stack.append(State.AfterOperand);
611
612 const rbracket_token = self.getNextToken();
613 if (rbracket_token.id != Token.Id.RBracket) {
614 self.putBackToken(rbracket_token);
615 try stack.append(State {
616 .ExpectTokenSave = ExpectTokenSave {
617 .id = Token.Id.RBracket,
618 .ptr = &node.rtoken,
619 }
620 });
621 try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } });
622 } else {
623 node.rtoken = rbracket_token;
624 }
625 continue;
626 },
627 Token.Id.RBracket => {
628 node.rtoken = token;
629 try stack.append(State { .SuffixOp = node });
630 try stack.append(State.AfterOperand);
631 continue;
632 },
633 else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id))
634 }
635 },
636
590637 State.AfterOperand => {
591638 // we'll either get an infix operator (like != or ^),
592639 // or a postfix operator (like () or {}),
......@@ -610,7 +657,13 @@ pub const Parser = struct {
610657 });
611658 try stack.append(State { .SuffixOp = node });
612659 try stack.append(State.AfterOperand);
613 try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params });
660 try stack.append(State {
661 .ExprListItemOrEnd = ExprListState {
662 .list = &node.op.Call.params,
663 .end = Token.Id.RParen,
664 .ptr = &node.rtoken,
665 }
666 });
614667 try stack.append(State {
615668 .ExpectTokenSave = ExpectTokenSave {
616669 .id = Token.Id.LParen,
......@@ -620,8 +673,17 @@ pub const Parser = struct {
620673 continue;
621674
622675 } else if (token.id == Token.Id.LBracket) {
623 try stack.append(State.SliceOrArrayAccess);
624 try stack.append(State.ExpectOperand);
676 const node = try arena.create(ast.NodeSuffixOp);
677 *node = ast.NodeSuffixOp {
678 .base = self.initNode(ast.Node.Id.SuffixOp),
679 .lhs = undefined,
680 .op = ast.NodeSuffixOp.SuffixOp {
681 .ArrayAccess = undefined,
682 },
683 .rtoken = undefined,
684 };
685 try stack.append(State { .SliceOrArrayAccess = node });
686 try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }});
625687 continue;
626688
627689 // TODO: Parse postfix operator
......@@ -637,48 +699,6 @@ pub const Parser = struct {
637699 try dest_ptr.store(expression);
638700 break;
639701 },
640 State.SliceOrArrayAccess => {
641 var rbracket_or_ellipsis2_token = self.getNextToken();
642
643 switch (rbracket_or_ellipsis2_token.id) {
644 Token.Id.Ellipsis2 => {
645 const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp {
646 .Slice = ast.NodeSuffixOp.SliceRange {
647 .start = expression,
648 .end = null,
649 }
650 });
651
652 try stack.append(State { .SuffixOp = node });
653 try stack.append(State.AfterOperand);
654
655 const rbracket_token = self.getNextToken();
656 if (rbracket_token.id != Token.Id.RBracket) {
657 self.putBackToken(rbracket_token);
658 try stack.append(State {
659 .ExpectTokenSave = ExpectTokenSave {
660 .id = Token.Id.RBracket,
661 .ptr = &node.rtoken,
662 }
663 });
664 try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } });
665 } else {
666 node.rtoken = rbracket_token;
667 }
668 break;
669 },
670 Token.Id.RBracket => {
671 const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp {
672 .ArrayAccess = expression
673 });
674 node.rtoken = token;
675 try stack.append(State { .SuffixOp = node });
676 try stack.append(State.AfterOperand);
677 break;
678 },
679 else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id))
680 }
681 },
682702 State.InfixOp => |infix_op| {
683703 infix_op.rhs = expression;
684704 infix_op.lhs = popSuffixOp(&stack);
......@@ -697,26 +717,31 @@ pub const Parser = struct {
697717 }
698718 },
699719
700 State.ExprListItemOrEnd => |params| {
720 State.ExprListItemOrEnd => |expr_list_state| {
701721 var token = self.getNextToken();
702722 switch (token.id) {
703723 Token.Id.RParen => continue,
704724 else => {
705725 self.putBackToken(token);
706 stack.append(State { .ExprListCommaOrEnd = params }) catch unreachable;
707 try stack.append(State { .Expression = DestPtr{.List = params} });
726 stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable;
727 try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} });
708728 },
709729 }
710730 },
711731
712 State.ExprListCommaOrEnd => |params| {
732 State.ExprListCommaOrEnd => |expr_list_state| {
713733 var token = self.getNextToken();
714734 switch (token.id) {
715735 Token.Id.Comma => {
716 stack.append(State { .ExprListItemOrEnd = params }) catch unreachable;
736 stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable;
737 },
738 else => {
739 const IdTag = @TagType(Token.Id);
740 if (IdTag(expr_list_state.end) == token.id)
741 continue;
742
743 return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id));
717744 },
718 Token.Id.RParen => continue,
719 else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
720745 }
721746 },
722747
......@@ -933,7 +958,6 @@ pub const Parser = struct {
933958 State.PrefixOp => unreachable,
934959 State.SuffixOp => unreachable,
935960 State.Operand => unreachable,
936 State.SliceOrArrayAccess => unreachable,
937961 }
938962 }
939963 }
......@@ -1649,6 +1673,11 @@ pub const Parser = struct {
16491673
16501674 try stack.append(RenderState { .Expression = suffix_op.lhs });
16511675 },
1676 ast.Node.Id.FieldInitializer => {
1677 const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base);
1678 try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token));
1679 try stack.append(RenderState { .Expression = field_init.expr });
1680 },
16521681 ast.Node.Id.IntegerLiteral => {
16531682 const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
16541683 try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));