authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 14:20:34+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-03 14:20:34+02:00
log0b9247fb636a47add80f7c4ec194df436629df91
treea59e3916a05af37d8aacbb1f096250fb92db7eb3
parentb424cd75ab9df69f44fe47c10ded3fd25c2d27bb

std.zig.parser Refactor:

* Slice/Array access is now not parsed in the expr contruction loop * State.ExprListItemOrEnd now takes a token id for the end token

2 files changed, 118 insertions(+), 61 deletions(-)

std/zig/ast.zig+29-1
...@@ -18,6 +18,7 @@ pub const Node = struct {...@@ -18,6 +18,7 @@ pub const Node = struct {
18 InfixOp,18 InfixOp,
19 PrefixOp,19 PrefixOp,
20 SuffixOp,20 SuffixOp,
21 FieldInitializer,
21 IntegerLiteral,22 IntegerLiteral,
22 FloatLiteral,23 FloatLiteral,
23 StringLiteral,24 StringLiteral,
...@@ -45,6 +46,7 @@ pub const Node = struct {...@@ -45,6 +46,7 @@ pub const Node = struct {
45 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),46 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
46 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),47 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
47 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index),48 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index),
49 Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).iterate(index),
48 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),50 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
49 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),51 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
50 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),52 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
...@@ -73,6 +75,7 @@ pub const Node = struct {...@@ -73,6 +75,7 @@ pub const Node = struct {
73 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),75 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),
74 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),76 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
75 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(),77 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(),
78 Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).firstToken(),
76 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),79 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
77 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),80 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
78 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),81 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
...@@ -101,6 +104,7 @@ pub const Node = struct {...@@ -101,6 +104,7 @@ pub const Node = struct {
101 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),104 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),
102 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),105 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
103 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(),106 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(),
107 Id.FieldInitializer => @fieldParentPtr(NodeFieldInitializer, "base", base).lastToken(),
104 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),108 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
105 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),109 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
106 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),110 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
...@@ -526,6 +530,30 @@ pub const NodePrefixOp = struct {...@@ -526,6 +530,30 @@ pub const NodePrefixOp = struct {
526 }530 }
527};531};
528532
533pub const NodeFieldInitializer = struct {
534 base: Node,
535 dot_token: Token,
536 name_token: Token,
537 expr: &Node,
538
539 pub fn iterate(self: &NodeFieldInitializer, index: usize) ?&Node {
540 var i = index;
541
542 if (i < 1) return self.expr;
543 i -= 1;
544
545 return null;
546 }
547
548 pub fn firstToken(self: &NodeFieldInitializer) Token {
549 return self.dot_token;
550 }
551
552 pub fn lastToken(self: &NodeFieldInitializer) Token {
553 return self.expr.lastToken();
554 }
555};
556
529pub const NodeSuffixOp = struct {557pub const NodeSuffixOp = struct {
530 base: Node,558 base: Node,
531 lhs: &Node,559 lhs: &Node,
...@@ -537,7 +565,7 @@ pub const NodeSuffixOp = struct {...@@ -537,7 +565,7 @@ pub const NodeSuffixOp = struct {
537 ArrayAccess: &Node,565 ArrayAccess: &Node,
538 Slice: SliceRange,566 Slice: SliceRange,
539 ArrayInitializer: ArrayList(&Node),567 ArrayInitializer: ArrayList(&Node),
540 StructInitializer: ArrayList(&Node),568 StructInitializer: ArrayList(&NodeFieldInitializer),
541 };569 };
542570
543 const CallInfo = struct {571 const CallInfo = struct {
std/zig/parser.zig+89-60
...@@ -77,6 +77,12 @@ pub const Parser = struct {...@@ -77,6 +77,12 @@ pub const Parser = struct {
77 ptr: &Token,77 ptr: &Token,
78 };78 };
7979
80 const ExprListState = struct {
81 list: &ArrayList(&ast.Node),
82 end: Token.Id,
83 ptr: &Token,
84 };
85
80 const State = union(enum) {86 const State = union(enum) {
81 TopLevel,87 TopLevel,
82 TopLevelExtern: ?Token,88 TopLevelExtern: ?Token,
...@@ -88,7 +94,7 @@ pub const Parser = struct {...@@ -88,7 +94,7 @@ pub const Parser = struct {
88 InfixOp: &ast.NodeInfixOp,94 InfixOp: &ast.NodeInfixOp,
89 PrefixOp: &ast.NodePrefixOp,95 PrefixOp: &ast.NodePrefixOp,
90 SuffixOp: &ast.NodeSuffixOp,96 SuffixOp: &ast.NodeSuffixOp,
91 SliceOrArrayAccess,97 SliceOrArrayAccess: &ast.NodeSuffixOp,
92 AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,98 AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
93 TypeExpr: DestPtr,99 TypeExpr: DestPtr,
94 VarDecl: &ast.NodeVarDecl,100 VarDecl: &ast.NodeVarDecl,
...@@ -104,8 +110,8 @@ pub const Parser = struct {...@@ -104,8 +110,8 @@ pub const Parser = struct {
104 FnDef: &ast.NodeFnProto,110 FnDef: &ast.NodeFnProto,
105 Block: &ast.NodeBlock,111 Block: &ast.NodeBlock,
106 Statement: &ast.NodeBlock,112 Statement: &ast.NodeBlock,
107 ExprListItemOrEnd: &ArrayList(&ast.Node),113 ExprListItemOrEnd: ExprListState,
108 ExprListCommaOrEnd: &ArrayList(&ast.Node),114 ExprListCommaOrEnd: ExprListState,
109 };115 };
110116
111 /// Returns an AST tree, allocated with the parser's allocator.117 /// Returns an AST tree, allocated with the parser's allocator.
...@@ -529,13 +535,14 @@ pub const Parser = struct {...@@ -529,13 +535,14 @@ pub const Parser = struct {
529 .Operand = &node.base535 .Operand = &node.base
530 });536 });
531 try stack.append(State.AfterOperand);537 try stack.append(State.AfterOperand);
532 try stack.append(State {.ExprListItemOrEnd = &node.params });
533 try stack.append(State {538 try stack.append(State {
534 .ExpectTokenSave = ExpectTokenSave {539 .ExprListItemOrEnd = ExprListState {
535 .id = Token.Id.LParen,540 .list = &node.params,
541 .end = Token.Id.RParen,
536 .ptr = &node.rparen_token,542 .ptr = &node.rparen_token,
537 },543 }
538 });544 });
545 try stack.append(State { .ExpectToken = Token.Id.LParen, });
539 continue;546 continue;
540 },547 },
541 Token.Id.StringLiteral => {548 Token.Id.StringLiteral => {
...@@ -587,6 +594,46 @@ pub const Parser = struct {...@@ -587,6 +594,46 @@ pub const Parser = struct {
587 }594 }
588 },595 },
589596
597 State.SliceOrArrayAccess => |node| {
598 var token = self.getNextToken();
599
600 switch (token.id) {
601 Token.Id.Ellipsis2 => {
602 const start = node.op.ArrayAccess;
603 node.op = ast.NodeSuffixOp.SuffixOp {
604 .Slice = ast.NodeSuffixOp.SliceRange {
605 .start = start,
606 .end = undefined,
607 }
608 };
609 try stack.append(State { .SuffixOp = node });
610 try stack.append(State.AfterOperand);
611
612 const rbracket_token = self.getNextToken();
613 if (rbracket_token.id != Token.Id.RBracket) {
614 self.putBackToken(rbracket_token);
615 try stack.append(State {
616 .ExpectTokenSave = ExpectTokenSave {
617 .id = Token.Id.RBracket,
618 .ptr = &node.rtoken,
619 }
620 });
621 try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } });
622 } else {
623 node.rtoken = rbracket_token;
624 }
625 continue;
626 },
627 Token.Id.RBracket => {
628 node.rtoken = token;
629 try stack.append(State { .SuffixOp = node });
630 try stack.append(State.AfterOperand);
631 continue;
632 },
633 else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id))
634 }
635 },
636
590 State.AfterOperand => {637 State.AfterOperand => {
591 // we'll either get an infix operator (like != or ^),638 // we'll either get an infix operator (like != or ^),
592 // or a postfix operator (like () or {}),639 // or a postfix operator (like () or {}),
...@@ -610,7 +657,13 @@ pub const Parser = struct {...@@ -610,7 +657,13 @@ pub const Parser = struct {
610 });657 });
611 try stack.append(State { .SuffixOp = node });658 try stack.append(State { .SuffixOp = node });
612 try stack.append(State.AfterOperand);659 try stack.append(State.AfterOperand);
613 try stack.append(State {.ExprListItemOrEnd = &node.op.Call.params });660 try stack.append(State {
661 .ExprListItemOrEnd = ExprListState {
662 .list = &node.op.Call.params,
663 .end = Token.Id.RParen,
664 .ptr = &node.rtoken,
665 }
666 });
614 try stack.append(State {667 try stack.append(State {
615 .ExpectTokenSave = ExpectTokenSave {668 .ExpectTokenSave = ExpectTokenSave {
616 .id = Token.Id.LParen,669 .id = Token.Id.LParen,
...@@ -620,8 +673,17 @@ pub const Parser = struct {...@@ -620,8 +673,17 @@ pub const Parser = struct {
620 continue;673 continue;
621674
622 } else if (token.id == Token.Id.LBracket) {675 } else if (token.id == Token.Id.LBracket) {
623 try stack.append(State.SliceOrArrayAccess);676 const node = try arena.create(ast.NodeSuffixOp);
624 try stack.append(State.ExpectOperand);677 *node = ast.NodeSuffixOp {
678 .base = self.initNode(ast.Node.Id.SuffixOp),
679 .lhs = undefined,
680 .op = ast.NodeSuffixOp.SuffixOp {
681 .ArrayAccess = undefined,
682 },
683 .rtoken = undefined,
684 };
685 try stack.append(State { .SliceOrArrayAccess = node });
686 try stack.append(State { .Expression = DestPtr { .Field = &node.op.ArrayAccess }});
625 continue;687 continue;
626688
627 // TODO: Parse postfix operator689 // TODO: Parse postfix operator
...@@ -637,48 +699,6 @@ pub const Parser = struct {...@@ -637,48 +699,6 @@ pub const Parser = struct {
637 try dest_ptr.store(expression);699 try dest_ptr.store(expression);
638 break;700 break;
639 },701 },
640 State.SliceOrArrayAccess => {
641 var rbracket_or_ellipsis2_token = self.getNextToken();
642
643 switch (rbracket_or_ellipsis2_token.id) {
644 Token.Id.Ellipsis2 => {
645 const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp {
646 .Slice = ast.NodeSuffixOp.SliceRange {
647 .start = expression,
648 .end = null,
649 }
650 });
651
652 try stack.append(State { .SuffixOp = node });
653 try stack.append(State.AfterOperand);
654
655 const rbracket_token = self.getNextToken();
656 if (rbracket_token.id != Token.Id.RBracket) {
657 self.putBackToken(rbracket_token);
658 try stack.append(State {
659 .ExpectTokenSave = ExpectTokenSave {
660 .id = Token.Id.RBracket,
661 .ptr = &node.rtoken,
662 }
663 });
664 try stack.append(State { .Expression = DestPtr { .NullableField = &node.op.Slice.end } });
665 } else {
666 node.rtoken = rbracket_token;
667 }
668 break;
669 },
670 Token.Id.RBracket => {
671 const node = try self.createSuffixOp(arena, ast.NodeSuffixOp.SuffixOp {
672 .ArrayAccess = expression
673 });
674 node.rtoken = token;
675 try stack.append(State { .SuffixOp = node });
676 try stack.append(State.AfterOperand);
677 break;
678 },
679 else => return self.parseError(token, "expected ']' or '..', found {}", @tagName(token.id))
680 }
681 },
682 State.InfixOp => |infix_op| {702 State.InfixOp => |infix_op| {
683 infix_op.rhs = expression;703 infix_op.rhs = expression;
684 infix_op.lhs = popSuffixOp(&stack);704 infix_op.lhs = popSuffixOp(&stack);
...@@ -697,26 +717,31 @@ pub const Parser = struct {...@@ -697,26 +717,31 @@ pub const Parser = struct {
697 }717 }
698 },718 },
699719
700 State.ExprListItemOrEnd => |params| {720 State.ExprListItemOrEnd => |expr_list_state| {
701 var token = self.getNextToken();721 var token = self.getNextToken();
702 switch (token.id) {722 switch (token.id) {
703 Token.Id.RParen => continue,723 Token.Id.RParen => continue,
704 else => {724 else => {
705 self.putBackToken(token);725 self.putBackToken(token);
706 stack.append(State { .ExprListCommaOrEnd = params }) catch unreachable;726 stack.append(State { .ExprListCommaOrEnd = expr_list_state }) catch unreachable;
707 try stack.append(State { .Expression = DestPtr{.List = params} });727 try stack.append(State { .Expression = DestPtr{.List = expr_list_state.list} });
708 },728 },
709 }729 }
710 },730 },
711731
712 State.ExprListCommaOrEnd => |params| {732 State.ExprListCommaOrEnd => |expr_list_state| {
713 var token = self.getNextToken();733 var token = self.getNextToken();
714 switch (token.id) {734 switch (token.id) {
715 Token.Id.Comma => {735 Token.Id.Comma => {
716 stack.append(State { .ExprListItemOrEnd = params }) catch unreachable;736 stack.append(State { .ExprListItemOrEnd = expr_list_state }) catch unreachable;
737 },
738 else => {
739 const IdTag = @TagType(Token.Id);
740 if (IdTag(expr_list_state.end) == token.id)
741 continue;
742
743 return self.parseError(token, "expected ',' or {}, found {}", @tagName(expr_list_state.end), @tagName(token.id));
717 },744 },
718 Token.Id.RParen => continue,
719 else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
720 }745 }
721 },746 },
722747
...@@ -933,7 +958,6 @@ pub const Parser = struct {...@@ -933,7 +958,6 @@ pub const Parser = struct {
933 State.PrefixOp => unreachable,958 State.PrefixOp => unreachable,
934 State.SuffixOp => unreachable,959 State.SuffixOp => unreachable,
935 State.Operand => unreachable,960 State.Operand => unreachable,
936 State.SliceOrArrayAccess => unreachable,
937 }961 }
938 }962 }
939 }963 }
...@@ -1649,6 +1673,11 @@ pub const Parser = struct {...@@ -1649,6 +1673,11 @@ pub const Parser = struct {
16491673
1650 try stack.append(RenderState { .Expression = suffix_op.lhs });1674 try stack.append(RenderState { .Expression = suffix_op.lhs });
1651 },1675 },
1676 ast.Node.Id.FieldInitializer => {
1677 const field_init = @fieldParentPtr(ast.NodeFieldInitializer, "base", base);
1678 try stream.print(".{} = ", self.tokenizer.getTokenSlice(field_init.name_token));
1679 try stack.append(RenderState { .Expression = field_init.expr });
1680 },
1652 ast.Node.Id.IntegerLiteral => {1681 ast.Node.Id.IntegerLiteral => {
1653 const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);1682 const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
1654 try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));1683 try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));