authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-08 17:05:08+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-08 17:05:08+02:00
loge4d0b46c0c07f3151b4959c5e2e0d4c304981ead
tree881f8a0f7da63c5486a8851c9029e210fc4ea2e2
parentbdff5bfa3e9f6ab490771b54109cb200b180b4da

std.zig.parser WIP generalizing parsing of payloads

* Note, it doesn't work :)

2 files changed, 161 insertions(+), 51 deletions(-)

std/zig/ast.zig+98-10
......@@ -20,9 +20,11 @@ pub const Node = struct {
2020 FnProto,
2121 ParamDecl,
2222 Block,
23 Payload,
2324 Switch,
2425 SwitchCase,
2526 SwitchElse,
27 While,
2628 InfixOp,
2729 PrefixOp,
2830 SuffixOp,
......@@ -58,9 +60,11 @@ pub const Node = struct {
5860 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
5961 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
6062 Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
63 Id.Payload => @fieldParentPtr(NodePayload, "base", base).iterate(index),
6164 Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).iterate(index),
6265 Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).iterate(index),
6366 Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).iterate(index),
67 Id.While => @fieldParentPtr(NodeWhile, "base", base).iterate(index),
6468 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
6569 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
6670 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).iterate(index),
......@@ -97,9 +101,11 @@ pub const Node = struct {
97101 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(),
98102 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(),
99103 Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(),
104 Id.Payload => @fieldParentPtr(NodePayload, "base", base).firstToken(),
100105 Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).firstToken(),
101106 Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).firstToken(),
102107 Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(),
108 Id.While => @fieldParentPtr(NodeWhile, "base", base).firstToken(),
103109 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(),
104110 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(),
105111 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).firstToken(),
......@@ -136,9 +142,11 @@ pub const Node = struct {
136142 Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(),
137143 Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(),
138144 Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(),
145 Id.Payload => @fieldParentPtr(NodePayload, "base", base).lastToken(),
139146 Id.Switch => @fieldParentPtr(NodeSwitch, "base", base).lastToken(),
140147 Id.SwitchCase => @fieldParentPtr(NodeSwitchCase, "base", base).lastToken(),
141 Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).firstToken(),
148 Id.SwitchElse => @fieldParentPtr(NodeSwitchElse, "base", base).lastToken(),
149 Id.While => @fieldParentPtr(NodeWhile, "base", base).lastToken(),
142150 Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(),
143151 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(),
144152 Id.SuffixOp => @fieldParentPtr(NodeSuffixOp, "base", base).lastToken(),
......@@ -545,6 +553,31 @@ pub const NodeBlock = struct {
545553 }
546554};
547555
556pub const NodePayload = struct {
557 base: Node,
558 lpipe: Token,
559 is_ptr: bool,
560 symbol: &NodeIdentifier,
561 rpipe: Token,
562
563 pub fn iterate(self: &NodePayload, index: usize) ?&Node {
564 var i = index;
565
566 if (i < 1) return &self.symbol.base;
567 i -= 1;
568
569 return null;
570 }
571
572 pub fn firstToken(self: &NodePayload) Token {
573 return self.lpipe;
574 }
575
576 pub fn lastToken(self: &NodePayload) Token {
577 return self.rpipe;
578 }
579};
580
548581pub const NodeSwitch = struct {
549582 base: Node,
550583 switch_token: Token,
......@@ -576,22 +609,17 @@ pub const NodeSwitch = struct {
576609pub const NodeSwitchCase = struct {
577610 base: Node,
578611 items: ArrayList(&Node),
579 capture: ?Capture,
612 payload: ?&Node,
580613 expr: &Node,
581614
582 const Capture = struct {
583 symbol: &NodeIdentifier,
584 is_ptr: bool,
585 };
586
587615 pub fn iterate(self: &NodeSwitchCase, index: usize) ?&Node {
588616 var i = index;
589617
590618 if (i < self.items.len) return self.items.at(i);
591619 i -= self.items.len;
592620
593 if (self.capture) |capture| {
594 if (i < 1) return &capture.base;
621 if (self.payload) |payload| {
622 if (i < 1) return payload;
595623 i -= 1;
596624 }
597625
......@@ -627,6 +655,66 @@ pub const NodeSwitchElse = struct {
627655 }
628656};
629657
658pub const NodeWhile = struct {
659 base: Node,
660 while_token: Token,
661 condition: &Node,
662 payload: ?&NodePayload,
663 continue_expr: ?&Node,
664 body: &Node,
665 @"else": ?Else,
666
667 const Else = struct {
668 capture: ?&NodeIdentifier,
669 body: &Node,
670 };
671
672
673 pub fn iterate(self: &NodeWhile, index: usize) ?&Node {
674 var i = index;
675
676 if (i < 1) return self.condition;
677 i -= 1;
678
679 if (self.payload) |payload| {
680 if (i < 1) return &payload.base;
681 i -= 1;
682 }
683
684 if (self.continue_expr) |continue_expr| {
685 if (i < 1) return continue_expr;
686 i -= 1;
687 }
688
689 if (i < 1) return self.body;
690 i -= 1;
691
692 if (self.@"else") |@"else"| {
693 if (@"else".capture) |capture| {
694 if (i < 1) return &capture.base;
695 i -= 1;
696 }
697
698 if (i < 1) return @"else".body;
699 i -= 1;
700 }
701
702 return null;
703 }
704
705 pub fn firstToken(self: &NodeWhile) Token {
706 return self.while_token;
707 }
708
709 pub fn lastToken(self: &NodeWhile) Token {
710 if (self.@"else") |@"else"| {
711 return @"else".body.lastToken();
712 }
713
714 return self.body.lastToken();
715 }
716};
717
630718pub const NodeInfixOp = struct {
631719 base: Node,
632720 op_token: Token,
......@@ -661,7 +749,7 @@ pub const NodeInfixOp = struct {
661749 BitXor,
662750 BoolAnd,
663751 BoolOr,
664 Catch: ?&NodeIdentifier,
752 Catch: ?&Node,
665753 Div,
666754 EqualEqual,
667755 ErrorUnion,
std/zig/parser.zig+63-41
......@@ -104,6 +104,11 @@ pub const Parser = struct {
104104 ptr: &Token,
105105 };
106106
107 const ElseCtx = struct {
108 payload: ?DestPtr,
109 body: DestPtr,
110 };
111
107112 fn ListSave(comptime T: type) type {
108113 return struct {
109114 list: &ArrayList(T),
......@@ -140,7 +145,7 @@ pub const Parser = struct {
140145 FieldInitListCommaOrEnd: ListSave(&ast.NodeFieldInitializer),
141146 FieldListCommaOrEnd: &ast.NodeContainerDecl,
142147 SwitchCaseOrEnd: ListSave(&ast.NodeSwitchCase),
143 SwitchCaseCapture: &?ast.NodeSwitchCase.Capture,
148 Payload: DestPtr,
144149 SwitchCaseCommaOrEnd: ListSave(&ast.NodeSwitchCase),
145150 SwitchCaseItem: &ArrayList(&ast.Node),
146151 SwitchCaseItemCommaOrEnd: &ArrayList(&ast.Node),
......@@ -635,9 +640,6 @@ pub const Parser = struct {
635640 Token.Id.Keyword_await => {
636641 @panic("TODO: await");
637642 },
638 Token.Id.Keyword_suspend => {
639 @panic("TODO: suspend");
640 },
641643 else => {
642644 self.putBackToken(token);
643645 stack.append(State { .AssignmentExpressionBegin = dest_ptr }) catch unreachable;
......@@ -705,21 +707,14 @@ pub const Parser = struct {
705707
706708 stack.append(State { .UnwrapExpressionEnd = dest_ptr }) catch unreachable;
707709 try stack.append(State { .Expression = DestPtr { .Field = &node.rhs } });
708
709 const next = self.getNextToken();
710 if (next.id != Token.Id.Pipe) {
711 self.putBackToken(next);
712 continue;
713 }
714
715 node.op.Catch = try self.createIdentifier(arena, Token(undefined));
716 try stack.append(State { .ExpectToken = Token.Id.Pipe });
717710 try stack.append(State {
718 .ExpectTokenSave = ExpectTokenSave {
719 .id = Token.Id.Identifier,
720 .ptr = &(??node.op.Catch).name_token
711 .Optional = RevertState {
712 .tokenizer = *self.tokenizer,
713 .parser = *self,
714 .ptr = &node.op.Catch,
721715 }
722716 });
717 try stack.append(State { .Payload = DestPtr { .NullableField = &node.op.Catch } });
723718 continue;
724719 },
725720 Token.Id.QuestionMarkQuestionMark => {
......@@ -1404,12 +1399,25 @@ pub const Parser = struct {
14041399 @panic("TODO: inline if");
14051400 },
14061401 Token.Id.Keyword_while => {
1402 const node = try arena.create(ast.NodeWhile);
1403 *node = ast.NodeWhile {
1404 .base = self.initNode(ast.Node.Id.While),
1405 .while_token = token,
1406 .condition = undefined,
1407 .payload = null,
1408 .continue_expr = null,
1409 .body = undefined,
1410 .@"else" = null,
1411 };
1412 dest_ptr.store(&node.base);
1413
14071414 @panic("TODO: inline while");
14081415 },
14091416 Token.Id.Keyword_for => {
14101417 @panic("TODO: inline for");
14111418 },
14121419 Token.Id.Keyword_switch => {
1420 @breakpoint();
14131421 const node = try arena.create(ast.NodeSwitch);
14141422 *node = ast.NodeSwitch {
14151423 .base = self.initNode(ast.Node.Id.Switch),
......@@ -1434,9 +1442,6 @@ pub const Parser = struct {
14341442 Token.Id.Keyword_comptime => {
14351443 @panic("TODO: inline comptime");
14361444 },
1437 Token.Id.Keyword_suspend => {
1438 @panic("TODO: inline suspend");
1439 },
14401445 else => {
14411446 try self.parseError(&stack, token, "expected primary expression, found {}", @tagName(token.id));
14421447 continue;
......@@ -1547,13 +1552,21 @@ pub const Parser = struct {
15471552 *node = ast.NodeSwitchCase {
15481553 .base = self.initNode(ast.Node.Id.SwitchCase),
15491554 .items = ArrayList(&ast.Node).init(arena),
1550 .capture = null,
1555 .payload = null,
15511556 .expr = undefined,
15521557 };
15531558 try list_state.list.append(node);
15541559 stack.append(State { .SwitchCaseCommaOrEnd = list_state }) catch unreachable;
15551560 try stack.append(State { .Expression = DestPtr{ .Field = &node.expr } });
1556 try stack.append(State { .SwitchCaseCapture = &node.capture });
1561 try stack.append(State {
1562 .Optional = RevertState {
1563 .tokenizer = *self.tokenizer,
1564 .parser = *self,
1565 .ptr = &node.payload,
1566 }
1567 });
1568 try stack.append(State { .Payload = DestPtr { .NullableField = &node.payload } });
1569 try stack.append(State.Required);
15571570
15581571 const maybe_else = self.getNextToken();
15591572 if (maybe_else.id == Token.Id.Keyword_else) {
......@@ -1572,12 +1585,8 @@ pub const Parser = struct {
15721585 }
15731586 },
15741587
1575 State.SwitchCaseCapture => |capture| {
1576 const token = self.getNextToken();
1577 if (token.id != Token.Id.Pipe) {
1578 self.putBackToken(token);
1579 continue;
1580 }
1588 State.Payload => |dest_ptr| {
1589 const lpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue;
15811590
15821591 const is_ptr = blk: {
15831592 const asterik = self.getNextToken();
......@@ -1590,11 +1599,16 @@ pub const Parser = struct {
15901599 };
15911600
15921601 const ident = (try self.eatToken(&stack, Token.Id.Identifier)) ?? continue;
1593 _ = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue;
1594 *capture = ast.NodeSwitchCase.Capture {
1602 const rpipe = (try self.eatToken(&stack, Token.Id.Pipe)) ?? continue;
1603 const node = try arena.create(ast.NodePayload);
1604 *node = ast.NodePayload {
1605 .base = self.initNode(ast.Node.Id.Payload),
1606 .lpipe = lpipe,
1607 .is_ptr = is_ptr,
15951608 .symbol = try self.createIdentifier(arena, ident),
1596 .is_ptr = is_ptr
1609 .rpipe = rpipe
15971610 };
1611 dest_ptr.store(&node.base);
15981612 },
15991613
16001614 State.SwitchCaseItem => |case_items| {
......@@ -1856,6 +1870,8 @@ pub const Parser = struct {
18561870 stack.append(State { .Block = inner_block }) catch unreachable;
18571871 continue;
18581872 },
1873 Token.Id.Keyword_suspend, Token.Id.Keyword_if,
1874 Token.Id.Keyword_while, Token.Id.Keyword_for,
18591875 Token.Id.Keyword_switch => {
18601876 self.putBackToken(next);
18611877 stack.append(State { .Expression = DestPtr{.Field = try block.statements.addOne() } }) catch unreachable;
......@@ -2572,9 +2588,8 @@ pub const Parser = struct {
25722588
25732589 if (prefix_op_node.op == ast.NodeInfixOp.InfixOp.Catch) {
25742590 if (prefix_op_node.op.Catch) |payload| {
2575 try stack.append(RenderState { .Text = "| " });
2576 try stack.append(RenderState { .Expression = &payload.base });
2577 try stack.append(RenderState { .Text = "|" });
2591 try stack.append(RenderState { .Text = " " });
2592 try stack.append(RenderState { .Expression = payload });
25782593 }
25792594 try stack.append(RenderState { .Text = " catch " });
25802595 } else {
......@@ -2764,6 +2779,17 @@ pub const Parser = struct {
27642779 try stack.append(RenderState { .Expression = rhs });
27652780 }
27662781 },
2782 ast.Node.Id.Payload => {
2783 const payload = @fieldParentPtr(ast.NodePayload, "base", base);
2784 try stack.append(RenderState { .Text = "|"});
2785 try stack.append(RenderState { .Expression = &payload.symbol.base });
2786
2787 if (payload.is_ptr) {
2788 try stack.append(RenderState { .Text = "*"});
2789 }
2790
2791 try stack.append(RenderState { .Text = "|"});
2792 },
27672793 ast.Node.Id.GroupedExpression => {
27682794 const grouped_expr = @fieldParentPtr(ast.NodeGroupedExpression, "base", base);
27692795 try stack.append(RenderState { .Text = ")"});
......@@ -2985,14 +3011,9 @@ pub const Parser = struct {
29853011 const switch_case = @fieldParentPtr(ast.NodeSwitchCase, "base", base);
29863012
29873013 try stack.append(RenderState { .Expression = switch_case.expr });
2988 if (switch_case.capture) |capture| {
2989 try stack.append(RenderState { .Text = "| "});
2990 try stack.append(RenderState { .Expression = &capture.symbol.base });
2991
2992 if (capture.is_ptr) {
2993 try stack.append(RenderState { .Text = "*"});
2994 }
2995 try stack.append(RenderState { .Text = "|"});
3014 if (switch_case.payload) |payload| {
3015 try stack.append(RenderState { .Text = " " });
3016 try stack.append(RenderState { .Expression = payload });
29963017 }
29973018 try stack.append(RenderState { .Text = " => "});
29983019
......@@ -3011,6 +3032,7 @@ pub const Parser = struct {
30113032 const switch_else = @fieldParentPtr(ast.NodeSwitchElse, "base", base);
30123033 try stream.print("{}", self.tokenizer.getTokenSlice(switch_else.token));
30133034 },
3035 ast.Node.Id.While => @panic("TODO: Render while"),
30143036
30153037 ast.Node.Id.StructField,
30163038 ast.Node.Id.UnionTag,