authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 14:43:53+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-04-11 14:43:53+02:00
logfe7146277d5a7f7c74a8c2f36719c946e6061c50
treeaf2fcea408dccc792180726923b6a7301949afd0
parent28ea364e5e60464f68501fdfa69ba41b9cf9c47e

std.zig.parser now accept both string and multiline string for strings

Related #909 Allows it to parse `std/special/compiler_rt/aullrem.zig`, `std/special/compiler_rt/aulldiv.zig` and `std/math/x86_64/sqrt.zig`

2 files changed, 81 insertions(+), 54 deletions(-)

std/zig/ast.zig+7-7
...@@ -1608,7 +1608,7 @@ pub const NodeThisLiteral = struct {...@@ -1608,7 +1608,7 @@ pub const NodeThisLiteral = struct {
1608pub const NodeAsmOutput = struct {1608pub const NodeAsmOutput = struct {
1609 base: Node,1609 base: Node,
1610 symbolic_name: &NodeIdentifier,1610 symbolic_name: &NodeIdentifier,
1611 constraint: &NodeStringLiteral,1611 constraint: &Node,
1612 kind: Kind,1612 kind: Kind,
16131613
1614 const Kind = union(enum) {1614 const Kind = union(enum) {
...@@ -1622,7 +1622,7 @@ pub const NodeAsmOutput = struct {...@@ -1622,7 +1622,7 @@ pub const NodeAsmOutput = struct {
1622 if (i < 1) return &self.symbolic_name.base;1622 if (i < 1) return &self.symbolic_name.base;
1623 i -= 1;1623 i -= 1;
16241624
1625 if (i < 1) return &self.constraint.base;1625 if (i < 1) return self.constraint;
1626 i -= 1;1626 i -= 1;
16271627
1628 switch (self.kind) {1628 switch (self.kind) {
...@@ -1654,7 +1654,7 @@ pub const NodeAsmOutput = struct {...@@ -1654,7 +1654,7 @@ pub const NodeAsmOutput = struct {
1654pub const NodeAsmInput = struct {1654pub const NodeAsmInput = struct {
1655 base: Node,1655 base: Node,
1656 symbolic_name: &NodeIdentifier,1656 symbolic_name: &NodeIdentifier,
1657 constraint: &NodeStringLiteral,1657 constraint: &Node,
1658 expr: &Node,1658 expr: &Node,
16591659
1660 pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node {1660 pub fn iterate(self: &NodeAsmInput, index: usize) ?&Node {
...@@ -1663,7 +1663,7 @@ pub const NodeAsmInput = struct {...@@ -1663,7 +1663,7 @@ pub const NodeAsmInput = struct {
1663 if (i < 1) return &self.symbolic_name.base;1663 if (i < 1) return &self.symbolic_name.base;
1664 i -= 1;1664 i -= 1;
16651665
1666 if (i < 1) return &self.constraint.base;1666 if (i < 1) return self.constraint;
1667 i -= 1;1667 i -= 1;
16681668
1669 if (i < 1) return self.expr;1669 if (i < 1) return self.expr;
...@@ -1685,11 +1685,11 @@ pub const NodeAsm = struct {...@@ -1685,11 +1685,11 @@ pub const NodeAsm = struct {
1685 base: Node,1685 base: Node,
1686 asm_token: Token,1686 asm_token: Token,
1687 is_volatile: bool,1687 is_volatile: bool,
1688 template: Token,1688 template: &Node,
1689 //tokens: ArrayList(AsmToken),1689 //tokens: ArrayList(AsmToken),
1690 outputs: ArrayList(&NodeAsmOutput),1690 outputs: ArrayList(&NodeAsmOutput),
1691 inputs: ArrayList(&NodeAsmInput),1691 inputs: ArrayList(&NodeAsmInput),
1692 cloppers: ArrayList(&NodeStringLiteral),1692 cloppers: ArrayList(&Node),
1693 rparen: Token,1693 rparen: Token,
16941694
1695 pub fn iterate(self: &NodeAsm, index: usize) ?&Node {1695 pub fn iterate(self: &NodeAsm, index: usize) ?&Node {
...@@ -1701,7 +1701,7 @@ pub const NodeAsm = struct {...@@ -1701,7 +1701,7 @@ pub const NodeAsm = struct {
1701 if (i < self.inputs.len) return &self.inputs.at(index).base;1701 if (i < self.inputs.len) return &self.inputs.at(index).base;
1702 i -= self.inputs.len;1702 i -= self.inputs.len;
17031703
1704 if (i < self.cloppers.len) return &self.cloppers.at(index).base;1704 if (i < self.cloppers.len) return self.cloppers.at(index);
1705 i -= self.cloppers.len;1705 i -= self.cloppers.len;
17061706
1707 return null;1707 return null;
std/zig/parser.zig+74-47
...@@ -171,7 +171,7 @@ pub const Parser = struct {...@@ -171,7 +171,7 @@ pub const Parser = struct {
171 Semicolon: &const &const ast.Node,171 Semicolon: &const &const ast.Node,
172 AsmOutputItems: &ArrayList(&ast.NodeAsmOutput),172 AsmOutputItems: &ArrayList(&ast.NodeAsmOutput),
173 AsmInputItems: &ArrayList(&ast.NodeAsmInput),173 AsmInputItems: &ArrayList(&ast.NodeAsmInput),
174 AsmClopperItems: &ArrayList(&ast.NodeStringLiteral),174 AsmClopperItems: &ArrayList(&ast.Node),
175 ExprListItemOrEnd: ExprListCtx,175 ExprListItemOrEnd: ExprListCtx,
176 ExprListCommaOrEnd: ExprListCtx,176 ExprListCommaOrEnd: ExprListCtx,
177 FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer),177 FieldInitListItemOrEnd: ListSave(&ast.NodeFieldInitializer),
...@@ -301,7 +301,11 @@ pub const Parser = struct {...@@ -301,7 +301,11 @@ pub const Parser = struct {
301 Token.Id.Keyword_test => {301 Token.Id.Keyword_test => {
302 stack.append(State.TopLevel) catch unreachable;302 stack.append(State.TopLevel) catch unreachable;
303303
304 const name_token = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;304 const name_token = self.getNextToken();
305 const name = (try self.parseStringLiteral(arena, name_token)) ?? {
306 try self.parseError(&stack, name_token, "expected string literal, found {}", @tagName(name_token.id));
307 continue;
308 };
305 const lbrace = (try self.expectToken(&stack, Token.Id.LBrace)) ?? continue;309 const lbrace = (try self.expectToken(&stack, Token.Id.LBrace)) ?? continue;
306310
307 const block = try self.createNode(arena, ast.NodeBlock,311 const block = try self.createNode(arena, ast.NodeBlock,
...@@ -317,7 +321,7 @@ pub const Parser = struct {...@@ -317,7 +321,7 @@ pub const Parser = struct {
317 ast.NodeTestDecl {321 ast.NodeTestDecl {
318 .base = undefined,322 .base = undefined,
319 .test_token = token,323 .test_token = token,
320 .name = &(try self.createLiteral(arena, ast.NodeStringLiteral, name_token)).base,324 .name = name,
321 .body_node = &block.base,325 .body_node = &block.base,
322 }326 }
323 );327 );
...@@ -389,15 +393,12 @@ pub const Parser = struct {...@@ -389,15 +393,12 @@ pub const Parser = struct {
389 continue;393 continue;
390 },394 },
391 Token.Id.Keyword_extern => {395 Token.Id.Keyword_extern => {
392 const lib_name_token = self.getNextToken();
393 const lib_name = blk: {396 const lib_name = blk: {
394 if (lib_name_token.id == Token.Id.StringLiteral) {397 const lib_name_token = self.getNextToken();
395 const res = try self.createLiteral(arena, ast.NodeStringLiteral, lib_name_token);398 break :blk (try self.parseStringLiteral(arena, lib_name_token)) ?? {
396 break :blk &res.base;
397 } else {
398 self.putBackToken(lib_name_token);399 self.putBackToken(lib_name_token);
399 break :blk null;400 break :blk null;
400 }401 };
401 };402 };
402403
403 stack.append(State {404 stack.append(State {
...@@ -1504,10 +1505,6 @@ pub const Parser = struct {...@@ -1504,10 +1505,6 @@ pub const Parser = struct {
1504 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeFloatLiteral, token)).base);1505 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeFloatLiteral, token)).base);
1505 continue;1506 continue;
1506 },1507 },
1507 Token.Id.StringLiteral => {
1508 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base);
1509 continue;
1510 },
1511 Token.Id.CharLiteral => {1508 Token.Id.CharLiteral => {
1512 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeCharLiteral, token)).base);1509 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeCharLiteral, token)).base);
1513 continue;1510 continue;
...@@ -1536,24 +1533,8 @@ pub const Parser = struct {...@@ -1536,24 +1533,8 @@ pub const Parser = struct {
1536 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUnreachable, token)).base);1533 dest_ptr.store(&(try self.createLiteral(arena, ast.NodeUnreachable, token)).base);
1537 continue;1534 continue;
1538 },1535 },
1539 Token.Id.MultilineStringLiteralLine => {1536 Token.Id.StringLiteral, Token.Id.MultilineStringLiteralLine => {
1540 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeMultilineStringLiteral,1537 dest_ptr.store((try self.parseStringLiteral(arena, token)) ?? unreachable);
1541 ast.NodeMultilineStringLiteral {
1542 .base = undefined,
1543 .tokens = ArrayList(Token).init(arena),
1544 }
1545 );
1546 try node.tokens.append(token);
1547 while (true) {
1548 const multiline_str = self.getNextToken();
1549 if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
1550 self.putBackToken(multiline_str);
1551 break;
1552 }
1553
1554 try node.tokens.append(multiline_str);
1555 }
1556 continue;
1557 },1538 },
1558 Token.Id.LParen => {1539 Token.Id.LParen => {
1559 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeGroupedExpression,1540 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeGroupedExpression,
...@@ -1781,7 +1762,12 @@ pub const Parser = struct {...@@ -1781,7 +1762,12 @@ pub const Parser = struct {
1781 break :blk true;1762 break :blk true;
1782 };1763 };
1783 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;1764 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
1784 const template = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;1765
1766 const template_token = self.getNextToken();
1767 const template = (try self.parseStringLiteral(arena, template_token)) ?? {
1768 try self.parseError(&stack, template_token, "expected string literal, found {}", @tagName(template_token.id));
1769 continue;
1770 };
1785 // TODO parse template1771 // TODO parse template
17861772
1787 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm,1773 const node = try self.createToDestNode(arena, dest_ptr, ast.NodeAsm,
...@@ -1793,7 +1779,7 @@ pub const Parser = struct {...@@ -1793,7 +1779,7 @@ pub const Parser = struct {
1793 //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena),1779 //.tokens = ArrayList(ast.NodeAsm.AsmToken).init(arena),
1794 .outputs = ArrayList(&ast.NodeAsmOutput).init(arena),1780 .outputs = ArrayList(&ast.NodeAsmOutput).init(arena),
1795 .inputs = ArrayList(&ast.NodeAsmInput).init(arena),1781 .inputs = ArrayList(&ast.NodeAsmInput).init(arena),
1796 .cloppers = ArrayList(&ast.NodeStringLiteral).init(arena),1782 .cloppers = ArrayList(&ast.Node).init(arena),
1797 .rparen = undefined,1783 .rparen = undefined,
1798 }1784 }
1799 );1785 );
...@@ -1881,7 +1867,12 @@ pub const Parser = struct {...@@ -1881,7 +1867,12 @@ pub const Parser = struct {
18811867
1882 const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;1868 const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;
1883 _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;1869 _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;
1884 const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;1870
1871 const constraint_token = self.getNextToken();
1872 const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? {
1873 try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id));
1874 continue;
1875 };
18851876
1886 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;1877 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
1887 try stack.append(State { .ExpectToken = Token.Id.RParen });1878 try stack.append(State { .ExpectToken = Token.Id.RParen });
...@@ -1890,7 +1881,7 @@ pub const Parser = struct {...@@ -1890,7 +1881,7 @@ pub const Parser = struct {
1890 ast.NodeAsmOutput {1881 ast.NodeAsmOutput {
1891 .base = undefined,1882 .base = undefined,
1892 .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),1883 .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),
1893 .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint),1884 .constraint = constraint,
1894 .kind = undefined,1885 .kind = undefined,
1895 }1886 }
1896 );1887 );
...@@ -1926,7 +1917,12 @@ pub const Parser = struct {...@@ -1926,7 +1917,12 @@ pub const Parser = struct {
19261917
1927 const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;1918 const symbolic_name = (try self.expectToken(&stack, Token.Id.Identifier)) ?? continue;
1928 _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;1919 _ = (try self.expectToken(&stack, Token.Id.RBracket)) ?? continue;
1929 const constraint = (try self.expectToken(&stack, Token.Id.StringLiteral)) ?? continue;1920
1921 const constraint_token = self.getNextToken();
1922 const constraint = (try self.parseStringLiteral(arena, constraint_token)) ?? {
1923 try self.parseError(&stack, constraint_token, "expected string literal, found {}", @tagName(constraint_token.id));
1924 continue;
1925 };
19301926
1931 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;1927 _ = (try self.expectToken(&stack, Token.Id.LParen)) ?? continue;
1932 try stack.append(State { .ExpectToken = Token.Id.RParen });1928 try stack.append(State { .ExpectToken = Token.Id.RParen });
...@@ -1935,7 +1931,7 @@ pub const Parser = struct {...@@ -1935,7 +1931,7 @@ pub const Parser = struct {
1935 ast.NodeAsmInput {1931 ast.NodeAsmInput {
1936 .base = undefined,1932 .base = undefined,
1937 .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),1933 .symbolic_name = try self.createLiteral(arena, ast.NodeIdentifier, symbolic_name),
1938 .constraint = try self.createLiteral(arena, ast.NodeStringLiteral, constraint),1934 .constraint = constraint,
1939 .expr = undefined,1935 .expr = undefined,
1940 }1936 }
1941 );1937 );
...@@ -1944,13 +1940,13 @@ pub const Parser = struct {...@@ -1944,13 +1940,13 @@ pub const Parser = struct {
1944 },1940 },
19451941
1946 State.AsmClopperItems => |items| {1942 State.AsmClopperItems => |items| {
1947 const string = self.getNextToken();1943 const string_token = self.getNextToken();
1948 if (string.id != Token.Id.StringLiteral) {1944 const string = (try self.parseStringLiteral(arena, string_token)) ?? {
1949 self.putBackToken(string);1945 self.putBackToken(string_token);
1950 continue;1946 continue;
1951 }1947 };
1948 try items.append(string);
19521949
1953 try items.append(try self.createLiteral(arena, ast.NodeStringLiteral, string));
1954 stack.append(State { .AsmClopperItems = items }) catch unreachable;1950 stack.append(State { .AsmClopperItems = items }) catch unreachable;
1955 try stack.append(State { .IfToken = Token.Id.Comma });1951 try stack.append(State { .IfToken = Token.Id.Comma });
1956 },1952 },
...@@ -2742,6 +2738,37 @@ pub const Parser = struct {...@@ -2742,6 +2738,37 @@ pub const Parser = struct {
2742 }2738 }
2743 }2739 }
27442740
2741 fn parseStringLiteral(self: &Parser, arena: &mem.Allocator, token: &const Token) !?&ast.Node {
2742 switch (token.id) {
2743 Token.Id.StringLiteral => {
2744 return &(try self.createLiteral(arena, ast.NodeStringLiteral, token)).base;
2745 },
2746 Token.Id.MultilineStringLiteralLine => {
2747 const node = try self.createNode(arena, ast.NodeMultilineStringLiteral,
2748 ast.NodeMultilineStringLiteral {
2749 .base = undefined,
2750 .tokens = ArrayList(Token).init(arena),
2751 }
2752 );
2753 try node.tokens.append(token);
2754 while (true) {
2755 const multiline_str = self.getNextToken();
2756 if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
2757 self.putBackToken(multiline_str);
2758 break;
2759 }
2760
2761 try node.tokens.append(multiline_str);
2762 }
2763
2764 return &node.base;
2765 },
2766 // TODO: We shouldn't need a cast, but:
2767 // zig: /home/jc/Documents/zig/src/ir.cpp:7962: TypeTableEntry* ir_resolve_peer_types(IrAnalyze*, AstNode*, IrInstruction**, size_t): Assertion `err_set_type != nullptr' failed.
2768 else => return (?&ast.Node)(null),
2769 }
2770 }
2771
2745 fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool {2772 fn parseBlockExpr(self: &Parser, stack: &ArrayList(State), arena: &mem.Allocator, dest_ptr: &const DestPtr, token: &const Token) !bool {
2746 switch (token.id) {2773 switch (token.id) {
2747 Token.Id.Keyword_suspend => {2774 Token.Id.Keyword_suspend => {
...@@ -4085,8 +4112,6 @@ pub const Parser = struct {...@@ -4085,8 +4112,6 @@ pub const Parser = struct {
4085 try stream.write("volatile ");4112 try stream.write("volatile ");
4086 }4113 }
40874114
4088 try stream.print("({}", self.tokenizer.getTokenSlice(asm_node.template));
4089
4090 try stack.append(RenderState { .Indent = indent });4115 try stack.append(RenderState { .Indent = indent });
4091 try stack.append(RenderState { .Text = ")" });4116 try stack.append(RenderState { .Text = ")" });
4092 {4117 {
...@@ -4094,7 +4119,7 @@ pub const Parser = struct {...@@ -4094,7 +4119,7 @@ pub const Parser = struct {
4094 var i = cloppers.len;4119 var i = cloppers.len;
4095 while (i != 0) {4120 while (i != 0) {
4096 i -= 1;4121 i -= 1;
4097 try stack.append(RenderState { .Expression = &cloppers[i].base });4122 try stack.append(RenderState { .Expression = cloppers[i] });
40984123
4099 if (i != 0) {4124 if (i != 0) {
4100 try stack.append(RenderState { .Text = ", " });4125 try stack.append(RenderState { .Text = ", " });
...@@ -4163,6 +4188,8 @@ pub const Parser = struct {...@@ -4163,6 +4188,8 @@ pub const Parser = struct {
4163 try stack.append(RenderState.PrintIndent);4188 try stack.append(RenderState.PrintIndent);
4164 try stack.append(RenderState { .Indent = indent + indent_delta});4189 try stack.append(RenderState { .Indent = indent + indent_delta});
4165 try stack.append(RenderState { .Text = "\n" });4190 try stack.append(RenderState { .Text = "\n" });
4191 try stack.append(RenderState { .Expression = asm_node.template });
4192 try stack.append(RenderState { .Text = "(" });
4166 },4193 },
4167 ast.Node.Id.AsmInput => {4194 ast.Node.Id.AsmInput => {
4168 const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base);4195 const asm_input = @fieldParentPtr(ast.NodeAsmInput, "base", base);
...@@ -4170,7 +4197,7 @@ pub const Parser = struct {...@@ -4170,7 +4197,7 @@ pub const Parser = struct {
4170 try stack.append(RenderState { .Text = ")"});4197 try stack.append(RenderState { .Text = ")"});
4171 try stack.append(RenderState { .Expression = asm_input.expr});4198 try stack.append(RenderState { .Expression = asm_input.expr});
4172 try stack.append(RenderState { .Text = " ("});4199 try stack.append(RenderState { .Text = " ("});
4173 try stack.append(RenderState { .Expression = &asm_input.constraint.base});4200 try stack.append(RenderState { .Expression = asm_input.constraint });
4174 try stack.append(RenderState { .Text = "] "});4201 try stack.append(RenderState { .Text = "] "});
4175 try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base});4202 try stack.append(RenderState { .Expression = &asm_input.symbolic_name.base});
4176 try stack.append(RenderState { .Text = "["});4203 try stack.append(RenderState { .Text = "["});
...@@ -4189,7 +4216,7 @@ pub const Parser = struct {...@@ -4189,7 +4216,7 @@ pub const Parser = struct {
4189 },4216 },
4190 }4217 }
4191 try stack.append(RenderState { .Text = " ("});4218 try stack.append(RenderState { .Text = " ("});
4192 try stack.append(RenderState { .Expression = &asm_output.constraint.base});4219 try stack.append(RenderState { .Expression = asm_output.constraint });
4193 try stack.append(RenderState { .Text = "] "});4220 try stack.append(RenderState { .Text = "] "});
4194 try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base});4221 try stack.append(RenderState { .Expression = &asm_output.symbolic_name.base});
4195 try stack.append(RenderState { .Text = "["});4222 try stack.append(RenderState { .Text = "["});