authorgravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:04:54+02:00
committergravatar for jhc@dismail.deJimmi Holst Christensen <jhc@dismail.de> 2018-03-31 21:04:54+02:00
log4d8f9e2295bb672f70550b3bb5a4cb667f68bb70
tree434ac90b56af673e28fb1b42effb8351c595a7f2
parent4793c3397e17169317d2feded9ca6901ff0e99e8

std.zig.parser now parses multi line strings

3 files changed, 108 insertions(+), 5 deletions(-)

std/zig/ast.zig+21
...@@ -20,6 +20,7 @@ pub const Node = struct {...@@ -20,6 +20,7 @@ pub const Node = struct {
20 IntegerLiteral,20 IntegerLiteral,
21 FloatLiteral,21 FloatLiteral,
22 StringLiteral,22 StringLiteral,
23 MultilineStringLiteral,
23 UndefinedLiteral,24 UndefinedLiteral,
24 BuiltinCall,25 BuiltinCall,
25 Call,26 Call,
...@@ -40,6 +41,7 @@ pub const Node = struct {...@@ -40,6 +41,7 @@ pub const Node = struct {
40 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),41 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
41 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),42 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
42 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),43 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).iterate(index),
44 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).iterate(index),
43 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),45 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).iterate(index),
44 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),46 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
45 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),47 Id.Call => @fieldParentPtr(NodeCall, "base", base).iterate(index),
...@@ -61,6 +63,7 @@ pub const Node = struct {...@@ -61,6 +63,7 @@ pub const Node = struct {
61 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),63 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(),
62 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),64 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(),
63 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),65 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(),
66 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).firstToken(),
64 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),67 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).firstToken(),
65 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),68 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(),
66 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),69 Id.Call => @fieldParentPtr(NodeCall, "base", base).firstToken(),
...@@ -82,6 +85,7 @@ pub const Node = struct {...@@ -82,6 +85,7 @@ pub const Node = struct {
82 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),85 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(),
83 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),86 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(),
84 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),87 Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(),
88 Id.MultilineStringLiteral => @fieldParentPtr(NodeMultilineStringLiteral, "base", base).lastToken(),
85 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),89 Id.UndefinedLiteral => @fieldParentPtr(NodeUndefinedLiteral, "base", base).lastToken(),
86 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),90 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(),
87 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),91 Id.Call => @fieldParentPtr(NodeCall, "base", base).lastToken(),
...@@ -587,6 +591,23 @@ pub const NodeStringLiteral = struct {...@@ -587,6 +591,23 @@ pub const NodeStringLiteral = struct {
587 }591 }
588};592};
589593
594pub const NodeMultilineStringLiteral = struct {
595 base: Node,
596 tokens: ArrayList(Token),
597
598 pub fn iterate(self: &NodeMultilineStringLiteral, index: usize) ?&Node {
599 return null;
600 }
601
602 pub fn firstToken(self: &NodeMultilineStringLiteral) Token {
603 return self.tokens.at(0);
604 }
605
606 pub fn lastToken(self: &NodeMultilineStringLiteral) Token {
607 return self.tokens.at(self.tokens.len - 1);
608 }
609};
610
590pub const NodeUndefinedLiteral = struct {611pub const NodeUndefinedLiteral = struct {
591 base: Node,612 base: Node,
592 token: Token,613 token: Token,
std/zig/parser.zig+48-4
...@@ -456,6 +456,30 @@ pub const Parser = struct {...@@ -456,6 +456,30 @@ pub const Parser = struct {
456 try stack.append(State.AfterOperand);456 try stack.append(State.AfterOperand);
457 continue;457 continue;
458 },458 },
459 Token.Id.MultilineStringLiteralLine => {
460 const node = try arena.create(ast.NodeMultilineStringLiteral);
461 *node = ast.NodeMultilineStringLiteral {
462 .base = self.initNode(ast.Node.Id.MultilineStringLiteral),
463 .tokens = ArrayList(Token).init(arena),
464 };
465 try node.tokens.append(token);
466
467 while (true) {
468 const multiline_str = self.getNextToken();
469 if (multiline_str.id != Token.Id.MultilineStringLiteralLine) {
470 self.putBackToken(multiline_str);
471 break;
472 }
473
474 try node.tokens.append(multiline_str);
475 }
476
477 try stack.append(State {
478 .Operand = &node.base
479 });
480 try stack.append(State.AfterOperand);
481 continue;
482 },
459483
460 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),484 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
461 }485 }
...@@ -1427,6 +1451,20 @@ pub const Parser = struct {...@@ -1427,6 +1451,20 @@ pub const Parser = struct {
1427 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);1451 const string_literal = @fieldParentPtr(ast.NodeStringLiteral, "base", base);
1428 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));1452 try stream.print("{}", self.tokenizer.getTokenSlice(string_literal.token));
1429 },1453 },
1454 ast.Node.Id.MultilineStringLiteral => {
1455 const multiline_str_literal = @fieldParentPtr(ast.NodeMultilineStringLiteral, "base", base);
1456 try stream.print("\n");
1457
1458 var i : usize = 0;
1459 indent += 4;
1460 while (i < multiline_str_literal.tokens.len) : (i += 1) {
1461 const t = multiline_str_literal.tokens.at(i);
1462 try stream.writeByteNTimes(' ', indent);
1463 try stream.print("{}", self.tokenizer.getTokenSlice(t));
1464 }
1465 try stream.writeByteNTimes(' ', indent);
1466 indent -= 4;
1467 },
1430 ast.Node.Id.UndefinedLiteral => {1468 ast.Node.Id.UndefinedLiteral => {
1431 const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base);1469 const undefined_literal = @fieldParentPtr(ast.NodeUndefinedLiteral, "base", base);
1432 try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token));1470 try stream.print("{}", self.tokenizer.getTokenSlice(undefined_literal.token));
...@@ -1806,6 +1844,16 @@ test "zig fmt: extern function" {...@@ -1806,6 +1844,16 @@ test "zig fmt: extern function" {
1806 );1844 );
1807}1845}
18081846
1847test "zig fmt: multiline string" {
1848 try testCanonical(
1849 \\const s =
1850 \\ \\ something
1851 \\ \\ something else
1852 \\ ;
1853 \\
1854 );
1855}
1856
1809test "zig fmt: values" {1857test "zig fmt: values" {
1810 try testCanonical(1858 try testCanonical(
1811 \\test "values" {1859 \\test "values" {
...@@ -1813,10 +1861,6 @@ test "zig fmt: values" {...@@ -1813,10 +1861,6 @@ test "zig fmt: values" {
1813 \\ 1.0;1861 \\ 1.0;
1814 \\ "string";1862 \\ "string";
1815 \\ c"cstring";1863 \\ c"cstring";
1816 \\ \\ Multi
1817 \\ \\ line
1818 \\ \\ string
1819 \\ ;
1820 \\ 'c';1864 \\ 'c';
1821 \\ true;1865 \\ true;
1822 \\ false;1866 \\ false;
std/zig/tokenizer.zig+39-1
...@@ -72,6 +72,7 @@ pub const Token = struct {...@@ -72,6 +72,7 @@ pub const Token = struct {
72 Invalid,72 Invalid,
73 Identifier,73 Identifier,
74 StringLiteral: StrLitKind,74 StringLiteral: StrLitKind,
75 MultilineStringLiteralLine: StrLitKind,
75 StringIdentifier,76 StringIdentifier,
76 Eof,77 Eof,
77 Builtin,78 Builtin,
...@@ -225,6 +226,9 @@ pub const Tokenizer = struct {...@@ -225,6 +226,9 @@ pub const Tokenizer = struct {
225 C,226 C,
226 StringLiteral,227 StringLiteral,
227 StringLiteralBackslash,228 StringLiteralBackslash,
229 MultilineStringLiteralLine,
230 MultilineStringLiteralLineBackslash,
231 Backslash,
228 Equal,232 Equal,
229 Bang,233 Bang,
230 Pipe,234 Pipe,
...@@ -352,6 +356,10 @@ pub const Tokenizer = struct {...@@ -352,6 +356,10 @@ pub const Tokenizer = struct {
352 '^' => {356 '^' => {
353 state = State.Caret;357 state = State.Caret;
354 },358 },
359 '\\' => {
360 state = State.Backslash;
361 result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.Normal };
362 },
355 '{' => {363 '{' => {
356 result.id = Token.Id.LBrace;364 result.id = Token.Id.LBrace;
357 self.index += 1;365 self.index += 1;
...@@ -532,8 +540,17 @@ pub const Tokenizer = struct {...@@ -532,8 +540,17 @@ pub const Tokenizer = struct {
532 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},540 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
533 else => break,541 else => break,
534 },542 },
543 State.Backslash => switch (c) {
544 '\\' => {
545 state = State.MultilineStringLiteralLine;
546 },
547 else => break,
548 },
535 State.C => switch (c) {549 State.C => switch (c) {
536 '\\' => @panic("TODO"),550 '\\' => {
551 state = State.Backslash;
552 result.id = Token.Id { .MultilineStringLiteralLine = Token.StrLitKind.C };
553 },
537 '"' => {554 '"' => {
538 state = State.StringLiteral;555 state = State.StringLiteral;
539 result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };556 result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
...@@ -562,6 +579,24 @@ pub const Tokenizer = struct {...@@ -562,6 +579,24 @@ pub const Tokenizer = struct {
562 },579 },
563 },580 },
564581
582 State.MultilineStringLiteralLine => switch (c) {
583 '\\' => {
584 state = State.MultilineStringLiteralLineBackslash;
585 },
586 '\n' => {
587 self.index += 1;
588 break;
589 },
590 else => self.checkLiteralCharacter(),
591 },
592
593 State.MultilineStringLiteralLineBackslash => switch (c) {
594 '\n' => break, // Look for this error later.
595 else => {
596 state = State.MultilineStringLiteralLine;
597 },
598 },
599
565 State.Bang => switch (c) {600 State.Bang => switch (c) {
566 '=' => {601 '=' => {
567 result.id = Token.Id.BangEqual;602 result.id = Token.Id.BangEqual;
...@@ -811,6 +846,7 @@ pub const Tokenizer = struct {...@@ -811,6 +846,7 @@ pub const Tokenizer = struct {
811 State.FloatFraction,846 State.FloatFraction,
812 State.FloatExponentNumber,847 State.FloatExponentNumber,
813 State.StringLiteral, // find this error later848 State.StringLiteral, // find this error later
849 State.MultilineStringLiteralLine,
814 State.Builtin => {},850 State.Builtin => {},
815851
816 State.Identifier => {852 State.Identifier => {
...@@ -825,6 +861,8 @@ pub const Tokenizer = struct {...@@ -825,6 +861,8 @@ pub const Tokenizer = struct {
825 State.NumberDot,861 State.NumberDot,
826 State.FloatExponentUnsigned,862 State.FloatExponentUnsigned,
827 State.SawAtSign,863 State.SawAtSign,
864 State.Backslash,
865 State.MultilineStringLiteralLineBackslash,
828 State.StringLiteralBackslash => {866 State.StringLiteralBackslash => {
829 result.id = Token.Id.Invalid;867 result.id = Token.Id.Invalid;
830 },868 },