authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-31 14:04:15-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2024-07-31 16:57:42-07:00
logc2b8afcac9e427102370dc5bac8c3d9621eee6d8
treed1fb10fbbd76607cb486fed08bd9a1ddf7ed221d
parenta7029496d153bd2ba4e91ef561a399cad6d77307

tokenizer: tabs and carriage returns spec conformance

2 files changed, 94 insertions(+), 43 deletions(-)

lib/std/zig/tokenizer.zig+94-35
...@@ -424,10 +424,7 @@ pub const Tokenizer = struct {...@@ -424,10 +424,7 @@ pub const Tokenizer = struct {
424 };424 };
425 state = .invalid;425 state = .invalid;
426 },426 },
427 '\r' => {427 ' ', '\n', '\t', '\r' => {
428 state = .expect_newline;
429 },
430 ' ', '\n', '\t' => {
431 result.loc.start = self.index + 1;428 result.loc.start = self.index + 1;
432 },429 },
433 '"' => {430 '"' => {
...@@ -553,6 +550,13 @@ pub const Tokenizer = struct {...@@ -553,6 +550,13 @@ pub const Tokenizer = struct {
553 },550 },
554551
555 .expect_newline => switch (c) {552 .expect_newline => switch (c) {
553 0 => {
554 if (self.index == self.buffer.len) {
555 result.tag = .invalid;
556 break;
557 }
558 state = .invalid;
559 },
556 '\n' => {560 '\n' => {
557 result.loc.start = self.index + 1;561 result.loc.start = self.index + 1;
558 state = .start;562 state = .start;
...@@ -846,7 +850,15 @@ pub const Tokenizer = struct {...@@ -846,7 +850,15 @@ pub const Tokenizer = struct {
846 self.index += 1;850 self.index += 1;
847 break;851 break;
848 },852 },
849 0x01...0x08, 0x0b...0x1f, 0x7f => {853 '\r' => {
854 if (self.buffer[self.index + 1] == '\n') {
855 self.index += 2;
856 break;
857 } else {
858 state = .invalid;
859 }
860 },
861 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
850 state = .invalid;862 state = .invalid;
851 },863 },
852 else => continue,864 else => continue,
...@@ -1091,7 +1103,7 @@ pub const Tokenizer = struct {...@@ -1091,7 +1103,7 @@ pub const Tokenizer = struct {
1091 state = .start;1103 state = .start;
1092 result.loc.start = self.index + 1;1104 result.loc.start = self.index + 1;
1093 },1105 },
1094 0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {1106 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1095 state = .invalid;1107 state = .invalid;
1096 },1108 },
1097 else => {1109 else => {
...@@ -1099,14 +1111,23 @@ pub const Tokenizer = struct {...@@ -1099,14 +1111,23 @@ pub const Tokenizer = struct {
1099 },1111 },
1100 },1112 },
1101 .doc_comment_start => switch (c) {1113 .doc_comment_start => switch (c) {
1102 0, '\n', '\r' => {1114 0, '\n' => {
1103 result.tag = .doc_comment;1115 result.tag = .doc_comment;
1104 break;1116 break;
1105 },1117 },
1118 '\r' => {
1119 if (self.buffer[self.index + 1] == '\n') {
1120 self.index += 1;
1121 result.tag = .doc_comment;
1122 break;
1123 } else {
1124 state = .invalid;
1125 }
1126 },
1106 '/' => {1127 '/' => {
1107 state = .line_comment;1128 state = .line_comment;
1108 },1129 },
1109 0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {1130 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1110 state = .invalid;1131 state = .invalid;
1111 },1132 },
1112 else => {1133 else => {
...@@ -1135,16 +1156,24 @@ pub const Tokenizer = struct {...@@ -1135,16 +1156,24 @@ pub const Tokenizer = struct {
1135 state = .start;1156 state = .start;
1136 result.loc.start = self.index + 1;1157 result.loc.start = self.index + 1;
1137 },1158 },
1138 0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {1159 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1139 state = .invalid;1160 state = .invalid;
1140 },1161 },
1141 else => continue,1162 else => continue,
1142 },1163 },
1143 .doc_comment => switch (c) {1164 .doc_comment => switch (c) {
1144 0, '\n', '\r' => {1165 0, '\n' => {
1145 break;1166 break;
1146 },1167 },
1147 0x01...0x08, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {1168 '\r' => {
1169 if (self.buffer[self.index + 1] == '\n') {
1170 self.index += 1;
1171 break;
1172 } else {
1173 state = .invalid;
1174 }
1175 },
1176 0x01...0x09, 0x0b...0x0c, 0x0e...0x1f, 0x7f => {
1148 state = .invalid;1177 state = .invalid;
1149 },1178 },
1150 else => continue,1179 else => continue,
...@@ -1386,30 +1415,6 @@ test "string identifier and builtin fns" {...@@ -1386,30 +1415,6 @@ test "string identifier and builtin fns" {
1386 });1415 });
1387}1416}
13881417
1389test "multiline string literal with literal tab" {
1390 try testTokenize(
1391 \\\\foo bar
1392 , &.{
1393 .multiline_string_literal_line,
1394 });
1395}
1396
1397test "comments with literal tab" {
1398 try testTokenize(
1399 \\//foo bar
1400 \\//!foo bar
1401 \\///foo bar
1402 \\// foo
1403 \\/// foo
1404 \\/// /foo
1405 , &.{
1406 .container_doc_comment,
1407 .doc_comment,
1408 .doc_comment,
1409 .doc_comment,
1410 });
1411}
1412
1413test "pipe and then invalid" {1418test "pipe and then invalid" {
1414 try testTokenize("||=", &.{1419 try testTokenize("||=", &.{
1415 .pipe_pipe,1420 .pipe_pipe,
...@@ -1767,6 +1772,60 @@ test "null byte before eof" {...@@ -1767,6 +1772,60 @@ test "null byte before eof" {
1767 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });1772 try testTokenize("/// NUL\x00\n", &.{ .doc_comment, .invalid });
1768}1773}
17691774
1775test "invalid tabs and carriage returns" {
1776 // "Inside Line Comments and Documentation Comments, Any TAB is rejected by
1777 // the grammar since it is ambiguous how it should be rendered."
1778 // https://github.com/ziglang/zig-spec/issues/38
1779 try testTokenize("//\t", &.{.invalid});
1780 try testTokenize("// \t", &.{.invalid});
1781 try testTokenize("///\t", &.{.invalid});
1782 try testTokenize("/// \t", &.{.invalid});
1783 try testTokenize("//!\t", &.{.invalid});
1784 try testTokenize("//! \t", &.{.invalid});
1785
1786 // "Inside Line Comments and Documentation Comments, CR directly preceding
1787 // NL is unambiguously part of the newline sequence. It is accepted by the
1788 // grammar and removed by zig fmt, leaving only NL. CR anywhere else is
1789 // rejected by the grammar."
1790 // https://github.com/ziglang/zig-spec/issues/38
1791 try testTokenize("//\r", &.{.invalid});
1792 try testTokenize("// \r", &.{.invalid});
1793 try testTokenize("///\r", &.{.invalid});
1794 try testTokenize("/// \r", &.{.invalid});
1795 try testTokenize("//\r ", &.{.invalid});
1796 try testTokenize("// \r ", &.{.invalid});
1797 try testTokenize("///\r ", &.{.invalid});
1798 try testTokenize("/// \r ", &.{.invalid});
1799 try testTokenize("//\r\n", &.{});
1800 try testTokenize("// \r\n", &.{});
1801 try testTokenize("///\r\n", &.{.doc_comment});
1802 try testTokenize("/// \r\n", &.{.doc_comment});
1803 try testTokenize("//!\r", &.{.invalid});
1804 try testTokenize("//! \r", &.{.invalid});
1805 try testTokenize("//!\r ", &.{.invalid});
1806 try testTokenize("//! \r ", &.{.invalid});
1807 try testTokenize("//!\r\n", &.{.container_doc_comment});
1808 try testTokenize("//! \r\n", &.{.container_doc_comment});
1809
1810 // The control characters TAB and CR are rejected by the grammar inside multi-line string literals,
1811 // except if CR is directly before NL.
1812 // https://github.com/ziglang/zig-spec/issues/38
1813 try testTokenize("\\\\\r", &.{.invalid});
1814 try testTokenize("\\\\\r ", &.{.invalid});
1815 try testTokenize("\\\\ \r", &.{.invalid});
1816 try testTokenize("\\\\\t", &.{.invalid});
1817 try testTokenize("\\\\\t ", &.{.invalid});
1818 try testTokenize("\\\\ \t", &.{.invalid});
1819 try testTokenize("\\\\\r\n", &.{.multiline_string_literal_line});
1820
1821 // "TAB used as whitespace is...accepted by the grammar. CR used as
1822 // whitespace, whether directly preceding NL or stray, is...accepted by the
1823 // grammar."
1824 // https://github.com/ziglang/zig-spec/issues/38
1825 try testTokenize("\tpub\tswitch\t", &.{ .keyword_pub, .keyword_switch });
1826 try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch });
1827}
1828
1770fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {1829fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
1771 var tokenizer = Tokenizer.init(source);1830 var tokenizer = Tokenizer.init(source);
1772 for (expected_token_tags) |expected_token_tag| {1831 for (expected_token_tags) |expected_token_tag| {
test/compile_errors.zig-8
...@@ -38,14 +38,6 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {...@@ -38,14 +38,6 @@ pub fn addCases(ctx: *Cases, b: *std.Build) !void {
38 });38 });
39 }39 }
4040
41 {
42 const case = ctx.obj("isolated carriage return in multiline string literal", b.graph.host);
43
44 case.addError("const foo = \\\\\test\r\r rogue carriage return\n;", &[_][]const u8{
45 ":1:13: error: expected expression, found 'invalid token'",
46 });
47 }
48
49 {41 {
50 const case = ctx.obj("missing semicolon at EOF", b.graph.host);42 const case = ctx.obj("missing semicolon at EOF", b.graph.host);
51 case.addError(43 case.addError(