authorgravatar for daniele.cocca@gmail.comDaniele Cocca <daniele.cocca@gmail.com> 2021-07-01 11:54:09+01:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2021-07-07 20:03:19+03:00
log9de452f9a69d5590743a194bc2d0817d26d66a0b
treeb991b65c8fbe0864a6c636f0b0235bc9e3ace840
parentb936bbd2f10e28b4b6c34aa75fcf8dff23cb6aed

Skip over CRs at the end of multiline literals

Fixes #9257. This is needed when tokenizing input containing DOS line endings, i.e. the CRLF sequence.

1 files changed, 5 insertions(+), 1 deletions(-)

lib/std/zig/tokenizer.zig+5-1
......@@ -845,7 +845,7 @@ pub const Tokenizer = struct {
845845 self.index += 1;
846846 break;
847847 },
848 '\t' => {},
848 '\t', '\r' => {},
849849 else => self.checkLiteralCharacter(),
850850 },
851851
......@@ -1936,6 +1936,10 @@ test "tokenizer - invalid builtin identifiers" {
19361936 try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
19371937}
19381938
1939test "tokenizer - multiline string literal with a DOS-encoded CRLF line endings (issue #9257)" {
1940 try testTokenize("\\\\foobar\r\n", &.{.multiline_string_literal_line});
1941}
1942
19391943fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
19401944 var tokenizer = Tokenizer.init(source);
19411945 for (expected_tokens) |expected_token_id| {