authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-25 18:13:10-07:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2021-05-25 18:13:10-07:00
log44de8849803fd41b5e298926dcb835fcf3d31071
tree2a3d4780ae16119c35ff02c9b3c635fb8c3683d9
parent3ed6379192ceffdc1b85cd83c16e87cd477e2043

tokenizer: fix crash on multiline string with only 1 backslash

Closes #8904

1 files changed, 9 insertions(+), 2 deletions(-)

lib/std/zig/tokenizer.zig+9-2
...@@ -698,7 +698,10 @@ pub const Tokenizer = struct {...@@ -698,7 +698,10 @@ pub const Tokenizer = struct {
698 '\\' => {698 '\\' => {
699 state = .multiline_string_literal_line;699 state = .multiline_string_literal_line;
700 },700 },
701 else => break,701 else => {
702 result.tag = .invalid;
703 break;
704 },
702 },705 },
703 .string_literal => switch (c) {706 .string_literal => switch (c) {
704 '\\' => {707 '\\' => {
...@@ -1960,7 +1963,7 @@ test "tokenizer - number literals octal" {...@@ -1960,7 +1963,7 @@ test "tokenizer - number literals octal" {
1960 try testTokenize("0o_,", &.{ .invalid, .identifier, .comma });1963 try testTokenize("0o_,", &.{ .invalid, .identifier, .comma });
1961}1964}
19621965
1963test "tokenizer - number literals hexadeciaml" {1966test "tokenizer - number literals hexadecimal" {
1964 try testTokenize("0x0", &.{.integer_literal});1967 try testTokenize("0x0", &.{.integer_literal});
1965 try testTokenize("0x1", &.{.integer_literal});1968 try testTokenize("0x1", &.{.integer_literal});
1966 try testTokenize("0x2", &.{.integer_literal});1969 try testTokenize("0x2", &.{.integer_literal});
...@@ -2046,6 +2049,10 @@ test "tokenizer - number literals hexadeciaml" {...@@ -2046,6 +2049,10 @@ test "tokenizer - number literals hexadeciaml" {
2046 try testTokenize("0x0.0p0_", &.{ .invalid, .eof });2049 try testTokenize("0x0.0p0_", &.{ .invalid, .eof });
2047}2050}
20482051
2052test "tokenizer - multi line string literal with only 1 backslash" {
2053 try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });
2054}
2055
2049fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {2056fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
2050 var tokenizer = Tokenizer.init(source);2057 var tokenizer = Tokenizer.init(source);
2051 for (expected_tokens) |expected_token_id| {2058 for (expected_tokens) |expected_token_id| {