Commit 34af38e09b33957a9f677e42d57e9cd96f859b76 | zig

author	Jimmi Holst Christensen <jhc@dismail.de> 2018-04-10 11:35:41+02:00
committer	Jimmi Holst Christensen <jhc@dismail.de> 2018-04-10 11:35:41+02:00
log	34af38e09b33957a9f677e42d57e9cd96f859b76
tree	351345ae647f4c138451df5897dac65c9d0b1ba8
parent	f85b9f2bf3ac0c5af7804921d2003c2992558cb7

std.zig.tokinizer: fixed failing tests

1 files changed, 19 insertions(+), 7 deletions(-)

std
- zig
  - tokenizer.zig

std/zig/tokenizer.zig+19-7

 @@ -613,17 +613,25 @@ pub const Tokenizer = struct {
                     '\\' => {
                         state = State.CharLiteralBackslash;
                     },
-                    '\'' => break, // Look for this error later.
+                    '\'' => {
+                        result.id = Token.Id.Invalid;
+                        break;
+                    },
                     else => {
-                        if (c < 0x20 or c == 0x7f)
+                        if (c < 0x20 or c == 0x7f) {
-                            break; // Look for this error later.
+                            result.id = Token.Id.Invalid;
+                            break;
+                        }
                         state = State.CharLiteralEnd;
                     }
                 },
                 State.CharLiteralBackslash => switch (c) {
-                    '\n' => break, // Look for this error later.
+                    '\n' => {
+                        result.id = Token.Id.Invalid;
+                        break;
+                    },
                     else => {
                         state = State.CharLiteralEnd;
                     },
 @@ -635,7 +643,10 @@ pub const Tokenizer = struct {
                         self.index += 1;
                         break;
                     },
-                    else => break, // Look for this error later.
+                    else => {
+                        result.id = Token.Id.Invalid;
+                        break;
+                    },
                 },
                 State.MultilineStringLiteralLine => switch (c) {
 @@ -903,7 +914,6 @@ pub const Tokenizer = struct {
                 State.FloatExponentNumber,
                 State.StringLiteral, // find this error later
                 State.MultilineStringLiteralLine,
-                State.CharLiteralEnd,
                 State.Builtin => {},
                 State.Identifier => {
 @@ -922,6 +932,7 @@ pub const Tokenizer = struct {
                 State.MultilineStringLiteralLineBackslash,
                 State.CharLiteral,
                 State.CharLiteralBackslash,
+                State.CharLiteralEnd,
                 State.StringLiteralBackslash => {
                     result.id = Token.Id.Invalid;
                 },
 @@ -1073,7 +1084,7 @@ test "tokenizer - invalid token characters" {
     testTokenize("`", []Token.Id{Token.Id.Invalid});
     testTokenize("'c", []Token.Id {Token.Id.Invalid});
     testTokenize("'", []Token.Id {Token.Id.Invalid});
-    testTokenize("''", []Token.Id {Token.Id.Invalid});
+    testTokenize("''", []Token.Id {Token.Id.Invalid, Token.Id.Invalid});
 }
 test "tokenizer - invalid literal/comment characters" {
 @@ -1147,6 +1158,7 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
     var tokenizer = Tokenizer.init(source);
     for (expected_tokens) |expected_token_id| {
         const token = tokenizer.next();
+        std.debug.warn("{} {}\n", @tagName(expected_token_id), @tagName(token.id));
         std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
         switch (expected_token_id) {
             Token.Id.StringLiteral => |expected_kind| {