authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-13 10:28:55-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-13 10:28:55-05:00
log2dcff95bd2fb8f377491ac48b0ecf961183abcd7
tree8fc8547d6f4bb443d01ff8a5923b6e4332a344e2
parentdfbb8254ca97154b5314bde03655417c1dca86ae

self hosted: add tokenizer test fix eof handling

1 files changed, 30 insertions(+), 3 deletions(-)

std/zig/tokenizer.zig+30-3
...@@ -72,6 +72,8 @@ pub const Token = struct {...@@ -72,6 +72,8 @@ pub const Token = struct {
72 Eof,72 Eof,
73 Builtin,73 Builtin,
74 Bang,74 Bang,
75 Pipe,
76 PipeEqual,
75 Equal,77 Equal,
76 EqualEqual,78 EqualEqual,
77 BangEqual,79 BangEqual,
...@@ -193,6 +195,7 @@ pub const Tokenizer = struct {...@@ -193,6 +195,7 @@ pub const Tokenizer = struct {
193 StringLiteralBackslash,195 StringLiteralBackslash,
194 Equal,196 Equal,
195 Bang,197 Bang,
198 Pipe,
196 Minus,199 Minus,
197 Slash,200 Slash,
198 LineComment,201 LineComment,
...@@ -248,6 +251,9 @@ pub const Tokenizer = struct {...@@ -248,6 +251,9 @@ pub const Tokenizer = struct {
248 '!' => {251 '!' => {
249 state = State.Bang;252 state = State.Bang;
250 },253 },
254 '|' => {
255 state = State.Pipe;
256 },
251 '(' => {257 '(' => {
252 result.id = Token.Id.LParen;258 result.id = Token.Id.LParen;
253 self.index += 1;259 self.index += 1;
...@@ -394,6 +400,18 @@ pub const Tokenizer = struct {...@@ -394,6 +400,18 @@ pub const Tokenizer = struct {
394 },400 },
395 },401 },
396402
403 State.Pipe => switch (c) {
404 '=' => {
405 result.id = Token.Id.PipeEqual;
406 self.index += 1;
407 break;
408 },
409 else => {
410 result.id = Token.Id.Pipe;
411 break;
412 },
413 },
414
397 State.Equal => switch (c) {415 State.Equal => switch (c) {
398 '=' => {416 '=' => {
399 result.id = Token.Id.EqualEqual;417 result.id = Token.Id.EqualEqual;
...@@ -525,9 +543,7 @@ pub const Tokenizer = struct {...@@ -525,9 +543,7 @@ pub const Tokenizer = struct {
525 else => break,543 else => break,
526 },544 },
527 }545 }
528 }546 } else if (self.index == self.buffer.len) {
529 result.end = self.index;
530 if (self.index == self.buffer.len) {
531 switch (state) {547 switch (state) {
532 State.Start,548 State.Start,
533 State.C,549 State.C,
...@@ -578,6 +594,9 @@ pub const Tokenizer = struct {...@@ -578,6 +594,9 @@ pub const Tokenizer = struct {
578 State.Period2 => {594 State.Period2 => {
579 result.id = Token.Id.Ellipsis2;595 result.id = Token.Id.Ellipsis2;
580 },596 },
597 State.Pipe => {
598 result.id = Token.Id.Pipe;
599 },
581 }600 }
582 }601 }
583 if (result.id == Token.Id.Eof) {602 if (result.id == Token.Id.Eof) {
...@@ -587,6 +606,7 @@ pub const Tokenizer = struct {...@@ -587,6 +606,7 @@ pub const Tokenizer = struct {
587 }606 }
588 }607 }
589608
609 result.end = self.index;
590 return result;610 return result;
591 }611 }
592612
...@@ -716,6 +736,13 @@ test "tokenizer - string identifier and builtin fns" {...@@ -716,6 +736,13 @@ test "tokenizer - string identifier and builtin fns" {
716 );736 );
717}737}
718738
739test "tokenizer - pipe and then invalid" {
740 testTokenize("||=", []Token.Id{
741 Token.Id.Pipe,
742 Token.Id.PipeEqual,
743 });
744}
745
719fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {746fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
720 var tokenizer = Tokenizer.init(source);747 var tokenizer = Tokenizer.init(source);
721 for (expected_tokens) |expected_token_id| {748 for (expected_tokens) |expected_token_id| {