authorgravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2017-12-23 17:47:48-07:00
committergravatar for thejoshwolfe@gmail.comJosh Wolfe <thejoshwolfe@gmail.com> 2017-12-23 17:47:48-07:00
log45ab752f9acd5247cd970ab01a388390ac3bdd94
tree53c64e568c0f855a3800b958c29727206cf71963
parent39c7bd24e4f768b23074b8634ac637b175b7639f

source files must end with newline

1 files changed, 52 insertions(+), 5 deletions(-)

src-self-hosted/tokenizer.zig+52-5
...@@ -70,6 +70,7 @@ pub const Token = struct {...@@ -70,6 +70,7 @@ pub const Token = struct {
70 Identifier,70 Identifier,
71 StringLiteral: StrLitKind,71 StringLiteral: StrLitKind,
72 Eof,72 Eof,
73 NoEolAtEof,
73 Builtin,74 Builtin,
74 Bang,75 Bang,
75 Equal,76 Equal,
...@@ -139,6 +140,7 @@ pub const Token = struct {...@@ -139,6 +140,7 @@ pub const Token = struct {
139pub const Tokenizer = struct {140pub const Tokenizer = struct {
140 buffer: []const u8,141 buffer: []const u8,
141 index: usize,142 index: usize,
143 actual_file_end: usize,
142144
143 pub const Location = struct {145 pub const Location = struct {
144 line: usize,146 line: usize,
...@@ -177,10 +179,24 @@ pub const Tokenizer = struct {...@@ -177,10 +179,24 @@ pub const Tokenizer = struct {
177 }179 }
178180
179 pub fn init(buffer: []const u8) -> Tokenizer {181 pub fn init(buffer: []const u8) -> Tokenizer {
180 return Tokenizer {182 if (buffer.len == 0 or buffer[buffer.len - 1] == '\n') {
181 .buffer = buffer,183 return Tokenizer {
182 .index = 0,184 .buffer = buffer,
183 };185 .index = 0,
186 .actual_file_end = buffer.len,
187 };
188 } else {
189 // last line is incomplete, so skip it, and give an error when we get there.
190 var source_len = buffer.len;
191 while (source_len > 0) : (source_len -= 1) {
192 if (buffer[source_len - 1] == '\n') break;
193 }
194 return Tokenizer {
195 .buffer = buffer[0..source_len],
196 .index = 0,
197 .actual_file_end = buffer.len,
198 };
199 }
184 }200 }
185201
186 const State = enum {202 const State = enum {
...@@ -497,7 +513,11 @@ pub const Tokenizer = struct {...@@ -497,7 +513,11 @@ pub const Tokenizer = struct {
497 }513 }
498 }514 }
499 result.end = self.index;515 result.end = self.index;
500 // TODO check state when returning EOF516 if (result.id == Token.Id.Eof and self.actual_file_end != self.buffer.len) {
517 // instead of an Eof, give an error token
518 result.id = Token.Id.NoEolAtEof;
519 result.end = self.actual_file_end;
520 }
501 return result;521 return result;
502 }522 }
503523
...@@ -507,3 +527,30 @@ pub const Tokenizer = struct {...@@ -507,3 +527,30 @@ pub const Tokenizer = struct {
507};527};
508528
509529
530
531test "tokenizer" {
532 // source must end with eol
533 testTokenize("no newline", []Token.Id {
534 }, false);
535 testTokenize("test\n", []Token.Id {
536 Token.Id.Keyword_test,
537 }, true);
538 testTokenize("test\nno newline", []Token.Id {
539 Token.Id.Keyword_test,
540 }, false);
541}
542
543fn testTokenize(source: []const u8, expected_tokens: []const Token.Id, expected_eol_at_eof: bool) {
544 var tokenizer = Tokenizer.init(source);
545 for (expected_tokens) |expected_token_id| {
546 const token = tokenizer.next();
547 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
548 switch (expected_token_id) {
549 Token.Id.StringLiteral => |kind| {
550 @panic("TODO: how do i test this?");
551 },
552 else => {},
553 }
554 }
555 std.debug.assert(tokenizer.next().id == if (expected_eol_at_eof) Token.Id.Eof else Token.Id.NoEolAtEof);
556}