authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-12 21:25:38-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2018-02-12 21:26:15-05:00
logdfbb8254ca97154b5314bde03655417c1dca86ae
tree0360cdd64d2289a6321133e4e07e4eb09a26177d
parentb4e44c4e80fd4f477a46251fdaf485a742ffe6f7

fix self hosted tokenizer handling of EOF

2 files changed, 107 insertions(+), 27 deletions(-)

std/zig/ast.zig+17-15
......@@ -18,6 +18,7 @@ pub const Node = struct {
1818 PrefixOp,
1919 IntegerLiteral,
2020 FloatLiteral,
21 BuiltinCall,
2122 };
2223
2324 pub fn iterate(base: &Node, index: usize) ?&Node {
......@@ -32,21 +33,7 @@ pub const Node = struct {
3233 Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
3334 Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
3435 Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
35 };
36 }
37
38 pub fn destroy(base: &Node, allocator: &mem.Allocator) void {
39 return switch (base.id) {
40 Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
41 Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
42 Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
43 Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
44 Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
45 Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
46 Id.InfixOp => allocator.destroy(@fieldParentPtr(NodeInfixOp, "base", base)),
47 Id.PrefixOp => allocator.destroy(@fieldParentPtr(NodePrefixOp, "base", base)),
48 Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
49 Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
36 Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index),
5037 };
5138 }
5239};
......@@ -269,3 +256,18 @@ pub const NodeFloatLiteral = struct {
269256 return null;
270257 }
271258};
259
260pub const NodeBuiltinCall = struct {
261 base: Node,
262 builtin_token: Token,
263 params: ArrayList(&Node),
264
265 pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node {
266 var i = index;
267
268 if (i < self.params.len) return self.params.at(i);
269 i -= self.params.len;
270
271 return null;
272 }
273};
std/zig/tokenizer.zig+90-12
......@@ -68,6 +68,7 @@ pub const Token = struct {
6868 Invalid,
6969 Identifier,
7070 StringLiteral: StrLitKind,
71 StringIdentifier,
7172 Eof,
7273 Builtin,
7374 Bang,
......@@ -205,6 +206,7 @@ pub const Tokenizer = struct {
205206 Ampersand,
206207 Period,
207208 Period2,
209 SawAtSign,
208210 };
209211
210212 pub fn next(self: &Tokenizer) Token {
......@@ -238,8 +240,7 @@ pub const Tokenizer = struct {
238240 result.id = Token.Id.Identifier;
239241 },
240242 '@' => {
241 state = State.Builtin;
242 result.id = Token.Id.Builtin;
243 state = State.SawAtSign;
243244 },
244245 '=' => {
245246 state = State.Equal;
......@@ -313,6 +314,20 @@ pub const Tokenizer = struct {
313314 break;
314315 },
315316 },
317
318 State.SawAtSign => switch (c) {
319 '"' => {
320 result.id = Token.Id.StringIdentifier;
321 state = State.StringLiteral;
322 },
323 else => {
324 // reinterpret as a builtin
325 self.index -= 1;
326 state = State.Builtin;
327 result.id = Token.Id.Builtin;
328 },
329 },
330
316331 State.Ampersand => switch (c) {
317332 '=' => {
318333 result.id = Token.Id.AmpersandEqual;
......@@ -512,7 +527,59 @@ pub const Tokenizer = struct {
512527 }
513528 }
514529 result.end = self.index;
530 if (self.index == self.buffer.len) {
531 switch (state) {
532 State.Start,
533 State.C,
534 State.IntegerLiteral,
535 State.IntegerLiteralWithRadix,
536 State.FloatFraction,
537 State.FloatExponentNumber,
538 State.StringLiteral, // find this error later
539 State.Builtin => {},
540
541 State.Identifier => {
542 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
543 result.id = id;
544 }
545 },
546 State.LineComment => {
547 result.id = Token.Id.Eof;
548 },
549
550 State.NumberDot,
551 State.FloatExponentUnsigned,
552 State.SawAtSign,
553 State.StringLiteralBackslash => {
554 result.id = Token.Id.Invalid;
555 },
515556
557 State.Equal => {
558 result.id = Token.Id.Equal;
559 },
560 State.Bang => {
561 result.id = Token.Id.Bang;
562 },
563 State.Minus => {
564 result.id = Token.Id.Minus;
565 },
566 State.Slash => {
567 result.id = Token.Id.Slash;
568 },
569 State.Zero => {
570 result.id = Token.Id.IntegerLiteral;
571 },
572 State.Ampersand => {
573 result.id = Token.Id.Ampersand;
574 },
575 State.Period => {
576 result.id = Token.Id.Period;
577 },
578 State.Period2 => {
579 result.id = Token.Id.Ellipsis2;
580 },
581 }
582 }
516583 if (result.id == Token.Id.Eof) {
517584 if (self.pending_invalid_token) |token| {
518585 self.pending_invalid_token = null;
......@@ -551,7 +618,7 @@ pub const Tokenizer = struct {
551618 } else {
552619 // check utf8-encoded character.
553620 const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
554 if (self.index + length >= self.buffer.len) {
621 if (self.index + length > self.buffer.len) {
555622 return u3(self.buffer.len - self.index);
556623 }
557624 const bytes = self.buffer[self.index..self.index + length];
......@@ -632,15 +699,25 @@ test "tokenizer - illegal unicode codepoints" {
632699 testTokenize("//\xe2\x80\xaa", []Token.Id{});
633700}
634701
702test "tokenizer - string identifier and builtin fns" {
703 testTokenize(
704 \\const @"if" = @import("std");
705 ,
706 []Token.Id{
707 Token.Id.Keyword_const,
708 Token.Id.StringIdentifier,
709 Token.Id.Equal,
710 Token.Id.Builtin,
711 Token.Id.LParen,
712 Token.Id {.StringLiteral = Token.StrLitKind.Normal},
713 Token.Id.RParen,
714 Token.Id.Semicolon,
715 }
716 );
717}
718
635719fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
636 // (test authors, just make this bigger if you need it)
637 var padded_source: [0x100]u8 = undefined;
638 std.mem.copy(u8, padded_source[0..source.len], source);
639 padded_source[source.len + 0] = '\n';
640 padded_source[source.len + 1] = '\n';
641 padded_source[source.len + 2] = '\n';
642
643 var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
720 var tokenizer = Tokenizer.init(source);
644721 for (expected_tokens) |expected_token_id| {
645722 const token = tokenizer.next();
646723 std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
......@@ -651,5 +728,6 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
651728 else => {},
652729 }
653730 }
654 std.debug.assert(tokenizer.next().id == Token.Id.Eof);
731 const last_token = tokenizer.next();
732 std.debug.assert(last_token.id == Token.Id.Eof);
655733}