authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-03 22:34:47+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:49+02:00
log04b7cec42e85a74e5933f13a6d3d10c8e0207066
tree4f29ae8c3484bdc877d1b353eaa48eacc9bd48e9
parent2e5342512f32384777ebcb0bf3dcb177b9380080
signature Commit is signed but in an unrecognized format.

std-c tokenizer base

1 files changed, 132 insertions(+), 0 deletions(-)

lib/std/c/tokenizer.zig created+132
...@@ -0,0 +1,132 @@
1const std = @import("std");
2const expect = std.testing.expect;
3
4pub const Source = struct {
5 buffer: []const u8,
6 file_name: []const u8,
7};
8
9pub const Token = struct {
10 id: Id,
11 num_suffix: NumSuffix = .None,
12 start: usize,
13 end: usize,
14 source: *Source,
15
16 pub const Id = enum {
17 Invalid,
18 Eof,
19 Nl,
20 Identifier,
21 StringLiteral,
22 CharLiteral,
23 IntegerLiteral,
24 FloatLiteral,
25 Bang,
26 BangEqual,
27 Pipe,
28 PipePipe,
29 PipeEqual,
30 Equal,
31 EqualEqual,
32 EqualAngleBracketRight,
33 LParen,
34 RParen,
35 LBrace,
36 RBrace,
37 LBracket,
38 RBracket,
39 Period,
40 PeriodAsterisk,
41 Ellipsis,
42 Caret,
43 CaretEqual,
44 Plus,
45 PlusPlus,
46 PlusEqual,
47 Minus,
48 MinusMinus,
49 MinusEqual,
50 Asterisk,
51 AsteriskEqual,
52 Percent,
53 PercentEqual,
54 Arrow,
55 Colon,
56 Semicolon,
57 Slash,
58 SlashEqual,
59 Comma,
60 Ampersand,
61 AmpersandAmpersand,
62 AmpersandEqual,
63 QuestionMark,
64 AngleBracketLeft,
65 AngleBracketLeftEqual,
66 AngleBracketAngleBracketLeft,
67 AngleBracketAngleBracketLeftEqual,
68 AngleBracketRight,
69 AngleBracketRightEqual,
70 AngleBracketAngleBracketRight,
71 AngleBracketAngleBracketRightEqual,
72 Tilde,
73 LineComment,
74 MultiLineComment,
75 Hash,
76 HashHash,
77 };
78
79 pub const NumSuffix = enum {
80 None,
81 F,
82 L,
83 U,
84 LU,
85 LL,
86 LLU,
87 };
88};
89
90pub const Tokenizer = struct {
91 source: *Source,
92 index: usize = 0,
93
94 pub fn next(self: *Tokenizer) Token {
95 const start_index = self.index;
96 var result = Token{
97 .id = .Eof,
98 .start = self.index,
99 .end = undefined,
100 .source = self.source,
101 };
102 var state: enum {
103 Start,
104 } = .Start;
105 while (self.index < self.source.buffer.len) : (self.index += 1) {
106 const c = self.source.buffer[self.index];
107 switch (state) {
108 .Start => switch (c) {
109 else => @panic("TODO"),
110 },
111 else => @panic("TODO"),
112 }
113 }
114 }
115};
116
117fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
118 var tokenizer = Tokenizer{
119 .source = .{
120 .buffer = source,
121 .file_name = undefined,
122 },
123 };
124 for (expected_tokens) |expected_token_id| {
125 const token = tokenizer.next();
126 if (token.id != expected_token_id) {
127 std.debug.panic("expected {}, found {}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
128 }
129 }
130 const last_token = tokenizer.next();
131 std.testing.expect(last_token.id == .Eof);
132}