authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:40:33-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:40:33-05:00
log3976981ab3370d099535d79d42bd82d5924df7f5
treecfe7ed6a20e66488b8654d859cfbc7510b8b06df
parent7297baa9c6f3f1b894b47fe7b06b5ae2007d0f3c

tokenizing hello world

1 files changed, 46 insertions(+), 1 deletions(-)

src-self-hosted/main.zig+46-1
...@@ -77,6 +77,12 @@ const Token = struct {...@@ -77,6 +77,12 @@ const Token = struct {
77 LParen,77 LParen,
78 RParen,78 RParen,
79 Semicolon,79 Semicolon,
80 Percent,
81 LBrace,
82 RBrace,
83 Period,
84 Minus,
85 Arrow,
80 };86 };
81};87};
8288
...@@ -102,6 +108,7 @@ const Tokenizer = struct {...@@ -102,6 +108,7 @@ const Tokenizer = struct {
102 C,108 C,
103 StringLiteral,109 StringLiteral,
104 StringLiteralBackslash,110 StringLiteralBackslash,
111 Minus,
105 };112 };
106113
107 pub fn next(self: &Tokenizer) -> Token {114 pub fn next(self: &Tokenizer) -> Token {
...@@ -154,6 +161,29 @@ const Tokenizer = struct {...@@ -154,6 +161,29 @@ const Tokenizer = struct {
154 self.index += 1;161 self.index += 1;
155 break;162 break;
156 },163 },
164 '%' => {
165 result.id = Token.Id { .Percent = {} };
166 self.index += 1;
167 break;
168 },
169 '{' => {
170 result.id = Token.Id { .LBrace = {} };
171 self.index += 1;
172 break;
173 },
174 '}' => {
175 result.id = Token.Id { .RBrace = {} };
176 self.index += 1;
177 break;
178 },
179 '.' => {
180 result.id = Token.Id { .Period = {} };
181 self.index += 1;
182 break;
183 },
184 '-' => {
185 state = State.Minus;
186 },
157 else => {187 else => {
158 result.id = Token.Id { .Invalid = {} };188 result.id = Token.Id { .Invalid = {} };
159 self.index += 1;189 self.index += 1;
...@@ -195,9 +225,24 @@ const Tokenizer = struct {...@@ -195,9 +225,24 @@ const Tokenizer = struct {
195 '\n' => break, // Look for this error later.225 '\n' => break, // Look for this error later.
196 else => {},226 else => {},
197 },227 },
228
198 State.StringLiteralBackslash => switch (c) {229 State.StringLiteralBackslash => switch (c) {
199 '\n' => break, // Look for this error later.230 '\n' => break, // Look for this error later.
200 else => {},231 else => {
232 state = State.StringLiteral;
233 },
234 },
235
236 State.Minus => switch (c) {
237 '>' => {
238 result.id = Token.Id { .Arrow = {} };
239 self.index += 1;
240 break;
241 },
242 else => {
243 result.id = Token.Id { .Minus = {} };
244 break;
245 },
201 },246 },
202 }247 }
203 }248 }