authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:40:33-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-04 23:40:33-05:00
log3976981ab3370d099535d79d42bd82d5924df7f5
treecfe7ed6a20e66488b8654d859cfbc7510b8b06df
parent7297baa9c6f3f1b894b47fe7b06b5ae2007d0f3c

tokenizing hello world

1 files changed, 46 insertions(+), 1 deletions(-)

src-self-hosted/main.zig+46-1
......@@ -77,6 +77,12 @@ const Token = struct {
7777 LParen,
7878 RParen,
7979 Semicolon,
80 Percent,
81 LBrace,
82 RBrace,
83 Period,
84 Minus,
85 Arrow,
8086 };
8187};
8288
......@@ -102,6 +108,7 @@ const Tokenizer = struct {
102108 C,
103109 StringLiteral,
104110 StringLiteralBackslash,
111 Minus,
105112 };
106113
107114 pub fn next(self: &Tokenizer) -> Token {
......@@ -154,6 +161,29 @@ const Tokenizer = struct {
154161 self.index += 1;
155162 break;
156163 },
164 '%' => {
165 result.id = Token.Id { .Percent = {} };
166 self.index += 1;
167 break;
168 },
169 '{' => {
170 result.id = Token.Id { .LBrace = {} };
171 self.index += 1;
172 break;
173 },
174 '}' => {
175 result.id = Token.Id { .RBrace = {} };
176 self.index += 1;
177 break;
178 },
179 '.' => {
180 result.id = Token.Id { .Period = {} };
181 self.index += 1;
182 break;
183 },
184 '-' => {
185 state = State.Minus;
186 },
157187 else => {
158188 result.id = Token.Id { .Invalid = {} };
159189 self.index += 1;
......@@ -195,9 +225,24 @@ const Tokenizer = struct {
195225 '\n' => break, // Look for this error later.
196226 else => {},
197227 },
228
198229 State.StringLiteralBackslash => switch (c) {
199230 '\n' => break, // Look for this error later.
200 else => {},
231 else => {
232 state = State.StringLiteral;
233 },
234 },
235
236 State.Minus => switch (c) {
237 '>' => {
238 result.id = Token.Id { .Arrow = {} };
239 self.index += 1;
240 break;
241 },
242 else => {
243 result.id = Token.Id { .Minus = {} };
244 break;
245 },
201246 },
202247 }
203248 }