authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-08 23:15:43-05:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2017-12-08 23:15:43-05:00
logd431b0fb99342f1e2629f5dd8bf1ca578a5e5d9a
tree0924b414f2c0a0b392ea970b33d40219f9ed56f3
parent5ead3244a21599dae77d986d443185a43197b2d9

parse a simple variable declaration

2 files changed, 403 insertions(+), 42 deletions(-)

src-self-hosted/main.zig+392-42
...@@ -88,6 +88,8 @@ const Token = struct {...@@ -88,6 +88,8 @@ const Token = struct {
88 Period,88 Period,
89 Minus,89 Minus,
90 Arrow,90 Arrow,
91 Colon,
92 Slash,
91 Keyword_align,93 Keyword_align,
92 Keyword_and,94 Keyword_and,
93 Keyword_asm,95 Keyword_asm,
...@@ -135,6 +137,37 @@ const Tokenizer = struct {...@@ -135,6 +137,37 @@ const Tokenizer = struct {
135 buffer: []const u8,137 buffer: []const u8,
136 index: usize,138 index: usize,
137139
140 pub const Location = struct {
141 line: usize,
142 column: usize,
143 line_start: usize,
144 line_end: usize,
145 };
146
147 pub fn getTokenLocation(self: &Tokenizer, token: &const Token) -> Location {
148 var loc = Location {
149 .line = 0,
150 .column = 0,
151 .line_start = 0,
152 .line_end = 0,
153 };
154 for (self.buffer) |c, i| {
155 if (i == token.start) {
156 loc.line_end = i;
157 while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
158 return loc;
159 }
160 if (c == '\n') {
161 loc.line += 1;
162 loc.column = 0;
163 loc.line_start = i;
164 } else {
165 loc.column += 1;
166 }
167 }
168 return loc;
169 }
170
138 pub fn dump(self: &Tokenizer, token: &const Token) {171 pub fn dump(self: &Tokenizer, token: &const Token) {
139 warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);172 warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
140 }173 }
...@@ -154,6 +187,8 @@ const Tokenizer = struct {...@@ -154,6 +187,8 @@ const Tokenizer = struct {
154 StringLiteral,187 StringLiteral,
155 StringLiteralBackslash,188 StringLiteralBackslash,
156 Minus,189 Minus,
190 Slash,
191 LineComment,
157 };192 };
158193
159 pub fn next(self: &Tokenizer) -> Token {194 pub fn next(self: &Tokenizer) -> Token {
...@@ -206,6 +241,11 @@ const Tokenizer = struct {...@@ -206,6 +241,11 @@ const Tokenizer = struct {
206 self.index += 1;241 self.index += 1;
207 break;242 break;
208 },243 },
244 ':' => {
245 result.id = Token.Id.Colon;
246 self.index += 1;
247 break;
248 },
209 '%' => {249 '%' => {
210 result.id = Token.Id.Percent;250 result.id = Token.Id.Percent;
211 self.index += 1;251 self.index += 1;
...@@ -229,6 +269,9 @@ const Tokenizer = struct {...@@ -229,6 +269,9 @@ const Tokenizer = struct {
229 '-' => {269 '-' => {
230 state = State.Minus;270 state = State.Minus;
231 },271 },
272 '/' => {
273 state = State.Slash;
274 },
232 else => {275 else => {
233 result.id = Token.Id.Invalid;276 result.id = Token.Id.Invalid;
234 self.index += 1;277 self.index += 1;
...@@ -289,78 +332,363 @@ const Tokenizer = struct {...@@ -289,78 +332,363 @@ const Tokenizer = struct {
289 break;332 break;
290 },333 },
291 },334 },
335 State.Slash => switch (c) {
336 '/' => {
337 result.id = undefined;
338 state = State.LineComment;
339 },
340 else => {
341 result.id = Token.Id.Slash;
342 break;
343 },
344 },
345 State.LineComment => switch (c) {
346 '\n' => {
347 state = State.Start;
348 result = Token {
349 .id = Token.Id.Eof,
350 .start = self.index + 1,
351 .end = undefined,
352 };
353 },
354 else => {},
355 },
292 }356 }
293 }357 }
294 result.end = self.index;358 result.end = self.index;
359 // TODO check state when returning EOF
295 return result;360 return result;
296 }361 }
297};362};
298363
364const Visibility = enum {
365 Private,
366 Pub,
367 Export,
368};
369
370const Mutability = enum {
371 Const,
372 Var,
373};
374
299const AstNode = struct {375const AstNode = struct {
376 id: Id,
377
378 const Id = enum {
379 Root,
380 VarDecl,
381 Identifier,
382 };
300383
384 fn iterate(base: &AstNode, index: usize) -> ?&AstNode {
385 return switch (base.id) {
386 Id.Root => @fieldParentPtr(AstNodeRoot, "base", base).iterate(index),
387 Id.VarDecl => @fieldParentPtr(AstNodeVarDecl, "base", base).iterate(index),
388 Id.Identifier => @fieldParentPtr(AstNodeIdentifier, "base", base).iterate(index),
389 };
390 }
391};
392
393const AstNodeRoot = struct {
394 base: AstNode,
395 decls: ArrayList(&AstNode),
396
397 fn iterate(self: &AstNodeRoot, index: usize) -> ?&AstNode {
398 if (index < self.decls.len) {
399 return self.decls.items[index];
400 }
401 return null;
402 }
301};403};
302404
405const AstNodeVarDecl = struct {
406 base: AstNode,
407 visib: Visibility,
408 name_token: Token,
409 eq_token: Token,
410 mut: Mutability,
411 is_comptime: bool,
412 type_node: ?&AstNode,
413 align_node: ?&AstNode,
414 init_node: ?&AstNode,
415
416 fn iterate(self: &AstNodeVarDecl, index: usize) -> ?&AstNode {
417 var i = index;
418
419 if (self.type_node) |type_node| {
420 if (i < 1) return type_node;
421 i -= 1;
422 }
423
424 if (self.align_node) |align_node| {
425 if (i < 1) return align_node;
426 i -= 1;
427 }
428
429 if (self.init_node) |init_node| {
430 if (i < 1) return init_node;
431 i -= 1;
432 }
433
434 return null;
435 }
436};
437
438const AstNodeIdentifier = struct {
439 base: AstNode,
440 name_token: Token,
441
442 fn iterate(self: &AstNodeIdentifier, index: usize) -> ?&AstNode {
443 return null;
444 }
445};
446
447error ParseError;
448
303const Parser = struct {449const Parser = struct {
304 tokenizer: &Tokenizer,450 tokenizer: &Tokenizer,
305 allocator: &mem.Allocator,451 allocator: &mem.Allocator,
452 put_back_tokens: [1]Token,
453 put_back_count: usize,
454 source_file_name: []const u8,
306455
307 fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator) -> Parser {456 fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) -> Parser {
308 return Parser {457 return Parser {
309 .tokenizer = tokenizer,458 .tokenizer = tokenizer,
310 .allocator = allocator,459 .allocator = allocator,
460 .put_back_tokens = undefined,
461 .put_back_count = 0,
462 .source_file_name = source_file_name,
311 };463 };
312 }464 }
313465
314 const StackFrame = struct {466 const State = union(enum) {
315
316 };
317
318 const State = enum {
319 TopLevel, 467 TopLevel,
320 Expression,468 TopLevelModifier: Visibility,
469 Expression: &?&AstNode,
470 GroupedExpression: &?&AstNode,
471 PrimaryExpression: &?&AstNode,
472 TypeExpr: &?&AstNode,
473 VarDecl: &AstNodeVarDecl,
474 VarDeclAlign: &AstNodeVarDecl,
475 VarDeclEq: &AstNodeVarDecl,
476 ExpectSemicolon,
321 };477 };
322478
323 fn parse(self: &Parser) -> %void {479 pub fn parse(self: &Parser) -> %&AstNode {
324 var stack = ArrayList(StackFrame).init(self.allocator);480 var stack = ArrayList(State).init(self.allocator);
325 defer stack.deinit();481 defer stack.deinit();
326482
327 var state = State.TopLevel;483 %return stack.append(State.TopLevel);
484
485 const root_node = %return self.createRoot();
486 // TODO %defer self.freeAst();
487
328 while (true) {488 while (true) {
329 const token = self.tokenizer.next();489 // This gives us 1 free append that can't fail
490 const state = stack.pop();
491
330 switch (state) {492 switch (state) {
331 State.TopLevel => switch (token.id) {493 State.TopLevel => {
332 Token.Id.Keyword_pub => {494 const token = self.getNextToken();
333 const next_token = self.tokenizer.next();495 switch (token.id) {
334 switch (next_token.id) {496 Token.Id.Keyword_pub => {
335 Token.Id.Keyword_fn => {497 stack.append(State {.TopLevelModifier = Visibility.Pub }) %% unreachable;
336 const fn_name = self.tokenizer.next();498 continue;
337 if (fn_name.id != Token.Id.Identifier) {499 },
338 @panic("parse error");500 Token.Id.Keyword_export => {
339 }501 stack.append(State {.TopLevelModifier = Visibility.Export }) %% unreachable;
340502 continue;
341 const lparen = self.tokenizer.next();503 },
342 if (lparen.id != Token.Id.LParen) {504 Token.Id.Keyword_const => {
343 @panic("parse error");505 stack.append(State.TopLevel) %% unreachable;
344 }506 const var_decl_node = %return self.createVarDecl(Visibility.Private, Mutability.Const, false);
345 },507 %return root_node.decls.append(&var_decl_node.base);
346 Token.Id.Keyword_const => @panic("TODO"),508 %return stack.append(State { .VarDecl = var_decl_node });
347 Token.Id.Keyword_var => @panic("TODO"),509 continue;
348 Token.Id.Keyword_use => @panic("TODO"),510 },
349 else => @panic("parse error"),511 Token.Id.Keyword_var => {
350 }512 stack.append(State.TopLevel) %% unreachable;
351 },513 const var_decl_node = %return self.createVarDecl(Visibility.Private, Mutability.Var, false);
352 Token.Id.Keyword_const => @panic("TODO"),514 %return root_node.decls.append(&var_decl_node.base);
353 Token.Id.Keyword_var => @panic("TODO"),515 %return stack.append(State { .VarDecl = var_decl_node });
354 Token.Id.Keyword_fn => @panic("TODO"),516 continue;
355 Token.Id.Keyword_export => @panic("TODO"),517 },
356 Token.Id.Keyword_use => @panic("TODO"),518 Token.Id.Eof => return &root_node.base,
357 Token.Id.Keyword_comptime => @panic("TODO"),519 else => return self.parseError(token, "expected top level declaration, found {}", @tagName(token.id)),
358 else => @panic("parse error"),520 }
521 },
522 State.TopLevelModifier => |visib| {
523 const token = self.getNextToken();
524 switch (token.id) {
525 Token.Id.Keyword_const => {
526 stack.append(State.TopLevel) %% unreachable;
527 const var_decl_node = %return self.createVarDecl(visib, Mutability.Const, false);
528 %return root_node.decls.append(&var_decl_node.base);
529 %return stack.append(State { .VarDecl = var_decl_node });
530 continue;
531 },
532 Token.Id.Keyword_var => {
533 stack.append(State.TopLevel) %% unreachable;
534 const var_decl_node = %return self.createVarDecl(visib, Mutability.Var, false);
535 %return root_node.decls.append(&var_decl_node.base);
536 %return stack.append(State { .VarDecl = var_decl_node });
537 continue;
538 },
539 else => return self.parseError(token, "expected top level declaration, found {}", @tagName(token.id)),
540 }
541 },
542 State.VarDecl => |var_decl| {
543 var_decl.name_token = %return self.eatToken(Token.Id.Identifier);
544 stack.append(State { .VarDeclAlign = var_decl }) %% unreachable;
545
546 const next_token = self.getNextToken();
547 if (next_token.id == Token.Id.Colon) {
548 %return stack.append(State { .TypeExpr = &var_decl.type_node });
549 continue;
550 }
551
552 self.putBackToken(next_token);
553 continue;
554 },
555 State.VarDeclAlign => |var_decl| {
556 stack.append(State { .VarDeclEq = var_decl }) %% unreachable;
557
558 const next_token = self.getNextToken();
559 if (next_token.id == Token.Id.Keyword_align) {
560 %return stack.append(State { .GroupedExpression = &var_decl.align_node });
561 continue;
562 }
563
564 self.putBackToken(next_token);
565 continue;
566 },
567 State.VarDeclEq => |var_decl| {
568 var_decl.eq_token = %return self.eatToken(Token.Id.Equal);
569 stack.append(State.ExpectSemicolon) %% unreachable;
570 %return stack.append(State {
571 .Expression = &var_decl.init_node,
572 });
573 continue;
574 },
575 State.ExpectSemicolon => {
576 _ = %return self.eatToken(Token.Id.Semicolon);
577 continue;
359 },578 },
360 State.Expression => @panic("TODO"),579 State.Expression => |result_ptr| {
580 // TODO this should not jump straight to primary expression
581 stack.append(State {.PrimaryExpression = result_ptr}) %% unreachable;
582 continue;
583 },
584 State.PrimaryExpression => |result_ptr| {
585 const token = self.getNextToken();
586 switch (token.id) {
587 Token.Id.Identifier => {
588 const identifier = %return self.createIdentifier(token);
589 *result_ptr = &identifier.base;
590 continue;
591 },
592 else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
593 }
594 },
595 State.TypeExpr => @panic("TODO"),
596 State.GroupedExpression => @panic("TODO"),
597 }
598 unreachable;
599 }
600 }
601
602 fn createRoot(self: &Parser) -> %&AstNodeRoot {
603 const node = %return self.allocator.create(AstNodeRoot);
604 %defer self.allocator.destroy(node);
605
606 *node = AstNodeRoot {
607 .base = AstNode {.id = AstNode.Id.Root},
608 .decls = ArrayList(&AstNode).init(self.allocator),
609 };
610 return node;
611 }
612
613 fn createVarDecl(self: &Parser, visib: Visibility, mut: Mutability, is_comptime: bool) -> %&AstNodeVarDecl {
614 const node = %return self.allocator.create(AstNodeVarDecl);
615 %defer self.allocator.destroy(node);
616
617 *node = AstNodeVarDecl {
618 .base = AstNode {.id = AstNode.Id.VarDecl},
619 .visib = visib,
620 .mut = mut,
621 .is_comptime = is_comptime,
622 .type_node = null,
623 .align_node = null,
624 .init_node = null,
625 // initialized later
626 .name_token = undefined,
627 .eq_token = undefined,
628 };
629 return node;
630 }
631
632 fn createIdentifier(self: &Parser, name_token: &const Token) -> %&AstNodeIdentifier {
633 const node = %return self.allocator.create(AstNodeIdentifier);
634 %defer self.allocator.destroy(node);
635
636 *node = AstNodeIdentifier {
637 .base = AstNode {.id = AstNode.Id.Identifier},
638 .name_token = *name_token,
639 };
640 return node;
641 }
642
643 fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) -> error {
644 const loc = self.tokenizer.getTokenLocation(token);
645 warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
646 warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
647 {
648 var i: usize = 0;
649 while (i < loc.column) : (i += 1) {
650 warn(" ");
361 }651 }
362 }652 }
653 {
654 const caret_count = token.end - token.start;
655 var i: usize = 0;
656 while (i < caret_count) : (i += 1) {
657 warn("~");
658 }
659 }
660 warn("\n");
661 return error.ParseError;
662 }
663
664 fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) -> %void {
665 if (token.id != id) {
666 return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
667 }
668 }
669
670 fn eatToken(self: &Parser, id: @TagType(Token.Id)) -> %Token {
671 const token = self.getNextToken();
672 %return self.expectToken(token, id);
673 return token;
363 }674 }
675
676 fn putBackToken(self: &Parser, token: &const Token) {
677 self.put_back_tokens[self.put_back_count] = *token;
678 self.put_back_count += 1;
679 }
680
681 fn getNextToken(self: &Parser) -> Token {
682 return if (self.put_back_count != 0) {
683 const put_back_index = self.put_back_count - 1;
684 const put_back_token = self.put_back_tokens[put_back_index];
685 self.put_back_count = put_back_index;
686 put_back_token
687 } else {
688 self.tokenizer.next()
689 };
690 }
691
364};692};
365693
366694
...@@ -384,8 +712,11 @@ pub fn main2() -> %void {...@@ -384,8 +712,11 @@ pub fn main2() -> %void {
384712
385 const target_file_buf = %return io.readFileAlloc(target_file, allocator);713 const target_file_buf = %return io.readFileAlloc(target_file, allocator);
386714
715 warn("====input:====\n");
716
387 warn("{}", target_file_buf);717 warn("{}", target_file_buf);
388718
719 warn("====tokenization:====\n");
389 {720 {
390 var tokenizer = Tokenizer.init(target_file_buf);721 var tokenizer = Tokenizer.init(target_file_buf);
391 while (true) {722 while (true) {
...@@ -397,7 +728,26 @@ pub fn main2() -> %void {...@@ -397,7 +728,26 @@ pub fn main2() -> %void {
397 }728 }
398 }729 }
399730
731 warn("====parse:====\n");
732
400 var tokenizer = Tokenizer.init(target_file_buf);733 var tokenizer = Tokenizer.init(target_file_buf);
401 var parser = Parser.init(&tokenizer, allocator);734 var parser = Parser.init(&tokenizer, allocator, target_file);
402 %return parser.parse();735 const node = %return parser.parse();
736
737
738 render(node, 0);
739}
740
741fn render(node: &AstNode, indent: usize) {
742 {
743 var i: usize = 0;
744 while (i < indent) : (i += 1) {
745 warn(" ");
746 }
747 }
748 warn("{}\n", @tagName(node.id));
749 var i: usize = 0;
750 while (node.iterate(i)) |child| : (i += 1) {
751 render(child, indent + 2);
752 }
403}753}
std/mem.zig+11
...@@ -8,6 +8,7 @@ pub const Cmp = math.Cmp;...@@ -8,6 +8,7 @@ pub const Cmp = math.Cmp;
8pub const Allocator = struct {8pub const Allocator = struct {
9 /// Allocate byte_count bytes and return them in a slice, with the9 /// Allocate byte_count bytes and return them in a slice, with the
10 /// slice's pointer aligned at least to alignment bytes.10 /// slice's pointer aligned at least to alignment bytes.
11 /// The returned newly allocated memory is undefined.
11 allocFn: fn (self: &Allocator, byte_count: usize, alignment: u29) -> %[]u8,12 allocFn: fn (self: &Allocator, byte_count: usize, alignment: u29) -> %[]u8,
1213
13 /// If `new_byte_count > old_mem.len`:14 /// If `new_byte_count > old_mem.len`:
...@@ -17,6 +18,8 @@ pub const Allocator = struct {...@@ -17,6 +18,8 @@ pub const Allocator = struct {
17 /// If `new_byte_count <= old_mem.len`:18 /// If `new_byte_count <= old_mem.len`:
18 /// * this function must return successfully. 19 /// * this function must return successfully.
19 /// * alignment <= alignment of old_mem.ptr20 /// * alignment <= alignment of old_mem.ptr
21 ///
22 /// The returned newly allocated memory is undefined.
20 reallocFn: fn (self: &Allocator, old_mem: []u8, new_byte_count: usize, alignment: u29) -> %[]u8,23 reallocFn: fn (self: &Allocator, old_mem: []u8, new_byte_count: usize, alignment: u29) -> %[]u8,
2124
22 /// Guaranteed: `old_mem.len` is the same as what was returned from `allocFn` or `reallocFn`25 /// Guaranteed: `old_mem.len` is the same as what was returned from `allocFn` or `reallocFn`
...@@ -40,6 +43,10 @@ pub const Allocator = struct {...@@ -40,6 +43,10 @@ pub const Allocator = struct {
40 {43 {
41 const byte_count = %return math.mul(usize, @sizeOf(T), n);44 const byte_count = %return math.mul(usize, @sizeOf(T), n);
42 const byte_slice = %return self.allocFn(self, byte_count, alignment);45 const byte_slice = %return self.allocFn(self, byte_count, alignment);
46 // This loop should get optimized out in ReleaseFast mode
47 for (byte_slice) |*byte| {
48 *byte = undefined;
49 }
43 return ([]align(alignment) T)(@alignCast(alignment, byte_slice));50 return ([]align(alignment) T)(@alignCast(alignment, byte_slice));
44 }51 }
4552
...@@ -56,6 +63,10 @@ pub const Allocator = struct {...@@ -56,6 +63,10 @@ pub const Allocator = struct {
5663
57 const byte_count = %return math.mul(usize, @sizeOf(T), n);64 const byte_count = %return math.mul(usize, @sizeOf(T), n);
58 const byte_slice = %return self.reallocFn(self, ([]u8)(old_mem), byte_count, alignment);65 const byte_slice = %return self.reallocFn(self, ([]u8)(old_mem), byte_count, alignment);
66 // This loop should get optimized out in ReleaseFast mode
67 for (byte_slice[old_mem.len..]) |*byte| {
68 *byte = undefined;
69 }
59 return ([]T)(@alignCast(alignment, byte_slice));70 return ([]T)(@alignCast(alignment, byte_slice));
60 }71 }
6172