authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-19 20:41:44+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-19 20:44:55+02:00
logad327fed05d2d809dfef612e3f4abbb5a9a5ed71
tree667f96f412aea8b77ddd294e1801710bf6d1afb7
parent28daddae81f354da89f917528dab2e5d87bca829
signature Commit is signed but in an unrecognized format.

std-c redo scoping, do string concatanation in parser

3 files changed, 69 insertions(+), 63 deletions(-)

lib/std/c/ast.zig+1-1
......@@ -576,7 +576,7 @@ pub const Node = struct {
576576 asterisk: ?TokenIndex,
577577 static: ?TokenIndex,
578578 qual: TypeQual,
579 // expr: *Expr,
579 expr: *Expr,
580580 },
581581 },
582582 rbracket: TokenIndex,
lib/std/c/parse.zig+49-35
......@@ -75,9 +75,12 @@ pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree
7575 }
7676 }
7777
78 var parse_arena = std.heap.ArenaAllocator.init(allocator);
79 defer parse_arena.deinit();
80
7881 var parser = Parser{
79 .symbols = Parser.SymbolList.init(allocator),
80 .arena = arena,
82 .scopes = Parser.SymbolList.init(allocator),
83 .arena = &parse_arena.allocator,
8184 .it = &it,
8285 .tree = tree,
8386 .options = options,
......@@ -93,11 +96,17 @@ const Parser = struct {
9396 it: *TokenIterator,
9497 tree: *Tree,
9598
96 /// only used for scopes
97 symbols: SymbolList,
99 arena: *Allocator,
100 scopes: ScopeList,
98101 options: Options,
99102
100 const SymbolList = std.ArrayList(Symbol);
103 const ScopeList = std.SegmentedLists(Scope);
104 const SymbolList = std.SegmentedLists(Symbol);
105
106 const Scope = struct {
107 kind: ScopeKind,
108 syms: SymbolList,
109 };
101110
102111 const Symbol = struct {
103112 name: []const u8,
......@@ -111,21 +120,27 @@ const Parser = struct {
111120 Switch,
112121 };
113122
114 fn pushScope(parser: *Parser, kind: ScopeKind) usize {
115 return parser.symbols.len;
123 fn pushScope(parser: *Parser, kind: ScopeKind) !void {
124 const new = try parser.scopes.addOne();
125 new.* = .{
126 .kind = kind,
127 .syms = SymbolList.init(parser.arena),
128 };
116129 }
117130
118131 fn popScope(parser: *Parser, len: usize) void {
119 parser.symbols.resize(len) catch unreachable;
132 _ = parser.scopes.pop();
120133 }
121134
122 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
135 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Symbol {
123136 const name = parser.tree.tokenSlice(tok);
124 const syms = parser.symbols.toSliceConst();
125 var i = syms.len;
126 while (i > 0) : (i -= 1) {
127 if (mem.eql(u8, name, syms[i].name)) {
128 return syms[i].ty;
137 var scope_it = parser.scopes.iterator(parser.scopes.len);
138 while (scope_it.prev()) |scope| {
139 var sym_it = scope.syms.iterator(scope.syms.len);
140 while (sym_it.prev()) |sym| {
141 if (mem.eql(u8, sym.name, name)) {
142 return sym;
143 }
129144 }
130145 }
131146 return null;
......@@ -137,8 +152,8 @@ const Parser = struct {
137152
138153 /// Root <- ExternalDeclaration* eof
139154 fn root(parser: *Parser) Allocator.Error!*Node.Root {
140 const scope = parser.pushScope(.Root);
141 defer parser.popScope(scope);
155 try parser.pushScope(.Root);
156 defer parser.popScope();
142157 const node = try parser.arena.create(Node.Root);
143158 node.* = .{
144159 .decls = Node.Root.DeclList.init(parser.arena),
......@@ -782,8 +797,8 @@ const Parser = struct {
782797 .ty = ty,
783798 });
784799 if (parser.eatToken(.LBrace)) |lbrace| {
785 const scope = parser.pushScope(.Block);
786 defer parser.popScope(scope);
800 try parser.pushScope(.Block);
801 defer parser.popScope();
787802 var fields = Node.RecordType.FieldList.init(parser.arena);
788803 while (true) {
789804 if (parser.eatToken(.RBrace)) |rbrace| {
......@@ -996,15 +1011,14 @@ const Parser = struct {
9961011 fn assignmentExpr(parser: *Parser) !*Node {}
9971012
9981013 /// ConstExpr <- ConditionalExpr
999 fn constExpr(parser: *Parser) Error!*Node {
1014 fn constExpr(parser: *Parser) Error!?*Expr {
10001015 const start = parser.it.index;
10011016 const expression = try parser.conditionalExpr();
1002 // TODO
1003 // if (expression == nullor expression.?.value == null)
1004 // return parser.err(.{
1005 // .ConsExpr = start,
1006 // });
1007 return expression.?;
1017 if (expression != null and expression.?.value == .None)
1018 return parser.err(.{
1019 .ConsExpr = start,
1020 });
1021 return expression;
10081022 }
10091023
10101024 /// ConditionalExpr <- LogicalOrExpr (QUESTIONMARK Expr COLON ConditionalExpr)?
......@@ -1085,8 +1099,8 @@ const Parser = struct {
10851099 /// CompoundStmt <- LBRACE (Declaration / Stmt)* RBRACE
10861100 fn compoundStmt(parser: *Parser) Error!?*Node {
10871101 const lbrace = parser.eatToken(.LBrace) orelse return null;
1088 const scope = parser.pushScope(.Block);
1089 defer parser.popScope(scope);
1102 try parser.pushScope(.Block);
1103 defer parser.popScope();
10901104 const body_node = try parser.arena.create(Node.CompoundStmt);
10911105 body_node.* = .{
10921106 .lbrace = lbrace,
......@@ -1142,8 +1156,8 @@ const Parser = struct {
11421156 return &node.base;
11431157 }
11441158 if (parser.eatToken(.Keyword_while)) |tok| {
1145 const scope = parser.pushScope(.Loop);
1146 defer parser.popScope(scope);
1159 try parser.pushScope(.Loop);
1160 defer parser.popScope();
11471161 _ = try parser.expectToken(.LParen);
11481162 const cond = (try parser.expr()) orelse return parser.err(.{
11491163 .ExpectedExpr = .{ .token = parser.it.index },
......@@ -1160,8 +1174,8 @@ const Parser = struct {
11601174 return &node.base;
11611175 }
11621176 if (parser.eatToken(.Keyword_do)) |tok| {
1163 const scope = parser.pushScope(.Loop);
1164 defer parser.popScope(scope);
1177 try parser.pushScope(.Loop);
1178 defer parser.popScope();
11651179 const body = try parser.stmt();
11661180 _ = try parser.expectToken(.LParen);
11671181 const cond = (try parser.expr()) orelse return parser.err(.{
......@@ -1179,8 +1193,8 @@ const Parser = struct {
11791193 return &node.base;
11801194 }
11811195 if (parser.eatToken(.Keyword_for)) |tok| {
1182 const scope = parser.pushScope(.Loop);
1183 defer parser.popScope(scope);
1196 try parser.pushScope(.Loop);
1197 defer parser.popScope();
11841198 _ = try parser.expectToken(.LParen);
11851199 const init = if (try parser.declaration()) |decl| blk:{
11861200 // TODO disallow storage class other than auto and register
......@@ -1203,8 +1217,8 @@ const Parser = struct {
12031217 return &node.base;
12041218 }
12051219 if (parser.eatToken(.Keyword_switch)) |tok| {
1206 const scope = parser.pushScope(.Switch);
1207 defer parser.popScope(scope);
1220 try parser.pushScope(.Switch);
1221 defer parser.popScope();
12081222 _ = try parser.expectToken(.LParen);
12091223 const switch_expr = try parser.exprStmt();
12101224 const rparen = try parser.expectToken(.RParen);
lib/std/c/tokenizer.zig+19-27
......@@ -401,7 +401,6 @@ pub const Tokenizer = struct {
401401 U,
402402 L,
403403 StringLiteral,
404 AfterStringLiteral,
405404 CharLiteralStart,
406405 CharLiteral,
407406 EscapeSequence,
......@@ -617,7 +616,7 @@ pub const Tokenizer = struct {
617616 },
618617 .BackSlash => switch (c) {
619618 '\n' => {
620 state = if (string) .AfterStringLiteral else .Start;
619 state = .Start;
621620 },
622621 '\r' => {
623622 state = .BackSlashCr;
......@@ -632,7 +631,7 @@ pub const Tokenizer = struct {
632631 },
633632 .BackSlashCr => switch (c) {
634633 '\n' => {
635 state = if (string) .AfterStringLiteral else .Start;
634 state = .Start;
636635 },
637636 else => {
638637 result.id = .Invalid;
......@@ -696,7 +695,8 @@ pub const Tokenizer = struct {
696695 state = .EscapeSequence;
697696 },
698697 '"' => {
699 state = .AfterStringLiteral;
698 self.index += 1;
699 break;
700700 },
701701 '\n', '\r' => {
702702 result.id = .Invalid;
......@@ -704,22 +704,6 @@ pub const Tokenizer = struct {
704704 },
705705 else => {},
706706 },
707 .AfterStringLiteral => switch (c) {
708 '"' => {
709 state = .StringLiteral;
710 },
711 '\\' => {
712 state = .BackSlash;
713 },
714 '\n', '\r' => {
715 if (self.pp_directive)
716 break;
717 },
718 '\t', '\x0B', '\x0C', ' ' => {},
719 else => {
720 break;
721 },
722 },
723707 .CharLiteralStart => switch (c) {
724708 '\\' => {
725709 string = false;
......@@ -1255,7 +1239,7 @@ pub const Tokenizer = struct {
12551239 }
12561240 } else if (self.index == self.source.buffer.len) {
12571241 switch (state) {
1258 .AfterStringLiteral, .Start => {},
1242 .Start => {},
12591243 .u, .u8, .U, .L, .Identifier => {
12601244 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
12611245 },
......@@ -1322,7 +1306,7 @@ pub const Tokenizer = struct {
13221306
13231307test "operators" {
13241308 expectTokens(
1325 \\ ! != | || |= = ==
1309 \\ ! != | || |= = ==
13261310 \\ ( ) { } [ ] . .. ...
13271311 \\ ^ ^= + ++ += - -- -=
13281312 \\ * *= % %= -> : ; / /=
......@@ -1505,24 +1489,27 @@ test "line continuation" {
15051489 .Identifier,
15061490 .Nl,
15071491 .{ .StringLiteral = .None },
1492 .Nl,
15081493 .Hash,
15091494 .Keyword_define,
15101495 .{ .StringLiteral = .None },
15111496 .Nl,
15121497 .{ .StringLiteral = .None },
1498 .Nl,
15131499 .Hash,
15141500 .Keyword_define,
15151501 .{ .StringLiteral = .None },
1502 .{ .StringLiteral = .None },
15161503 });
15171504}
15181505
15191506test "string prefix" {
15201507 expectTokens(
1521 \\"foo" "bar"
1522 \\u"foo" "bar"
1523 \\u8"foo" "bar"
1524 \\U"foo" "bar"
1525 \\L"foo" "bar"
1508 \\"foo"
1509 \\u"foo"
1510 \\u8"foo"
1511 \\U"foo"
1512 \\L"foo"
15261513 \\'foo'
15271514 \\u'foo'
15281515 \\U'foo'
......@@ -1530,10 +1517,15 @@ test "string prefix" {
15301517 \\
15311518 , &[_]Token.Id{
15321519 .{ .StringLiteral = .None },
1520 .Nl,
15331521 .{ .StringLiteral = .Utf16 },
1522 .Nl,
15341523 .{ .StringLiteral = .Utf8 },
1524 .Nl,
15351525 .{ .StringLiteral = .Utf32 },
1526 .Nl,
15361527 .{ .StringLiteral = .Wide },
1528 .Nl,
15371529 .{ .CharLiteral = .None },
15381530 .Nl,
15391531 .{ .CharLiteral = .Utf16 },