Commit ad327fed05d2d809dfef612e3f4abbb5a9a5ed71 | zig

author	Veikka Tuominen <git@vexu.eu> 2020-01-19 20:41:44+02:00
committer	Veikka Tuominen <git@vexu.eu> 2020-01-19 20:44:55+02:00
log	ad327fed05d2d809dfef612e3f4abbb5a9a5ed71
tree	667f96f412aea8b77ddd294e1801710bf6d1afb7
parent	28daddae81f354da89f917528dab2e5d87bca829
signature	Commit is signed but in an unrecognized format.

std-c redo scoping, do string concatanation in parser

3 files changed, 69 insertions(+), 63 deletions(-)

lib
- std
  - c
    ast.zig
    parse.zig
    tokenizer.zig

lib/std/c/ast.zig+1-1

...	@@ -576,7 +576,7 @@ pub const Node = struct {	...	@@ -576,7 +576,7 @@ pub const Node = struct {
576	asterisk: ?TokenIndex,	576	asterisk: ?TokenIndex,
577	static: ?TokenIndex,	577	static: ?TokenIndex,
578	qual: TypeQual,	578	qual: TypeQual,
579	// expr: *Expr,	579	expr: *Expr,
580	},	580	},
581	},	581	},
582	rbracket: TokenIndex,	582	rbracket: TokenIndex,

lib/std/c/parse.zig+49-35

 @@ -75,9 +75,12 @@ pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree
         }
     }
+    var parse_arena = std.heap.ArenaAllocator.init(allocator);
+    defer parse_arena.deinit();
     var parser = Parser{
-        .symbols = Parser.SymbolList.init(allocator),
+        .scopes = Parser.SymbolList.init(allocator),
-        .arena = arena,
+        .arena = &parse_arena.allocator,
         .it = &it,
         .tree = tree,
         .options = options,
 @@ -93,11 +96,17 @@ const Parser = struct {
     it: *TokenIterator,
     tree: *Tree,
-    /// only used for scopes
+    arena: *Allocator,
-    symbols: SymbolList,
+    scopes: ScopeList,
     options: Options,
-    const SymbolList = std.ArrayList(Symbol);
+    const ScopeList = std.SegmentedLists(Scope);
+    const SymbolList = std.SegmentedLists(Symbol);
+    const Scope = struct {
+        kind: ScopeKind,
+        syms: SymbolList,
+    };
     const Symbol = struct {
         name: []const u8,
 @@ -111,21 +120,27 @@ const Parser = struct {
         Switch,
     };
-    fn pushScope(parser: *Parser, kind: ScopeKind) usize {
+    fn pushScope(parser: *Parser, kind: ScopeKind) !void {
-        return parser.symbols.len;
+        const new = try parser.scopes.addOne();
+        new.* = .{
+            .kind = kind,
+            .syms = SymbolList.init(parser.arena),
+        };
     }
     fn popScope(parser: *Parser, len: usize) void {
-        parser.symbols.resize(len) catch unreachable;
+        _ = parser.scopes.pop();
     }
-    fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
+    fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Symbol {
         const name = parser.tree.tokenSlice(tok);
-        const syms = parser.symbols.toSliceConst();
+        var scope_it = parser.scopes.iterator(parser.scopes.len);
-        var i = syms.len;
+        while (scope_it.prev()) |scope| {
-        while (i > 0) : (i -= 1) {
+            var sym_it = scope.syms.iterator(scope.syms.len);
-            if (mem.eql(u8, name, syms[i].name)) {
+            while (sym_it.prev()) |sym| {
-                return syms[i].ty;
+                if (mem.eql(u8, sym.name, name)) {
+                    return sym;
+                }
             }
         }
         return null;
 @@ -137,8 +152,8 @@ const Parser = struct {
     /// Root <- ExternalDeclaration* eof
     fn root(parser: *Parser) Allocator.Error!*Node.Root {
-        const scope = parser.pushScope(.Root);
+        try parser.pushScope(.Root);
-        defer parser.popScope(scope);
+        defer parser.popScope();
         const node = try parser.arena.create(Node.Root);
         node.* = .{
             .decls = Node.Root.DeclList.init(parser.arena),
 @@ -782,8 +797,8 @@ const Parser = struct {
                 .ty = ty,
             });
         if (parser.eatToken(.LBrace)) |lbrace| {
-            const scope = parser.pushScope(.Block);
+            try parser.pushScope(.Block);
-            defer parser.popScope(scope);
+            defer parser.popScope();
             var fields = Node.RecordType.FieldList.init(parser.arena);
             while (true) {
                 if (parser.eatToken(.RBrace)) |rbrace| {
 @@ -996,15 +1011,14 @@ const Parser = struct {
     fn assignmentExpr(parser: *Parser) !*Node {}
     /// ConstExpr <- ConditionalExpr
-    fn constExpr(parser: *Parser) Error!*Node {
+    fn constExpr(parser: *Parser) Error!?*Expr {
         const start = parser.it.index;
         const expression = try parser.conditionalExpr();
-        // TODO
+        if (expression != null and expression.?.value == .None)
-        // if (expression == nullor expression.?.value == null)
+            return parser.err(.{
-        //     return parser.err(.{
+                .ConsExpr = start,
-        //         .ConsExpr = start,
+            });
-        //     });
+        return expression;
-        return expression.?;
     }
     /// ConditionalExpr <- LogicalOrExpr (QUESTIONMARK Expr COLON ConditionalExpr)?
 @@ -1085,8 +1099,8 @@ const Parser = struct {
     /// CompoundStmt <- LBRACE (Declaration / Stmt)* RBRACE
     fn compoundStmt(parser: *Parser) Error!?*Node {
         const lbrace = parser.eatToken(.LBrace) orelse return null;
-        const scope = parser.pushScope(.Block);
+        try parser.pushScope(.Block);
-        defer parser.popScope(scope);
+        defer parser.popScope();
         const body_node = try parser.arena.create(Node.CompoundStmt);
         body_node.* = .{
             .lbrace = lbrace,
 @@ -1142,8 +1156,8 @@ const Parser = struct {
             return &node.base;
         }
         if (parser.eatToken(.Keyword_while)) |tok| {
-            const scope = parser.pushScope(.Loop);
+            try parser.pushScope(.Loop);
-            defer parser.popScope(scope);
+            defer parser.popScope();
             _ = try parser.expectToken(.LParen);
             const cond = (try parser.expr()) orelse return parser.err(.{
                 .ExpectedExpr = .{ .token = parser.it.index },
 @@ -1160,8 +1174,8 @@ const Parser = struct {
             return &node.base;
         }
         if (parser.eatToken(.Keyword_do)) |tok| {
-            const scope = parser.pushScope(.Loop);
+            try parser.pushScope(.Loop);
-            defer parser.popScope(scope);
+            defer parser.popScope();
             const body = try parser.stmt();
             _ = try parser.expectToken(.LParen);
             const cond = (try parser.expr()) orelse return parser.err(.{
 @@ -1179,8 +1193,8 @@ const Parser = struct {
             return &node.base;
         }
         if (parser.eatToken(.Keyword_for)) |tok| {
-            const scope = parser.pushScope(.Loop);
+            try parser.pushScope(.Loop);
-            defer parser.popScope(scope);
+            defer parser.popScope();
             _ = try parser.expectToken(.LParen);
             const init = if (try parser.declaration()) |decl| blk:{
                 // TODO disallow storage class other than auto and register
 @@ -1203,8 +1217,8 @@ const Parser = struct {
             return &node.base;
         }
         if (parser.eatToken(.Keyword_switch)) |tok| {
-            const scope = parser.pushScope(.Switch);
+            try parser.pushScope(.Switch);
-            defer parser.popScope(scope);
+            defer parser.popScope();
             _ = try parser.expectToken(.LParen);
             const switch_expr = try parser.exprStmt();
             const rparen = try parser.expectToken(.RParen);

lib/std/c/tokenizer.zig+19-27

 @@ -401,7 +401,6 @@ pub const Tokenizer = struct {
             U,
             L,
             StringLiteral,
-            AfterStringLiteral,
             CharLiteralStart,
             CharLiteral,
             EscapeSequence,
 @@ -617,7 +616,7 @@ pub const Tokenizer = struct {
                 },
                 .BackSlash => switch (c) {
                     '\n' => {
-                        state = if (string) .AfterStringLiteral else .Start;
+                        state = .Start;
                     },
                     '\r' => {
                         state = .BackSlashCr;
 @@ -632,7 +631,7 @@ pub const Tokenizer = struct {
                 },
                 .BackSlashCr => switch (c) {
                     '\n' => {
-                        state = if (string) .AfterStringLiteral else .Start;
+                        state =  .Start;
                     },
                     else => {
                         result.id = .Invalid;
 @@ -696,7 +695,8 @@ pub const Tokenizer = struct {
                         state = .EscapeSequence;
                     },
                     '"' => {
-                        state = .AfterStringLiteral;
+                        self.index += 1;
+                        break;
                     },
                     '\n', '\r' => {
                         result.id = .Invalid;
 @@ -704,22 +704,6 @@ pub const Tokenizer = struct {
                     },
                     else => {},
                 },
-                .AfterStringLiteral => switch (c) {
-                    '"' => {
-                        state = .StringLiteral;
-                    },
-                    '\\' => {
-                        state = .BackSlash;
-                    },
-                    '\n', '\r' => {
-                        if (self.pp_directive)
-                            break;
-                    },
-                    '\t', '\x0B', '\x0C', ' ' => {},
-                    else => {
-                        break;
-                    },
-                },
                 .CharLiteralStart => switch (c) {
                     '\\' => {
                         string = false;
 @@ -1255,7 +1239,7 @@ pub const Tokenizer = struct {
             }
         } else if (self.index == self.source.buffer.len) {
             switch (state) {
-                .AfterStringLiteral, .Start => {},
+                .Start => {},
                 .u, .u8, .U, .L, .Identifier => {
                     result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
                 },
 @@ -1322,7 +1306,7 @@ pub const Tokenizer = struct {
 test "operators" {
     expectTokens(
-        \\ ! != | || |= = ==
+        \\ ! != | || |= = ==
         \\ ( ) { } [ ] . .. ...
         \\ ^ ^= + ++ += - -- -=
         \\ * *= % %= -> : ; / /=
 @@ -1505,24 +1489,27 @@ test "line continuation" {
         .Identifier,
         .Nl,
         .{ .StringLiteral = .None },
+        .Nl,
         .Hash,
         .Keyword_define,
         .{ .StringLiteral = .None },
         .Nl,
         .{ .StringLiteral = .None },
+        .Nl,
         .Hash,
         .Keyword_define,
         .{ .StringLiteral = .None },
+        .{ .StringLiteral = .None },
     });
 }
 test "string prefix" {
     expectTokens(
-        \\"foo" "bar"
+        \\"foo"
-        \\u"foo" "bar"
+        \\u"foo"
-        \\u8"foo" "bar"
+        \\u8"foo"
-        \\U"foo" "bar"
+        \\U"foo"
-        \\L"foo" "bar"
+        \\L"foo"
         \\'foo'
         \\u'foo'
         \\U'foo'
 @@ -1530,10 +1517,15 @@ test "string prefix" {
         \\
     , &[_]Token.Id{
         .{ .StringLiteral = .None },
+        .Nl,
         .{ .StringLiteral = .Utf16 },
+        .Nl,
         .{ .StringLiteral = .Utf8 },
+        .Nl,
         .{ .StringLiteral = .Utf32 },
+        .Nl,
         .{ .StringLiteral = .Wide },
+        .Nl,
         .{ .CharLiteral = .None },
         .Nl,
         .{ .CharLiteral = .Utf16 },