authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 22:43:44+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-07 22:43:44+02:00
log8b713ce88959a953c6d41b5d1372e9b3e666512f
treeedf8a8e3fecf0244256231ddba9e7bb62aa80fbc
parentdbc045706809e7135fdb55ebd7c4a7383f40bf0f
signature Commit is signed but in an unrecognized format.

std-c parser add options

3 files changed, 79 insertions(+), 19 deletions(-)

lib/std/c/ast.zig+24-3
......@@ -24,9 +24,14 @@ pub const Tree = struct {
2424 // self is destroyed
2525 }
2626
27 pub fn slice(tree: *Tree, token: TokenIndex) []const u8 {
28 const tok = tree.tokens.at(token);
29 return tok.source.buffer[tok.start..tok.end];
27 pub fn tokenSlice(tree: *Tree, token: TokenIndex) []const u8 {
28 return tree.tokens.at(token).slice();
29 }
30
31 pub fn tokenEql(tree: *Tree, a: TokenIndex, b: TokenIndex) bool {
32 const atok = tree.tokens.at(a);
33 const btok = tree.tokens.at(b);
34 return atok.eql(btok.*);
3035 }
3136};
3237
......@@ -205,6 +210,10 @@ pub const Type = struct {
205210 Typedef: *Type,
206211 Record: *Node.RecordType,
207212 Enum: *Node.EnumType,
213
214 /// Special case for macro parameters that can be any type.
215 /// Only present if `retain_macros == true`.
216 Macro,
208217 },
209218};
210219
......@@ -586,4 +595,16 @@ pub const Node = struct {
586595 expr: *Expr,
587596 pub const InitializerList = std.SegmentedList(*Initializer, 4);
588597 };
598
599 pub const Macro = struct {
600 base: Node = Node{ .id = Macro },
601 kind: union(enum) {
602 Undef: []const u8,
603 Fn: struct {
604 params: []const []const u8,
605 expr: *Expr,
606 },
607 Expr: *Expr,
608 },
609 };
589610};
lib/std/c/parse.zig+45-16
......@@ -1,4 +1,5 @@
11const std = @import("std");
2const mem = std.mem;
23const assert = std.debug.assert;
34const Allocator = std.mem.Allocator;
45const ast = std.c.ast;
......@@ -11,9 +12,26 @@ const TokenIterator = ast.Tree.TokenList.Iterator;
1112
1213pub const Error = error{ParseError} || Allocator.Error;
1314
15pub const Options = struct {
16 /// Keep simple macros unexpanded and add the definitions to the ast
17 retain_macros: bool = false,
18
19 /// Warning or error
20 warn_as_err: union(enum) {
21 /// All warnings are warnings
22 None,
23
24 /// Some warnings are errors
25 Some: []@TagType(ast.Error),
26
27 /// All warnings are errors
28 All,
29 } = .All,
30};
31
1432/// Result should be freed with tree.deinit() when there are
1533/// no more references to any of the tokens or nodes.
16pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
34pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree {
1735 const tree = blk: {
1836 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists
1937 // from being initialized with a pointer to this `arena`, which is created on
......@@ -62,6 +80,7 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
6280 .arena = arena,
6381 .it = &it,
6482 .tree = tree,
83 .options = options,
6584 };
6685 defer parser.symbols.deinit();
6786
......@@ -76,7 +95,7 @@ const Parser = struct {
7695
7796 /// only used for scopes
7897 symbols: SymbolList,
79 warnings: bool = true,
98 options: Options,
8099
81100 const SymbolList = std.ArrayList(Symbol);
82101
......@@ -94,11 +113,11 @@ const Parser = struct {
94113 }
95114
96115 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Type {
97 const name = parser.tree.slice(tok);
116 const name = parser.tree.tokenSlice(tok);
98117 const syms = parser.symbols.toSliceConst();
99118 var i = syms.len;
100119 while (i > 0) : (i -= 1) {
101 if (std.mem.eql(u8, name, syms[i].name)) {
120 if (mem.eql(u8, name, syms[i].name)) {
102121 return syms[i].ty;
103122 }
104123 }
......@@ -249,7 +268,7 @@ const Parser = struct {
249268 try node.initializers.push((try parser.initializer(dr)) orelse return parser.err(.{
250269 .ExpectedInitializer = .{ .token = parser.it.index },
251270 }));
252 } else
271 } else
253272 try node.initializers.push(&dr.base);
254273 if (parser.eatToken(.Comma) != null) break;
255274 dr = @fieldParentPtr(Node.Declarator, "base", (try parser.declarator(.Must)) orelse return parser.err(.{
......@@ -558,12 +577,18 @@ const Parser = struct {
558577 return false;
559578 };
560579 switch (ty.id) {
561 .Enum => |e| {
580 .Enum => |e| blk: {
581 if (e.name) |some|
582 if (!parser.tree.tokenEql(some, tok))
583 break :blk;
562584 return parser.err(.{
563585 .MustUseKwToRefer = .{ .kw = e.tok, .name = tok },
564586 });
565587 },
566 .Record => |r| {
588 .Record => |r| blk: {
589 if (r.name) |some|
590 if (!parser.tree.tokenEql(some, tok))
591 break :blk;
567592 return parser.err(.{
568593 .MustUseKwToRefer = .{
569594 .kw = r.tok,
......@@ -580,11 +605,10 @@ const Parser = struct {
580605 };
581606 return true;
582607 },
583 else => {
584 parser.putBackToken(tok);
585 return false;
586 },
608 else => {},
587609 }
610 parser.putBackToken(tok);
611 return false;
588612 }
589613 }
590614 return parser.err(.{
......@@ -680,7 +704,7 @@ const Parser = struct {
680704 };
681705 if (name) |some|
682706 try parser.symbols.append(.{
683 .name = parser.tree.slice(some),
707 .name = parser.tree.tokenSlice(some),
684708 .ty = ty,
685709 });
686710 if (parser.eatToken(.LBrace)) |lbrace| {
......@@ -718,7 +742,7 @@ const Parser = struct {
718742 fn recordSpec(parser: *Parser, tok: TokenIndex) !*Node.RecordType {
719743 const node = try parser.arena.create(Node.RecordType);
720744 const name = parser.eatToken(.Identifier);
721 const is_struct = parser.tree.slice(tok)[0] == 's';
745 const is_struct = parser.tree.tokenSlice(tok)[0] == 's';
722746 node.* = .{
723747 .tok = tok,
724748 .kind = if (is_struct) .Struct else .Union,
......@@ -733,7 +757,7 @@ const Parser = struct {
733757 };
734758 if (name) |some|
735759 try parser.symbols.append(.{
736 .name = parser.tree.slice(some),
760 .name = parser.tree.tokenSlice(some),
737761 .ty = ty,
738762 });
739763 if (parser.eatToken(.LBrace)) |lbrace| {
......@@ -1195,11 +1219,16 @@ const Parser = struct {
11951219 }
11961220
11971221 fn warn(parser: *Parser, msg: ast.Error) Error!void {
1222 const is_warning = switch (parser.options.warn_as_err) {
1223 .None => true,
1224 .Some => |list| for (list) |item| (if (item == msg) break false) else true,
1225 .All => false,
1226 };
11981227 try parser.tree.msgs.push(.{
1199 .kind = if (parser.warnings) .Warning else .Error,
1228 .kind = if (is_warning) .Warning else .Error,
12001229 .inner = msg,
12011230 });
1202 if (!parser.warnings) return error.ParseError;
1231 if (!is_warning) return error.ParseError;
12031232 }
12041233
12051234 fn note(parser: *Parser, msg: ast.Error) Error!void {
lib/std/c/tokenizer.zig+10
......@@ -251,6 +251,16 @@ pub const Token = struct {
251251 }
252252 };
253253
254 pub fn eql(a: Token, b: Token) bool {
255 // do we really need this cast here
256 if (@as(@TagType(Id), a.id) != b.id) return false;
257 return mem.eql(u8, a.slice(), b.slice());
258 }
259
260 pub fn slice(tok: Token) []const u8 {
261 return tok.source.buffer[tok.start..tok.end];
262 }
263
254264 pub const Keyword = struct {
255265 bytes: []const u8,
256266 id: Id,