authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 10:58:12+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
logc221593d7d6d441c04c9332aaa6d2be8b3d24bc0
treea1e6bb6da30004f49c82008f0d8185da894e7e4e
parent472ca947c94f703866eec75fc364810e655b4894
signature Commit is signed but in an unrecognized format.

std-c tokenizer better special case handling

1 files changed, 24 insertions(+), 29 deletions(-)

lib/std/c/tokenizer.zig+24-29
......@@ -124,7 +124,7 @@ pub const Token = struct {
124124 Keyword_static_assert,
125125 Keyword_thread_local,
126126
127 // Preprocessor
127 // Preprocessor directives
128128 Keyword_include,
129129 Keyword_define,
130130 Keyword_ifdef,
......@@ -199,7 +199,7 @@ pub const Token = struct {
199199 Keyword.init("_Static_assert", .Keyword_static_assert),
200200 Keyword.init("_Thread_local", .Keyword_thread_local),
201201
202 // Preprocessor
202 // Preprocessor directives
203203 Keyword.init("include", .Keyword_include),
204204 Keyword.init("define", .Keyword_define),
205205 Keyword.init("ifdef", .Keyword_ifdef),
......@@ -209,7 +209,7 @@ pub const Token = struct {
209209 };
210210
211211 // TODO perfect hash at comptime
212 pub fn getKeyword(bytes: []const u8, macro: bool) ?Id {
212 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {
213213 var hash = std.hash_map.hashString(bytes);
214214 for (keywords) |kw| {
215215 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {
......@@ -220,7 +220,7 @@ pub const Token = struct {
220220 .Keyword_ifndef,
221221 .Keyword_error,
222222 .Keyword_pragma,
223 => if (!macro) return null,
223 => if (!pp_directive) return null,
224224 else => {},
225225 }
226226 return kw.id;
......@@ -252,6 +252,7 @@ pub const Tokenizer = struct {
252252 source: *Source,
253253 index: usize = 0,
254254 prev_tok_id: @TagType(Token.Id) = .Invalid,
255 pp_directive: bool = false,
255256
256257 pub fn next(self: *Tokenizer) Token {
257258 const start_index = self.index;
......@@ -321,11 +322,20 @@ pub const Tokenizer = struct {
321322 switch (state) {
322323 .Start => switch (c) {
323324 '\n' => {
325 if (!self.pp_directive) {
326 result.start = self.index + 1;
327 continue;
328 }
329 self.pp_directive = false;
324330 result.id = .Nl;
325331 self.index += 1;
326332 break;
327333 },
328334 '\r' => {
335 if (!self.pp_directive) {
336 result.start = self.index + 1;
337 continue;
338 }
329339 state = .Cr;
330340 },
331341 '"' => {
......@@ -460,6 +470,7 @@ pub const Tokenizer = struct {
460470 },
461471 .Cr => switch (c) {
462472 '\n' => {
473 self.pp_directive = false;
463474 result.id = .Nl;
464475 self.index += 1;
465476 break;
......@@ -603,7 +614,9 @@ pub const Tokenizer = struct {
603614 .Identifier => switch (c) {
604615 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
605616 else => {
606 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;
617 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
618 if (self.prev_tok_id == .Hash)
619 self.pp_directive = true;
607620 break;
608621 },
609622 },
......@@ -1039,7 +1052,7 @@ pub const Tokenizer = struct {
10391052 switch (state) {
10401053 .Start => {},
10411054 .u, .u8, .U, .L, .Identifier => {
1042 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;
1055 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
10431056 },
10441057
10451058 .Cr,
......@@ -1116,8 +1129,6 @@ test "operators" {
11161129 .PipeEqual,
11171130 .Equal,
11181131 .EqualEqual,
1119 .Nl,
1120
11211132 .LParen,
11221133 .RParen,
11231134 .LBrace,
......@@ -1128,8 +1139,6 @@ test "operators" {
11281139 .Period,
11291140 .Period,
11301141 .Ellipsis,
1131 .Nl,
1132
11331142 .Caret,
11341143 .CaretEqual,
11351144 .Plus,
......@@ -1138,8 +1147,6 @@ test "operators" {
11381147 .Minus,
11391148 .MinusMinus,
11401149 .MinusEqual,
1141 .Nl,
1142
11431150 .Asterisk,
11441151 .AsteriskEqual,
11451152 .Percent,
......@@ -1149,8 +1156,6 @@ test "operators" {
11491156 .Semicolon,
11501157 .Slash,
11511158 .SlashEqual,
1152 .Nl,
1153
11541159 .Comma,
11551160 .Ampersand,
11561161 .AmpersandAmpersand,
......@@ -1159,8 +1164,6 @@ test "operators" {
11591164 .AngleBracketLeft,
11601165 .AngleBracketLeftEqual,
11611166 .AngleBracketAngleBracketLeft,
1162 .Nl,
1163
11641167 .AngleBracketAngleBracketLeftEqual,
11651168 .AngleBracketRight,
11661169 .AngleBracketRightEqual,
......@@ -1169,7 +1172,6 @@ test "operators" {
11691172 .Tilde,
11701173 .Hash,
11711174 .HashHash,
1172 .Nl,
11731175 },
11741176 );
11751177}
......@@ -1192,8 +1194,6 @@ test "keywords" {
11921194 .Keyword_continue,
11931195 .Keyword_default,
11941196 .Keyword_do,
1195 .Nl,
1196
11971197 .Keyword_double,
11981198 .Keyword_else,
11991199 .Keyword_enum,
......@@ -1203,8 +1203,6 @@ test "keywords" {
12031203 .Keyword_goto,
12041204 .Keyword_if,
12051205 .Keyword_int,
1206 .Nl,
1207
12081206 .Keyword_long,
12091207 .Keyword_register,
12101208 .Keyword_return,
......@@ -1212,8 +1210,6 @@ test "keywords" {
12121210 .Keyword_signed,
12131211 .Keyword_sizeof,
12141212 .Keyword_static,
1215 .Nl,
1216
12171213 .Keyword_struct,
12181214 .Keyword_switch,
12191215 .Keyword_typedef,
......@@ -1221,8 +1217,6 @@ test "keywords" {
12211217 .Keyword_unsigned,
12221218 .Keyword_void,
12231219 .Keyword_volatile,
1224 .Nl,
1225
12261220 .Keyword_while,
12271221 .Keyword_bool,
12281222 .Keyword_complex,
......@@ -1230,22 +1224,19 @@ test "keywords" {
12301224 .Keyword_inline,
12311225 .Keyword_restrict,
12321226 .Keyword_alignas,
1233 .Nl,
1234
12351227 .Keyword_alignof,
12361228 .Keyword_atomic,
12371229 .Keyword_generic,
12381230 .Keyword_noreturn,
12391231 .Keyword_static_assert,
12401232 .Keyword_thread_local,
1241 .Nl,
12421233 });
12431234}
12441235
12451236test "preprocessor keywords" {
12461237 expectTokens(
12471238 \\#include <test>
1248 \\#define
1239 \\#define #include <1
12491240 \\#ifdef
12501241 \\#ifndef
12511242 \\#error
......@@ -1258,6 +1249,10 @@ test "preprocessor keywords" {
12581249 .Nl,
12591250 .Hash,
12601251 .Keyword_define,
1252 .Hash,
1253 .Identifier,
1254 .AngleBracketLeft,
1255 .{ .IntegerLiteral = .None },
12611256 .Nl,
12621257 .Hash,
12631258 .Keyword_ifdef,