authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 10:58:12+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
logc221593d7d6d441c04c9332aaa6d2be8b3d24bc0
treea1e6bb6da30004f49c82008f0d8185da894e7e4e
parent472ca947c94f703866eec75fc364810e655b4894
signature Commit is signed but in an unrecognized format.

std-c tokenizer better special case handling

1 files changed, 24 insertions(+), 29 deletions(-)

lib/std/c/tokenizer.zig+24-29
...@@ -124,7 +124,7 @@ pub const Token = struct {...@@ -124,7 +124,7 @@ pub const Token = struct {
124 Keyword_static_assert,124 Keyword_static_assert,
125 Keyword_thread_local,125 Keyword_thread_local,
126126
127 // Preprocessor127 // Preprocessor directives
128 Keyword_include,128 Keyword_include,
129 Keyword_define,129 Keyword_define,
130 Keyword_ifdef,130 Keyword_ifdef,
...@@ -199,7 +199,7 @@ pub const Token = struct {...@@ -199,7 +199,7 @@ pub const Token = struct {
199 Keyword.init("_Static_assert", .Keyword_static_assert),199 Keyword.init("_Static_assert", .Keyword_static_assert),
200 Keyword.init("_Thread_local", .Keyword_thread_local),200 Keyword.init("_Thread_local", .Keyword_thread_local),
201201
202 // Preprocessor202 // Preprocessor directives
203 Keyword.init("include", .Keyword_include),203 Keyword.init("include", .Keyword_include),
204 Keyword.init("define", .Keyword_define),204 Keyword.init("define", .Keyword_define),
205 Keyword.init("ifdef", .Keyword_ifdef),205 Keyword.init("ifdef", .Keyword_ifdef),
...@@ -209,7 +209,7 @@ pub const Token = struct {...@@ -209,7 +209,7 @@ pub const Token = struct {
209 };209 };
210210
211 // TODO perfect hash at comptime211 // TODO perfect hash at comptime
212 pub fn getKeyword(bytes: []const u8, macro: bool) ?Id {212 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {
213 var hash = std.hash_map.hashString(bytes);213 var hash = std.hash_map.hashString(bytes);
214 for (keywords) |kw| {214 for (keywords) |kw| {
215 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {215 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {
...@@ -220,7 +220,7 @@ pub const Token = struct {...@@ -220,7 +220,7 @@ pub const Token = struct {
220 .Keyword_ifndef,220 .Keyword_ifndef,
221 .Keyword_error,221 .Keyword_error,
222 .Keyword_pragma,222 .Keyword_pragma,
223 => if (!macro) return null,223 => if (!pp_directive) return null,
224 else => {},224 else => {},
225 }225 }
226 return kw.id;226 return kw.id;
...@@ -252,6 +252,7 @@ pub const Tokenizer = struct {...@@ -252,6 +252,7 @@ pub const Tokenizer = struct {
252 source: *Source,252 source: *Source,
253 index: usize = 0,253 index: usize = 0,
254 prev_tok_id: @TagType(Token.Id) = .Invalid,254 prev_tok_id: @TagType(Token.Id) = .Invalid,
255 pp_directive: bool = false,
255256
256 pub fn next(self: *Tokenizer) Token {257 pub fn next(self: *Tokenizer) Token {
257 const start_index = self.index;258 const start_index = self.index;
...@@ -321,11 +322,20 @@ pub const Tokenizer = struct {...@@ -321,11 +322,20 @@ pub const Tokenizer = struct {
321 switch (state) {322 switch (state) {
322 .Start => switch (c) {323 .Start => switch (c) {
323 '\n' => {324 '\n' => {
325 if (!self.pp_directive) {
326 result.start = self.index + 1;
327 continue;
328 }
329 self.pp_directive = false;
324 result.id = .Nl;330 result.id = .Nl;
325 self.index += 1;331 self.index += 1;
326 break;332 break;
327 },333 },
328 '\r' => {334 '\r' => {
335 if (!self.pp_directive) {
336 result.start = self.index + 1;
337 continue;
338 }
329 state = .Cr;339 state = .Cr;
330 },340 },
331 '"' => {341 '"' => {
...@@ -460,6 +470,7 @@ pub const Tokenizer = struct {...@@ -460,6 +470,7 @@ pub const Tokenizer = struct {
460 },470 },
461 .Cr => switch (c) {471 .Cr => switch (c) {
462 '\n' => {472 '\n' => {
473 self.pp_directive = false;
463 result.id = .Nl;474 result.id = .Nl;
464 self.index += 1;475 self.index += 1;
465 break;476 break;
...@@ -603,7 +614,9 @@ pub const Tokenizer = struct {...@@ -603,7 +614,9 @@ pub const Tokenizer = struct {
603 .Identifier => switch (c) {614 .Identifier => switch (c) {
604 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},615 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
605 else => {616 else => {
606 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;617 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
618 if (self.prev_tok_id == .Hash)
619 self.pp_directive = true;
607 break;620 break;
608 },621 },
609 },622 },
...@@ -1039,7 +1052,7 @@ pub const Tokenizer = struct {...@@ -1039,7 +1052,7 @@ pub const Tokenizer = struct {
1039 switch (state) {1052 switch (state) {
1040 .Start => {},1053 .Start => {},
1041 .u, .u8, .U, .L, .Identifier => {1054 .u, .u8, .U, .L, .Identifier => {
1042 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;1055 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
1043 },1056 },
10441057
1045 .Cr,1058 .Cr,
...@@ -1116,8 +1129,6 @@ test "operators" {...@@ -1116,8 +1129,6 @@ test "operators" {
1116 .PipeEqual,1129 .PipeEqual,
1117 .Equal,1130 .Equal,
1118 .EqualEqual,1131 .EqualEqual,
1119 .Nl,
1120
1121 .LParen,1132 .LParen,
1122 .RParen,1133 .RParen,
1123 .LBrace,1134 .LBrace,
...@@ -1128,8 +1139,6 @@ test "operators" {...@@ -1128,8 +1139,6 @@ test "operators" {
1128 .Period,1139 .Period,
1129 .Period,1140 .Period,
1130 .Ellipsis,1141 .Ellipsis,
1131 .Nl,
1132
1133 .Caret,1142 .Caret,
1134 .CaretEqual,1143 .CaretEqual,
1135 .Plus,1144 .Plus,
...@@ -1138,8 +1147,6 @@ test "operators" {...@@ -1138,8 +1147,6 @@ test "operators" {
1138 .Minus,1147 .Minus,
1139 .MinusMinus,1148 .MinusMinus,
1140 .MinusEqual,1149 .MinusEqual,
1141 .Nl,
1142
1143 .Asterisk,1150 .Asterisk,
1144 .AsteriskEqual,1151 .AsteriskEqual,
1145 .Percent,1152 .Percent,
...@@ -1149,8 +1156,6 @@ test "operators" {...@@ -1149,8 +1156,6 @@ test "operators" {
1149 .Semicolon,1156 .Semicolon,
1150 .Slash,1157 .Slash,
1151 .SlashEqual,1158 .SlashEqual,
1152 .Nl,
1153
1154 .Comma,1159 .Comma,
1155 .Ampersand,1160 .Ampersand,
1156 .AmpersandAmpersand,1161 .AmpersandAmpersand,
...@@ -1159,8 +1164,6 @@ test "operators" {...@@ -1159,8 +1164,6 @@ test "operators" {
1159 .AngleBracketLeft,1164 .AngleBracketLeft,
1160 .AngleBracketLeftEqual,1165 .AngleBracketLeftEqual,
1161 .AngleBracketAngleBracketLeft,1166 .AngleBracketAngleBracketLeft,
1162 .Nl,
1163
1164 .AngleBracketAngleBracketLeftEqual,1167 .AngleBracketAngleBracketLeftEqual,
1165 .AngleBracketRight,1168 .AngleBracketRight,
1166 .AngleBracketRightEqual,1169 .AngleBracketRightEqual,
...@@ -1169,7 +1172,6 @@ test "operators" {...@@ -1169,7 +1172,6 @@ test "operators" {
1169 .Tilde,1172 .Tilde,
1170 .Hash,1173 .Hash,
1171 .HashHash,1174 .HashHash,
1172 .Nl,
1173 },1175 },
1174 );1176 );
1175}1177}
...@@ -1192,8 +1194,6 @@ test "keywords" {...@@ -1192,8 +1194,6 @@ test "keywords" {
1192 .Keyword_continue,1194 .Keyword_continue,
1193 .Keyword_default,1195 .Keyword_default,
1194 .Keyword_do,1196 .Keyword_do,
1195 .Nl,
1196
1197 .Keyword_double,1197 .Keyword_double,
1198 .Keyword_else,1198 .Keyword_else,
1199 .Keyword_enum,1199 .Keyword_enum,
...@@ -1203,8 +1203,6 @@ test "keywords" {...@@ -1203,8 +1203,6 @@ test "keywords" {
1203 .Keyword_goto,1203 .Keyword_goto,
1204 .Keyword_if,1204 .Keyword_if,
1205 .Keyword_int,1205 .Keyword_int,
1206 .Nl,
1207
1208 .Keyword_long,1206 .Keyword_long,
1209 .Keyword_register,1207 .Keyword_register,
1210 .Keyword_return,1208 .Keyword_return,
...@@ -1212,8 +1210,6 @@ test "keywords" {...@@ -1212,8 +1210,6 @@ test "keywords" {
1212 .Keyword_signed,1210 .Keyword_signed,
1213 .Keyword_sizeof,1211 .Keyword_sizeof,
1214 .Keyword_static,1212 .Keyword_static,
1215 .Nl,
1216
1217 .Keyword_struct,1213 .Keyword_struct,
1218 .Keyword_switch,1214 .Keyword_switch,
1219 .Keyword_typedef,1215 .Keyword_typedef,
...@@ -1221,8 +1217,6 @@ test "keywords" {...@@ -1221,8 +1217,6 @@ test "keywords" {
1221 .Keyword_unsigned,1217 .Keyword_unsigned,
1222 .Keyword_void,1218 .Keyword_void,
1223 .Keyword_volatile,1219 .Keyword_volatile,
1224 .Nl,
1225
1226 .Keyword_while,1220 .Keyword_while,
1227 .Keyword_bool,1221 .Keyword_bool,
1228 .Keyword_complex,1222 .Keyword_complex,
...@@ -1230,22 +1224,19 @@ test "keywords" {...@@ -1230,22 +1224,19 @@ test "keywords" {
1230 .Keyword_inline,1224 .Keyword_inline,
1231 .Keyword_restrict,1225 .Keyword_restrict,
1232 .Keyword_alignas,1226 .Keyword_alignas,
1233 .Nl,
1234
1235 .Keyword_alignof,1227 .Keyword_alignof,
1236 .Keyword_atomic,1228 .Keyword_atomic,
1237 .Keyword_generic,1229 .Keyword_generic,
1238 .Keyword_noreturn,1230 .Keyword_noreturn,
1239 .Keyword_static_assert,1231 .Keyword_static_assert,
1240 .Keyword_thread_local,1232 .Keyword_thread_local,
1241 .Nl,
1242 });1233 });
1243}1234}
12441235
1245test "preprocessor keywords" {1236test "preprocessor keywords" {
1246 expectTokens(1237 expectTokens(
1247 \\#include <test>1238 \\#include <test>
1248 \\#define1239 \\#define #include <1
1249 \\#ifdef1240 \\#ifdef
1250 \\#ifndef1241 \\#ifndef
1251 \\#error1242 \\#error
...@@ -1258,6 +1249,10 @@ test "preprocessor keywords" {...@@ -1258,6 +1249,10 @@ test "preprocessor keywords" {
1258 .Nl,1249 .Nl,
1259 .Hash,1250 .Hash,
1260 .Keyword_define,1251 .Keyword_define,
1252 .Hash,
1253 .Identifier,
1254 .AngleBracketLeft,
1255 .{ .IntegerLiteral = .None },
1261 .Nl,1256 .Nl,
1262 .Hash,1257 .Hash,
1263 .Keyword_ifdef,1258 .Keyword_ifdef,