authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 02:44:23+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:50+02:00
logd75697a6a3e2c8d96819c365dcb5690d4d8028e9
treebb942b9ceaaad90e5325e9f7039601f41acb137f
parent26bf410b061b9d6d18e4945417ddec62d7486e9c
signature Commit is signed but in an unrecognized format.

std-c tokenizer keywords

1 files changed, 184 insertions(+), 10 deletions(-)

lib/std/c/tokenizer.zig+184-10
...@@ -1,5 +1,5 @@...@@ -1,5 +1,5 @@
1const std = @import("std");1const std = @import("std");
2const expect = std.testing.expect;2const mem = std.mem;
33
4pub const Source = struct {4pub const Source = struct {
5 buffer: []const u8,5 buffer: []const u8,
...@@ -7,11 +7,19 @@ pub const Source = struct {...@@ -7,11 +7,19 @@ pub const Source = struct {
7};7};
88
9pub const Token = struct {9pub const Token = struct {
10 id: union(enum) {10 id: Id,
11 start: usize,
12 end: usize,
13 source: *Source,
14
15 pub const Id = union(enum) {
11 Invalid,16 Invalid,
12 Eof,17 Eof,
13 Nl,18 Nl,
14 Identifier,19 Identifier,
20
21 /// special case for #include <...>
22 MacroString,
15 StringLiteral: StrKind,23 StringLiteral: StrKind,
16 CharLiteral: StrKind,24 CharLiteral: StrKind,
17 IntegerLiteral: NumSuffix,25 IntegerLiteral: NumSuffix,
...@@ -68,10 +76,160 @@ pub const Token = struct {...@@ -68,10 +76,160 @@ pub const Token = struct {
68 MultiLineComment,76 MultiLineComment,
69 Hash,77 Hash,
70 HashHash,78 HashHash,
71 },79
72 start: usize,80 Keyword_auto,
73 end: usize,81 Keyword_break,
74 source: *Source,82 Keyword_case,
83 Keyword_char,
84 Keyword_const,
85 Keyword_continue,
86 Keyword_default,
87 Keyword_do,
88 Keyword_double,
89 Keyword_else,
90 Keyword_enum,
91 Keyword_extern,
92 Keyword_float,
93 Keyword_for,
94 Keyword_goto,
95 Keyword_if,
96 Keyword_int,
97 Keyword_long,
98 Keyword_register,
99 Keyword_return,
100 Keyword_short,
101 Keyword_signed,
102 Keyword_sizeof,
103 Keyword_static,
104 Keyword_struct,
105 Keyword_switch,
106 Keyword_typedef,
107 Keyword_union,
108 Keyword_unsigned,
109 Keyword_void,
110 Keyword_volatile,
111 Keyword_while,
112
113 // ISO C99
114 Keyword_bool,
115 Keyword_complex,
116 Keyword_imaginary,
117 Keyword_inline,
118 Keyword_restrict,
119
120 // ISO C11
121 Keyword_alignas,
122 Keyword_alignof,
123 Keyword_atomic,
124 Keyword_generic,
125 Keyword_noreturn,
126 Keyword_static_assert,
127 Keyword_thread_local,
128
129 // Preprocessor
130 Keyword_include,
131 Keyword_define,
132 Keyword_ifdef,
133 Keyword_ifndef,
134 Keyword_error,
135 Keyword_pragma,
136 };
137
138 pub const Keyword = struct {
139 bytes: []const u8,
140 id: Id,
141 hash: u32,
142
143 fn init(bytes: []const u8, id: Id) Keyword {
144 @setEvalBranchQuota(2000);
145 return .{
146 .bytes = bytes,
147 .id = id,
148 .hash = std.hash_map.hashString(bytes),
149 };
150 }
151 };
152
153 // TODO extensions
154 pub const keywords = [_]Keyword{
155 Keyword.init("auto", .Keyword_auto),
156 Keyword.init("break", .Keyword_break),
157 Keyword.init("case", .Keyword_case),
158 Keyword.init("char", .Keyword_char),
159 Keyword.init("const", .Keyword_const),
160 Keyword.init("continue", .Keyword_continue),
161 Keyword.init("default", .Keyword_default),
162 Keyword.init("do", .Keyword_do),
163 Keyword.init("double", .Keyword_double),
164 Keyword.init("else", .Keyword_else),
165 Keyword.init("enum", .Keyword_enum),
166 Keyword.init("extern", .Keyword_extern),
167 Keyword.init("float", .Keyword_float),
168 Keyword.init("for", .Keyword_for),
169 Keyword.init("goto", .Keyword_goto),
170 Keyword.init("if", .Keyword_if),
171 Keyword.init("int", .Keyword_int),
172 Keyword.init("long", .Keyword_long),
173 Keyword.init("register", .Keyword_register),
174 Keyword.init("return", .Keyword_return),
175 Keyword.init("short", .Keyword_short),
176 Keyword.init("signed", .Keyword_signed),
177 Keyword.init("sizeof", .Keyword_sizeof),
178 Keyword.init("static", .Keyword_static),
179 Keyword.init("struct", .Keyword_struct),
180 Keyword.init("switch", .Keyword_switch),
181 Keyword.init("typedef", .Keyword_typedef),
182 Keyword.init("union", .Keyword_union),
183 Keyword.init("unsigned", .Keyword_unsigned),
184 Keyword.init("void", .Keyword_void),
185 Keyword.init("volatile", .Keyword_volatile),
186 Keyword.init("while", .Keyword_while),
187
188 // ISO C99
189 Keyword.init("_Bool", .Keyword_bool),
190 Keyword.init("_Complex", .Keyword_complex),
191 Keyword.init("_Imaginary", .Keyword_imaginary),
192 Keyword.init("inline", .Keyword_inline),
193 Keyword.init("restrict", .Keyword_restrict),
194
195 // ISO C11
196 Keyword.init("_Alignas", .Keyword_alignas),
197 Keyword.init("_Alignof", .Keyword_alignof),
198 Keyword.init("_Atomic", .Keyword_atomic),
199 Keyword.init("_Generic", .Keyword_generic),
200 Keyword.init("_Noreturn", .Keyword_noreturn),
201 Keyword.init("_Static_assert", .Keyword_static_assert),
202 Keyword.init("_Thread_local", .Keyword_thread_local),
203
204 // Preprocessor
205 Keyword.init("include", .Keyword_include),
206 Keyword.init("define", .Keyword_define),
207 Keyword.init("ifdef", .Keyword_ifdef),
208 Keyword.init("ifndef", .Keyword_ifndef),
209 Keyword.init("error", .Keyword_error),
210 Keyword.init("pragma", .Keyword_pragma),
211 };
212
213 // TODO perfect hash at comptime
214 pub fn getKeyword(bytes: []const u8, macro: bool) ?Id {
215 var hash = std.hash_map.hashString(bytes);
216 for (keywords) |kw| {
217 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {
218 switch (kw.id) {
219 .Keyword_include,
220 .Keyword_define,
221 .Keyword_ifdef,
222 .Keyword_ifndef,
223 .Keyword_error,
224 .Keyword_pragma,
225 => if (!macro) return null,
226 else => {},
227 }
228 return kw.id;
229 }
230 }
231 return null;
232 }
75233
76 pub const NumSuffix = enum {234 pub const NumSuffix = enum {
77 None,235 None,
...@@ -95,6 +253,7 @@ pub const Token = struct {...@@ -95,6 +253,7 @@ pub const Token = struct {
95pub const Tokenizer = struct {253pub const Tokenizer = struct {
96 source: *Source,254 source: *Source,
97 index: usize = 0,255 index: usize = 0,
256 prev_tok_id: @TagType(Token.Id),
98257
99 pub fn next(self: *Tokenizer) Token {258 pub fn next(self: *Tokenizer) Token {
100 const start_index = self.index;259 const start_index = self.index;
...@@ -124,6 +283,9 @@ pub const Tokenizer = struct {...@@ -124,6 +283,9 @@ pub const Tokenizer = struct {
124 Percent,283 Percent,
125 Asterisk,284 Asterisk,
126 Plus,285 Plus,
286
287 /// special case for #include <...>
288 MacroString,
127 AngleBracketLeft,289 AngleBracketLeft,
128 AngleBracketAngleBracketLeft,290 AngleBracketAngleBracketLeft,
129 AngleBracketRight,291 AngleBracketRight,
...@@ -189,7 +351,6 @@ pub const Tokenizer = struct {...@@ -189,7 +351,6 @@ pub const Tokenizer = struct {
189 },351 },
190 'a'...'t', 'v'...'z', 'A'...'K', 'M'...'T', 'V'...'Z', '_' => {352 'a'...'t', 'v'...'z', 'A'...'K', 'M'...'T', 'V'...'Z', '_' => {
191 state = .Identifier;353 state = .Identifier;
192 result.id = .Identifier;
193 },354 },
194 '=' => {355 '=' => {
195 state = .Equal;356 state = .Equal;
...@@ -250,7 +411,10 @@ pub const Tokenizer = struct {...@@ -250,7 +411,10 @@ pub const Tokenizer = struct {
250 state = .Plus;411 state = .Plus;
251 },412 },
252 '<' => {413 '<' => {
253 state = .AngleBracketLeft;414 if (self.prev_tok_id == .Keyword_include)
415 state = .MacroString
416 else
417 state = .AngleBracketLeft;
254 },418 },
255 '>' => {419 '>' => {
256 state = .AngleBracketRight;420 state = .AngleBracketRight;
...@@ -442,7 +606,7 @@ pub const Tokenizer = struct {...@@ -442,7 +606,7 @@ pub const Tokenizer = struct {
442 .Identifier => switch (c) {606 .Identifier => switch (c) {
443 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},607 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
444 else => {608 else => {
445 result.id = .Identifier;609 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;
446 break;610 break;
447 },611 },
448 },612 },
...@@ -522,6 +686,14 @@ pub const Tokenizer = struct {...@@ -522,6 +686,14 @@ pub const Tokenizer = struct {
522 break;686 break;
523 },687 },
524 },688 },
689 .MacroString => switch (c) {
690 '>' => {
691 result.id = .MacroString;
692 self.index += 1;
693 break;
694 },
695 else => {},
696 },
525 .AngleBracketLeft => switch (c) {697 .AngleBracketLeft => switch (c) {
526 '<' => {698 '<' => {
527 state = .AngleBracketAngleBracketLeft;699 state = .AngleBracketAngleBracketLeft;
...@@ -859,7 +1031,7 @@ pub const Tokenizer = struct {...@@ -859,7 +1031,7 @@ pub const Tokenizer = struct {
859 switch (state) {1031 switch (state) {
860 .Start => {},1032 .Start => {},
861 .u, .u8, .U, .L, .Identifier => {1033 .u, .u8, .U, .L, .Identifier => {
862 result.id = .Identifier;1034 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash) orelse .Identifier;
863 },1035 },
8641036
865 .Cr,1037 .Cr,
...@@ -876,6 +1048,7 @@ pub const Tokenizer = struct {...@@ -876,6 +1048,7 @@ pub const Tokenizer = struct {
876 .FloatFractionHex,1048 .FloatFractionHex,
877 .FloatExponent,1049 .FloatExponent,
878 .FloatExponentDigits,1050 .FloatExponentDigits,
1051 .MacroString,
879 => result.id = .Invalid,1052 => result.id = .Invalid,
8801053
881 .IntegerLiteralOct,1054 .IntegerLiteralOct,
...@@ -910,6 +1083,7 @@ pub const Tokenizer = struct {...@@ -910,6 +1083,7 @@ pub const Tokenizer = struct {
910 }1083 }
911 }1084 }
9121085
1086 self.prev_tok_id = result.id;
913 result.end = self.index;1087 result.end = self.index;
914 return result;1088 return result;
915 }1089 }