authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 02:00:29+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:49+02:00
log26bf410b061b9d6d18e4945417ddec62d7486e9c
tree3ba9bb931511c9bd03d69daf0388aab34b993ded
parentf14a5287e92755f8d1f7f592caeed77bac940958
signature Commit is signed but in an unrecognized format.

std-c finish tokenizer

1 files changed, 108 insertions(+), 58 deletions(-)

lib/std/c/tokenizer.zig+108-58
......@@ -7,21 +7,15 @@ pub const Source = struct {
77};
88
99pub const Token = struct {
10 id: Id,
11 num_suffix: NumSuffix = .None,
12 start: usize,
13 end: usize,
14 source: *Source,
15
16 pub const Id = enum {
10 id: union(enum) {
1711 Invalid,
1812 Eof,
1913 Nl,
2014 Identifier,
21 StringLiteral,
22 CharLiteral,
23 IntegerLiteral,
24 FloatLiteral,
15 StringLiteral: StrKind,
16 CharLiteral: StrKind,
17 IntegerLiteral: NumSuffix,
18 FloatLiteral: NumSuffix,
2519 Bang,
2620 BangEqual,
2721 Pipe,
......@@ -74,7 +68,10 @@ pub const Token = struct {
7468 MultiLineComment,
7569 Hash,
7670 HashHash,
77 };
71 },
72 start: usize,
73 end: usize,
74 source: *Source,
7875
7976 pub const NumSuffix = enum {
8077 None,
......@@ -85,6 +82,14 @@ pub const Token = struct {
8582 LL,
8683 LLU,
8784 };
85
86 pub const StrKind = enum {
87 None,
88 Wide,
89 Utf8,
90 Utf16,
91 Utf32,
92 };
8893};
8994
9095pub const Tokenizer = struct {
......@@ -102,6 +107,10 @@ pub const Tokenizer = struct {
102107 var state: enum {
103108 Start,
104109 Cr,
110 u,
111 u8,
112 U,
113 L,
105114 StringLiteral,
106115 CharLiteral,
107116 EscapeSequence,
......@@ -162,13 +171,23 @@ pub const Tokenizer = struct {
162171 result.start = self.index + 1;
163172 },
164173 '"' => {
174 result.id = .{ .StringLiteral = .None };
165175 state = .StringLiteral;
166 result.id = .StringLiteral;
167176 },
168177 '\'' => {
178 result.id = .{ .CharLiteral = .None };
169179 state = .CharLiteral;
170180 },
171 'a'...'z', 'A'...'Z', '_' => {
181 'u' => {
182 state = .u;
183 },
184 'U' => {
185 state = .U;
186 },
187 'L' => {
188 state = .L;
189 },
190 'a'...'t', 'v'...'z', 'A'...'K', 'M'...'T', 'V'...'Z', '_' => {
172191 state = .Identifier;
173192 result.id = .Identifier;
174193 },
......@@ -268,11 +287,9 @@ pub const Tokenizer = struct {
268287 },
269288 '0' => {
270289 state = .Zero;
271 result.id = .IntegerLiteral;
272290 },
273291 '1'...'9' => {
274292 state = .IntegerLiteral;
275 result.id = .IntegerLiteral;
276293 },
277294 else => {
278295 result.id = .Invalid;
......@@ -291,14 +308,63 @@ pub const Tokenizer = struct {
291308 break;
292309 },
293310 },
294 // TODO l"" u"" U"" u8""
311 .u => switch (c) {
312 '8' => {
313 state = .u8;
314 },
315 '\'' => {
316 result.id = .{ .CharLiteral = .Utf16 };
317 state = .CharLiteral;
318 },
319 '\"' => {
320 result.id = .{ .StringLiteral = .Utf16 };
321 state = .StringLiteral;
322 },
323 else => {
324 state = .Identifier;
325 },
326 },
327 .u8 => switch (c) {
328 '\"' => {
329 result.id = .{ .StringLiteral = .Utf8 };
330 state = .StringLiteral;
331 },
332 else => {
333 state = .Identifier;
334 },
335 },
336 .U => switch (c) {
337 '\'' => {
338 result.id = .{ .CharLiteral = .Utf32 };
339 state = .CharLiteral;
340 },
341 '\"' => {
342 result.id = .{ .StringLiteral = .Utf32 };
343 state = .StringLiteral;
344 },
345 else => {
346 state = .Identifier;
347 },
348 },
349 .L => switch (c) {
350 '\'' => {
351 result.id = .{ .CharLiteral = .Wide };
352 state = .CharLiteral;
353 },
354 '\"' => {
355 result.id = .{ .StringLiteral = .Wide };
356 state = .StringLiteral;
357 },
358 else => {
359 state = .Identifier;
360 },
361 },
295362 .StringLiteral => switch (c) {
296363 '\\' => {
297364 string = true;
298365 state = .EscapeSequence;
299366 },
300367 '"' => {
301 result.id = .StringLiteral;
302368 self.index += 1;
303369 break;
304370 },
......@@ -308,7 +374,6 @@ pub const Tokenizer = struct {
308374 },
309375 else => {},
310376 },
311 // TODO l'' u'' U''
312377 .CharLiteral => switch (c) {
313378 '\\' => {
314379 string = false;
......@@ -683,7 +748,7 @@ pub const Tokenizer = struct {
683748 state = .IntegerSuffixL;
684749 },
685750 else => {
686 result.id = .IntegerLiteral;
751 result.id = .{ .IntegerLiteral = .None };
687752 break;
688753 },
689754 },
......@@ -692,8 +757,7 @@ pub const Tokenizer = struct {
692757 state = .IntegerSuffixUL;
693758 },
694759 else => {
695 result.id = .IntegerLiteral;
696 result.num_suffix = .U;
760 result.id = .{ .IntegerLiteral = .U };
697761 break;
698762 },
699763 },
......@@ -702,40 +766,34 @@ pub const Tokenizer = struct {
702766 state = .IntegerSuffixLL;
703767 },
704768 'u', 'U' => {
705 result.id = .IntegerLiteral;
706 result.num_suffix = .LU;
769 result.id = .{ .IntegerLiteral = .LU };
707770 self.index += 1;
708771 break;
709772 },
710773 else => {
711 result.id = .IntegerLiteral;
712 result.num_suffix = .L;
774 result.id = .{ .IntegerLiteral = .L };
713775 break;
714776 },
715777 },
716778 .IntegerSuffixLL => switch (c) {
717779 'u', 'U' => {
718 result.id = .IntegerLiteral;
719 result.num_suffix = .LLU;
780 result.id = .{ .IntegerLiteral = .LLU };
720781 self.index += 1;
721782 break;
722783 },
723784 else => {
724 result.id = .IntegerLiteral;
725 result.num_suffix = .LL;
785 result.id = .{ .IntegerLiteral = .LL };
726786 break;
727787 },
728788 },
729789 .IntegerSuffixUL => switch (c) {
730790 'l', 'L' => {
731 result.id = .IntegerLiteral;
732 result.num_suffix = .LLU;
791 result.id = .{ .IntegerLiteral = .LLU };
733792 self.index += 1;
734793 break;
735794 },
736795 else => {
737 result.id = .IntegerLiteral;
738 result.num_suffix = .LU;
796 result.id = .{ .IntegerLiteral = .LU };
739797 break;
740798 },
741799 },
......@@ -782,19 +840,17 @@ pub const Tokenizer = struct {
782840 },
783841 .FloatSuffix => switch (c) {
784842 'l', 'L' => {
785 result.id = .FloatLiteral;
786 result.num_suffix = .L;
843 result.id = .{ .FloatLiteral = .L };
787844 self.index += 1;
788845 break;
789846 },
790847 'f', 'F' => {
791 result.id = .FloatLiteral;
792 result.num_suffix = .F;
848 result.id = .{ .FloatLiteral = .F };
793849 self.index += 1;
794850 break;
795851 },
796852 else => {
797 result.id = .FloatLiteral;
853 result.id = .{ .FloatLiteral = .None };
798854 break;
799855 },
800856 },
......@@ -802,7 +858,7 @@ pub const Tokenizer = struct {
802858 } else if (self.index == self.source.buffer.len) {
803859 switch (state) {
804860 .Start => {},
805 .Identifier => {
861 .u, .u8, .U, .L, .Identifier => {
806862 result.id = .Identifier;
807863 },
808864
......@@ -822,25 +878,19 @@ pub const Tokenizer = struct {
822878 .FloatExponentDigits,
823879 => result.id = .Invalid,
824880
825 .IntegerLiteralOct, .IntegerLiteralBinary, .IntegerLiteralHex, .IntegerLiteral, .IntegerSuffix, .Zero => result.id = .IntegerLiteral,
826 .IntegerSuffixU => {
827 result.id = .IntegerLiteral;
828 result.num_suffix = .U;
829 },
830 .IntegerSuffixL => {
831 result.id = .IntegerLiteral;
832 result.num_suffix = .L;
833 },
834 .IntegerSuffixLL => {
835 result.id = .IntegerLiteral;
836 result.num_suffix = .LL;
837 },
838 .IntegerSuffixUL => {
839 result.id = .IntegerLiteral;
840 result.num_suffix = .LU;
841 },
881 .IntegerLiteralOct,
882 .IntegerLiteralBinary,
883 .IntegerLiteralHex,
884 .IntegerLiteral,
885 .IntegerSuffix,
886 .Zero,
887 => result.id = .{ .IntegerLiteral = .None },
888 .IntegerSuffixU => result.id = .{ .IntegerLiteral = .U },
889 .IntegerSuffixL => result.id = .{ .IntegerLiteral = .L },
890 .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .LL },
891 .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .LU },
842892
843 .FloatSuffix => result.id = .FloatLiteral,
893 .FloatSuffix => result.id = .{ .FloatLiteral = .None },
844894 .Equal => result.id = .Equal,
845895 .Bang => result.id = .Bang,
846896 .Minus => result.id = .Minus,