authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 02:00:29+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:49+02:00
log26bf410b061b9d6d18e4945417ddec62d7486e9c
tree3ba9bb931511c9bd03d69daf0388aab34b993ded
parentf14a5287e92755f8d1f7f592caeed77bac940958
signature Commit is signed but in an unrecognized format.

std-c finish tokenizer

1 files changed, 108 insertions(+), 58 deletions(-)

lib/std/c/tokenizer.zig+108-58
...@@ -7,21 +7,15 @@ pub const Source = struct {...@@ -7,21 +7,15 @@ pub const Source = struct {
7};7};
88
9pub const Token = struct {9pub const Token = struct {
10 id: Id,10 id: union(enum) {
11 num_suffix: NumSuffix = .None,
12 start: usize,
13 end: usize,
14 source: *Source,
15
16 pub const Id = enum {
17 Invalid,11 Invalid,
18 Eof,12 Eof,
19 Nl,13 Nl,
20 Identifier,14 Identifier,
21 StringLiteral,15 StringLiteral: StrKind,
22 CharLiteral,16 CharLiteral: StrKind,
23 IntegerLiteral,17 IntegerLiteral: NumSuffix,
24 FloatLiteral,18 FloatLiteral: NumSuffix,
25 Bang,19 Bang,
26 BangEqual,20 BangEqual,
27 Pipe,21 Pipe,
...@@ -74,7 +68,10 @@ pub const Token = struct {...@@ -74,7 +68,10 @@ pub const Token = struct {
74 MultiLineComment,68 MultiLineComment,
75 Hash,69 Hash,
76 HashHash,70 HashHash,
77 };71 },
72 start: usize,
73 end: usize,
74 source: *Source,
7875
79 pub const NumSuffix = enum {76 pub const NumSuffix = enum {
80 None,77 None,
...@@ -85,6 +82,14 @@ pub const Token = struct {...@@ -85,6 +82,14 @@ pub const Token = struct {
85 LL,82 LL,
86 LLU,83 LLU,
87 };84 };
85
86 pub const StrKind = enum {
87 None,
88 Wide,
89 Utf8,
90 Utf16,
91 Utf32,
92 };
88};93};
8994
90pub const Tokenizer = struct {95pub const Tokenizer = struct {
...@@ -102,6 +107,10 @@ pub const Tokenizer = struct {...@@ -102,6 +107,10 @@ pub const Tokenizer = struct {
102 var state: enum {107 var state: enum {
103 Start,108 Start,
104 Cr,109 Cr,
110 u,
111 u8,
112 U,
113 L,
105 StringLiteral,114 StringLiteral,
106 CharLiteral,115 CharLiteral,
107 EscapeSequence,116 EscapeSequence,
...@@ -162,13 +171,23 @@ pub const Tokenizer = struct {...@@ -162,13 +171,23 @@ pub const Tokenizer = struct {
162 result.start = self.index + 1;171 result.start = self.index + 1;
163 },172 },
164 '"' => {173 '"' => {
174 result.id = .{ .StringLiteral = .None };
165 state = .StringLiteral;175 state = .StringLiteral;
166 result.id = .StringLiteral;
167 },176 },
168 '\'' => {177 '\'' => {
178 result.id = .{ .CharLiteral = .None };
169 state = .CharLiteral;179 state = .CharLiteral;
170 },180 },
171 'a'...'z', 'A'...'Z', '_' => {181 'u' => {
182 state = .u;
183 },
184 'U' => {
185 state = .U;
186 },
187 'L' => {
188 state = .L;
189 },
190 'a'...'t', 'v'...'z', 'A'...'K', 'M'...'T', 'V'...'Z', '_' => {
172 state = .Identifier;191 state = .Identifier;
173 result.id = .Identifier;192 result.id = .Identifier;
174 },193 },
...@@ -268,11 +287,9 @@ pub const Tokenizer = struct {...@@ -268,11 +287,9 @@ pub const Tokenizer = struct {
268 },287 },
269 '0' => {288 '0' => {
270 state = .Zero;289 state = .Zero;
271 result.id = .IntegerLiteral;
272 },290 },
273 '1'...'9' => {291 '1'...'9' => {
274 state = .IntegerLiteral;292 state = .IntegerLiteral;
275 result.id = .IntegerLiteral;
276 },293 },
277 else => {294 else => {
278 result.id = .Invalid;295 result.id = .Invalid;
...@@ -291,14 +308,63 @@ pub const Tokenizer = struct {...@@ -291,14 +308,63 @@ pub const Tokenizer = struct {
291 break;308 break;
292 },309 },
293 },310 },
294 // TODO l"" u"" U"" u8""311 .u => switch (c) {
312 '8' => {
313 state = .u8;
314 },
315 '\'' => {
316 result.id = .{ .CharLiteral = .Utf16 };
317 state = .CharLiteral;
318 },
319 '\"' => {
320 result.id = .{ .StringLiteral = .Utf16 };
321 state = .StringLiteral;
322 },
323 else => {
324 state = .Identifier;
325 },
326 },
327 .u8 => switch (c) {
328 '\"' => {
329 result.id = .{ .StringLiteral = .Utf8 };
330 state = .StringLiteral;
331 },
332 else => {
333 state = .Identifier;
334 },
335 },
336 .U => switch (c) {
337 '\'' => {
338 result.id = .{ .CharLiteral = .Utf32 };
339 state = .CharLiteral;
340 },
341 '\"' => {
342 result.id = .{ .StringLiteral = .Utf32 };
343 state = .StringLiteral;
344 },
345 else => {
346 state = .Identifier;
347 },
348 },
349 .L => switch (c) {
350 '\'' => {
351 result.id = .{ .CharLiteral = .Wide };
352 state = .CharLiteral;
353 },
354 '\"' => {
355 result.id = .{ .StringLiteral = .Wide };
356 state = .StringLiteral;
357 },
358 else => {
359 state = .Identifier;
360 },
361 },
295 .StringLiteral => switch (c) {362 .StringLiteral => switch (c) {
296 '\\' => {363 '\\' => {
297 string = true;364 string = true;
298 state = .EscapeSequence;365 state = .EscapeSequence;
299 },366 },
300 '"' => {367 '"' => {
301 result.id = .StringLiteral;
302 self.index += 1;368 self.index += 1;
303 break;369 break;
304 },370 },
...@@ -308,7 +374,6 @@ pub const Tokenizer = struct {...@@ -308,7 +374,6 @@ pub const Tokenizer = struct {
308 },374 },
309 else => {},375 else => {},
310 },376 },
311 // TODO l'' u'' U''
312 .CharLiteral => switch (c) {377 .CharLiteral => switch (c) {
313 '\\' => {378 '\\' => {
314 string = false;379 string = false;
...@@ -683,7 +748,7 @@ pub const Tokenizer = struct {...@@ -683,7 +748,7 @@ pub const Tokenizer = struct {
683 state = .IntegerSuffixL;748 state = .IntegerSuffixL;
684 },749 },
685 else => {750 else => {
686 result.id = .IntegerLiteral;751 result.id = .{ .IntegerLiteral = .None };
687 break;752 break;
688 },753 },
689 },754 },
...@@ -692,8 +757,7 @@ pub const Tokenizer = struct {...@@ -692,8 +757,7 @@ pub const Tokenizer = struct {
692 state = .IntegerSuffixUL;757 state = .IntegerSuffixUL;
693 },758 },
694 else => {759 else => {
695 result.id = .IntegerLiteral;760 result.id = .{ .IntegerLiteral = .U };
696 result.num_suffix = .U;
697 break;761 break;
698 },762 },
699 },763 },
...@@ -702,40 +766,34 @@ pub const Tokenizer = struct {...@@ -702,40 +766,34 @@ pub const Tokenizer = struct {
702 state = .IntegerSuffixLL;766 state = .IntegerSuffixLL;
703 },767 },
704 'u', 'U' => {768 'u', 'U' => {
705 result.id = .IntegerLiteral;769 result.id = .{ .IntegerLiteral = .LU };
706 result.num_suffix = .LU;
707 self.index += 1;770 self.index += 1;
708 break;771 break;
709 },772 },
710 else => {773 else => {
711 result.id = .IntegerLiteral;774 result.id = .{ .IntegerLiteral = .L };
712 result.num_suffix = .L;
713 break;775 break;
714 },776 },
715 },777 },
716 .IntegerSuffixLL => switch (c) {778 .IntegerSuffixLL => switch (c) {
717 'u', 'U' => {779 'u', 'U' => {
718 result.id = .IntegerLiteral;780 result.id = .{ .IntegerLiteral = .LLU };
719 result.num_suffix = .LLU;
720 self.index += 1;781 self.index += 1;
721 break;782 break;
722 },783 },
723 else => {784 else => {
724 result.id = .IntegerLiteral;785 result.id = .{ .IntegerLiteral = .LL };
725 result.num_suffix = .LL;
726 break;786 break;
727 },787 },
728 },788 },
729 .IntegerSuffixUL => switch (c) {789 .IntegerSuffixUL => switch (c) {
730 'l', 'L' => {790 'l', 'L' => {
731 result.id = .IntegerLiteral;791 result.id = .{ .IntegerLiteral = .LLU };
732 result.num_suffix = .LLU;
733 self.index += 1;792 self.index += 1;
734 break;793 break;
735 },794 },
736 else => {795 else => {
737 result.id = .IntegerLiteral;796 result.id = .{ .IntegerLiteral = .LU };
738 result.num_suffix = .LU;
739 break;797 break;
740 },798 },
741 },799 },
...@@ -782,19 +840,17 @@ pub const Tokenizer = struct {...@@ -782,19 +840,17 @@ pub const Tokenizer = struct {
782 },840 },
783 .FloatSuffix => switch (c) {841 .FloatSuffix => switch (c) {
784 'l', 'L' => {842 'l', 'L' => {
785 result.id = .FloatLiteral;843 result.id = .{ .FloatLiteral = .L };
786 result.num_suffix = .L;
787 self.index += 1;844 self.index += 1;
788 break;845 break;
789 },846 },
790 'f', 'F' => {847 'f', 'F' => {
791 result.id = .FloatLiteral;848 result.id = .{ .FloatLiteral = .F };
792 result.num_suffix = .F;
793 self.index += 1;849 self.index += 1;
794 break;850 break;
795 },851 },
796 else => {852 else => {
797 result.id = .FloatLiteral;853 result.id = .{ .FloatLiteral = .None };
798 break;854 break;
799 },855 },
800 },856 },
...@@ -802,7 +858,7 @@ pub const Tokenizer = struct {...@@ -802,7 +858,7 @@ pub const Tokenizer = struct {
802 } else if (self.index == self.source.buffer.len) {858 } else if (self.index == self.source.buffer.len) {
803 switch (state) {859 switch (state) {
804 .Start => {},860 .Start => {},
805 .Identifier => {861 .u, .u8, .U, .L, .Identifier => {
806 result.id = .Identifier;862 result.id = .Identifier;
807 },863 },
808864
...@@ -822,25 +878,19 @@ pub const Tokenizer = struct {...@@ -822,25 +878,19 @@ pub const Tokenizer = struct {
822 .FloatExponentDigits,878 .FloatExponentDigits,
823 => result.id = .Invalid,879 => result.id = .Invalid,
824880
825 .IntegerLiteralOct, .IntegerLiteralBinary, .IntegerLiteralHex, .IntegerLiteral, .IntegerSuffix, .Zero => result.id = .IntegerLiteral,881 .IntegerLiteralOct,
826 .IntegerSuffixU => {882 .IntegerLiteralBinary,
827 result.id = .IntegerLiteral;883 .IntegerLiteralHex,
828 result.num_suffix = .U;884 .IntegerLiteral,
829 },885 .IntegerSuffix,
830 .IntegerSuffixL => {886 .Zero,
831 result.id = .IntegerLiteral;887 => result.id = .{ .IntegerLiteral = .None },
832 result.num_suffix = .L;888 .IntegerSuffixU => result.id = .{ .IntegerLiteral = .U },
833 },889 .IntegerSuffixL => result.id = .{ .IntegerLiteral = .L },
834 .IntegerSuffixLL => {890 .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .LL },
835 result.id = .IntegerLiteral;891 .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .LU },
836 result.num_suffix = .LL;
837 },
838 .IntegerSuffixUL => {
839 result.id = .IntegerLiteral;
840 result.num_suffix = .LU;
841 },
842892
843 .FloatSuffix => result.id = .FloatLiteral,893 .FloatSuffix => result.id = .{ .FloatLiteral = .None },
844 .Equal => result.id = .Equal,894 .Equal => result.id = .Equal,
845 .Bang => result.id = .Bang,895 .Bang => result.id = .Bang,
846 .Minus => result.id = .Minus,896 .Minus => result.id = .Minus,