authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 01:38:26+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:49+02:00
logf14a5287e92755f8d1f7f592caeed77bac940958
treed68cae243ea965320bf0282611270d85b4e5aab1
parent05acc0b0c14c19c9776633cd0d1ebbbbc30c3c47
signature Commit is signed but in an unrecognized format.

std-c tokenizer strings, floats and comments

1 files changed, 209 insertions(+), 12 deletions(-)

lib/std/c/tokenizer.zig+209-12
......@@ -104,6 +104,10 @@ pub const Tokenizer = struct {
104104 Cr,
105105 StringLiteral,
106106 CharLiteral,
107 EscapeSequence,
108 OctalEscape,
109 HexEscape,
110 UnicodeEscape,
107111 Identifier,
108112 Equal,
109113 Bang,
......@@ -117,9 +121,13 @@ pub const Tokenizer = struct {
117121 AngleBracketAngleBracketRight,
118122 Caret,
119123 Period,
124 Period2,
120125 Minus,
121126 Slash,
122127 Ampersand,
128 LineComment,
129 MultiLineComment,
130 MultiLineCommentAsterisk,
123131 Zero,
124132 IntegerLiteralOct,
125133 IntegerLiteralBinary,
......@@ -130,7 +138,14 @@ pub const Tokenizer = struct {
130138 IntegerSuffixL,
131139 IntegerSuffixLL,
132140 IntegerSuffixUL,
141 FloatFraction,
142 FloatFractionHex,
143 FloatExponent,
144 FloatExponentDigits,
145 FloatSuffix,
133146 } = .Start;
147 var string = false;
148 var counter: u32 = 0;
134149 while (self.index < self.source.buffer.len) : (self.index += 1) {
135150 const c = self.source.buffer[self.index];
136151 switch (state) {
......@@ -276,6 +291,89 @@ pub const Tokenizer = struct {
276291 break;
277292 },
278293 },
294 // TODO l"" u"" U"" u8""
295 .StringLiteral => switch (c) {
296 '\\' => {
297 string = true;
298 state = .EscapeSequence;
299 },
300 '"' => {
301 result.id = .StringLiteral;
302 self.index += 1;
303 break;
304 },
305 '\n', '\r' => {
306 result.id = .Invalid;
307 break;
308 },
309 else => {},
310 },
311 // TODO l'' u'' U''
312 .CharLiteral => switch (c) {
313 '\\' => {
314 string = false;
315 state = .EscapeSequence;
316 },
317 '\'', '\n' => {
318 result.id = .Invalid;
319 break;
320 },
321 else => {},
322 },
323 .EscapeSequence => switch (c) {
324 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v' => {},
325 '0'...'7' => {
326 counter = 1;
327 state = .OctalEscape;
328 },
329 'x' => {
330 state = .HexEscape;
331 },
332 'u' => {
333 counter = 4;
334 state = .OctalEscape;
335 },
336 'U' => {
337 counter = 8;
338 state = .OctalEscape;
339 },
340 else => {
341 result.id = .Invalid;
342 break;
343 },
344 },
345 .OctalEscape => switch (c) {
346 '0'...'7' => {
347 counter += 1;
348 if (counter == 3) {
349 state = if (string) .StringLiteral else .CharLiteral;
350 }
351 },
352 else => {
353 state = if (string) .StringLiteral else .CharLiteral;
354 },
355 },
356 .HexEscape => switch (c) {
357 '0'...'9', 'a'...'f', 'A'...'F' => {},
358 else => {
359 state = if (string) .StringLiteral else .CharLiteral;
360 },
361 },
362 .UnicodeEscape => switch (c) {
363 '0'...'9', 'a'...'f', 'A'...'F' => {
364 counter -= 1;
365 if (counter == 0) {
366 state = if (string) .StringLiteral else .CharLiteral;
367 }
368 },
369 else => {
370 if (counter != 0) {
371 result.id = .Invalid;
372 break;
373 }
374 state = if (string) .StringLiteral else .CharLiteral;
375 },
376 },
279377 .Identifier => switch (c) {
280378 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
281379 else => {
......@@ -328,7 +426,7 @@ pub const Tokenizer = struct {
328426 break;
329427 },
330428 else => {
331 result.id = .Id.Percent;
429 result.id = .Percent;
332430 break;
333431 },
334432 },
......@@ -468,7 +566,9 @@ pub const Tokenizer = struct {
468566 .Slash => switch (c) {
469567 '/' => {
470568 state = .LineComment;
471 result.id = .LineComment;
569 },
570 '*' => {
571 state = .MultiLineComment;
472572 },
473573 '=' => {
474574 result.id = .SlashEqual;
......@@ -496,6 +596,30 @@ pub const Tokenizer = struct {
496596 break;
497597 },
498598 },
599 .LineComment => switch (c) {
600 '\n' => {
601 result.id = .LineComment;
602 self.index += 1;
603 break;
604 },
605 else => {},
606 },
607 .MultiLineComment => switch (c) {
608 '*' => {
609 state = .MultiLineCommentAsterisk;
610 },
611 else => {},
612 },
613 .MultiLineCommentAsterisk => switch (c) {
614 '/' => {
615 result.id = .MultiLineComment;
616 self.index += 1;
617 break;
618 },
619 else => {
620 state = .MultiLineComment;
621 },
622 },
499623 .Zero => switch (c) {
500624 '0'...'9' => {
501625 state = .IntegerLiteralOct;
......@@ -531,7 +655,7 @@ pub const Tokenizer = struct {
531655 state = .FloatFractionHex;
532656 },
533657 'p', 'P' => {
534 state = .FloatExponentUnsignedHex;
658 state = .FloatExponent;
535659 },
536660 else => {
537661 state = .IntegerSuffix;
......@@ -544,7 +668,7 @@ pub const Tokenizer = struct {
544668 state = .FloatFraction;
545669 },
546670 'e', 'E' => {
547 state = .FloatExponentUnsigned;
671 state = .FloatExponent;
548672 },
549673 else => {
550674 state = .IntegerSuffix;
......@@ -615,18 +739,90 @@ pub const Tokenizer = struct {
615739 break;
616740 },
617741 },
742 .FloatFraction => switch (c) {
743 '0'...'9' => {},
744 'e', 'E' => {
745 state = .FloatExponent;
746 },
747 else => {
748 self.index -= 1;
749 state = .FloatSuffix;
750 },
751 },
752 .FloatFractionHex => switch (c) {
753 '0'...'9', 'a'...'f', 'A'...'F' => {},
754 'p', 'P' => {
755 state = .FloatExponent;
756 },
757 else => {
758 result.id = .Invalid;
759 break;
760 },
761 },
762 .FloatExponent => switch (c) {
763 '+', '-' => {
764 state = .FloatExponentDigits;
765 },
766 else => {
767 self.index -= 1;
768 state = .FloatExponentDigits;
769 },
770 },
771 .FloatExponentDigits => switch (c) {
772 '0'...'9' => {
773 counter += 1;
774 },
775 else => {
776 if (counter == 0) {
777 result.id = .Invalid;
778 break;
779 }
780 state = .FloatSuffix;
781 },
782 },
783 .FloatSuffix => switch (c) {
784 'l', 'L' => {
785 result.id = .FloatLiteral;
786 result.num_suffix = .L;
787 self.index += 1;
788 break;
789 },
790 'f', 'F' => {
791 result.id = .FloatLiteral;
792 result.num_suffix = .F;
793 self.index += 1;
794 break;
795 },
796 else => {
797 result.id = .FloatLiteral;
798 break;
799 },
800 },
618801 }
619802 } else if (self.index == self.source.buffer.len) {
620803 switch (state) {
804 .Start => {},
621805 .Identifier => {
622806 result.id = .Identifier;
623807 },
624 .IntegerLiteralOct,
625 .IntegerLiteralBinary,
626 .IntegerLiteralHex,
627 .IntegerLiteral,
628 .IntegerSuffix,
629 .Zero => result.id = .IntegerLiteral,
808
809 .Cr,
810 .Period2,
811 .StringLiteral,
812 .CharLiteral,
813 .EscapeSequence,
814 .OctalEscape,
815 .HexEscape,
816 .UnicodeEscape,
817 .MultiLineComment,
818 .MultiLineCommentAsterisk,
819 .FloatFraction,
820 .FloatFractionHex,
821 .FloatExponent,
822 .FloatExponentDigits,
823 => result.id = .Invalid,
824
825 .IntegerLiteralOct, .IntegerLiteralBinary, .IntegerLiteralHex, .IntegerLiteral, .IntegerSuffix, .Zero => result.id = .IntegerLiteral,
630826 .IntegerSuffixU => {
631827 result.id = .IntegerLiteral;
632828 result.num_suffix = .U;
......@@ -641,16 +837,16 @@ pub const Tokenizer = struct {
641837 },
642838 .IntegerSuffixUL => {
643839 result.id = .IntegerLiteral;
644 result.num_suffix = .Ul;
840 result.num_suffix = .LU;
645841 },
646842
843 .FloatSuffix => result.id = .FloatLiteral,
647844 .Equal => result.id = .Equal,
648845 .Bang => result.id = .Bang,
649846 .Minus => result.id = .Minus,
650847 .Slash => result.id = .Slash,
651848 .Ampersand => result.id = .Ampersand,
652849 .Period => result.id = .Period,
653 .Period2 => result.id = .Invalid,
654850 .Pipe => result.id = .Pipe,
655851 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,
656852 .AngleBracketRight => result.id = .AngleBracketRight,
......@@ -660,6 +856,7 @@ pub const Tokenizer = struct {
660856 .Percent => result.id = .Percent,
661857 .Caret => result.id = .Caret,
662858 .Asterisk => result.id = .Asterisk,
859 .LineComment => result.id = .LineComment,
663860 }
664861 }
665862