authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-04 01:38:26+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:49+02:00
logf14a5287e92755f8d1f7f592caeed77bac940958
treed68cae243ea965320bf0282611270d85b4e5aab1
parent05acc0b0c14c19c9776633cd0d1ebbbbc30c3c47
signature Commit is signed but in an unrecognized format.

std-c tokenizer strings, floats and comments

1 files changed, 209 insertions(+), 12 deletions(-)

lib/std/c/tokenizer.zig+209-12
...@@ -104,6 +104,10 @@ pub const Tokenizer = struct {...@@ -104,6 +104,10 @@ pub const Tokenizer = struct {
104 Cr,104 Cr,
105 StringLiteral,105 StringLiteral,
106 CharLiteral,106 CharLiteral,
107 EscapeSequence,
108 OctalEscape,
109 HexEscape,
110 UnicodeEscape,
107 Identifier,111 Identifier,
108 Equal,112 Equal,
109 Bang,113 Bang,
...@@ -117,9 +121,13 @@ pub const Tokenizer = struct {...@@ -117,9 +121,13 @@ pub const Tokenizer = struct {
117 AngleBracketAngleBracketRight,121 AngleBracketAngleBracketRight,
118 Caret,122 Caret,
119 Period,123 Period,
124 Period2,
120 Minus,125 Minus,
121 Slash,126 Slash,
122 Ampersand,127 Ampersand,
128 LineComment,
129 MultiLineComment,
130 MultiLineCommentAsterisk,
123 Zero,131 Zero,
124 IntegerLiteralOct,132 IntegerLiteralOct,
125 IntegerLiteralBinary,133 IntegerLiteralBinary,
...@@ -130,7 +138,14 @@ pub const Tokenizer = struct {...@@ -130,7 +138,14 @@ pub const Tokenizer = struct {
130 IntegerSuffixL,138 IntegerSuffixL,
131 IntegerSuffixLL,139 IntegerSuffixLL,
132 IntegerSuffixUL,140 IntegerSuffixUL,
141 FloatFraction,
142 FloatFractionHex,
143 FloatExponent,
144 FloatExponentDigits,
145 FloatSuffix,
133 } = .Start;146 } = .Start;
147 var string = false;
148 var counter: u32 = 0;
134 while (self.index < self.source.buffer.len) : (self.index += 1) {149 while (self.index < self.source.buffer.len) : (self.index += 1) {
135 const c = self.source.buffer[self.index];150 const c = self.source.buffer[self.index];
136 switch (state) {151 switch (state) {
...@@ -276,6 +291,89 @@ pub const Tokenizer = struct {...@@ -276,6 +291,89 @@ pub const Tokenizer = struct {
276 break;291 break;
277 },292 },
278 },293 },
294 // TODO l"" u"" U"" u8""
295 .StringLiteral => switch (c) {
296 '\\' => {
297 string = true;
298 state = .EscapeSequence;
299 },
300 '"' => {
301 result.id = .StringLiteral;
302 self.index += 1;
303 break;
304 },
305 '\n', '\r' => {
306 result.id = .Invalid;
307 break;
308 },
309 else => {},
310 },
311 // TODO l'' u'' U''
312 .CharLiteral => switch (c) {
313 '\\' => {
314 string = false;
315 state = .EscapeSequence;
316 },
317 '\'', '\n' => {
318 result.id = .Invalid;
319 break;
320 },
321 else => {},
322 },
323 .EscapeSequence => switch (c) {
324 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v' => {},
325 '0'...'7' => {
326 counter = 1;
327 state = .OctalEscape;
328 },
329 'x' => {
330 state = .HexEscape;
331 },
332 'u' => {
333 counter = 4;
334 state = .OctalEscape;
335 },
336 'U' => {
337 counter = 8;
338 state = .OctalEscape;
339 },
340 else => {
341 result.id = .Invalid;
342 break;
343 },
344 },
345 .OctalEscape => switch (c) {
346 '0'...'7' => {
347 counter += 1;
348 if (counter == 3) {
349 state = if (string) .StringLiteral else .CharLiteral;
350 }
351 },
352 else => {
353 state = if (string) .StringLiteral else .CharLiteral;
354 },
355 },
356 .HexEscape => switch (c) {
357 '0'...'9', 'a'...'f', 'A'...'F' => {},
358 else => {
359 state = if (string) .StringLiteral else .CharLiteral;
360 },
361 },
362 .UnicodeEscape => switch (c) {
363 '0'...'9', 'a'...'f', 'A'...'F' => {
364 counter -= 1;
365 if (counter == 0) {
366 state = if (string) .StringLiteral else .CharLiteral;
367 }
368 },
369 else => {
370 if (counter != 0) {
371 result.id = .Invalid;
372 break;
373 }
374 state = if (string) .StringLiteral else .CharLiteral;
375 },
376 },
279 .Identifier => switch (c) {377 .Identifier => switch (c) {
280 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},378 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
281 else => {379 else => {
...@@ -328,7 +426,7 @@ pub const Tokenizer = struct {...@@ -328,7 +426,7 @@ pub const Tokenizer = struct {
328 break;426 break;
329 },427 },
330 else => {428 else => {
331 result.id = .Id.Percent;429 result.id = .Percent;
332 break;430 break;
333 },431 },
334 },432 },
...@@ -468,7 +566,9 @@ pub const Tokenizer = struct {...@@ -468,7 +566,9 @@ pub const Tokenizer = struct {
468 .Slash => switch (c) {566 .Slash => switch (c) {
469 '/' => {567 '/' => {
470 state = .LineComment;568 state = .LineComment;
471 result.id = .LineComment;569 },
570 '*' => {
571 state = .MultiLineComment;
472 },572 },
473 '=' => {573 '=' => {
474 result.id = .SlashEqual;574 result.id = .SlashEqual;
...@@ -496,6 +596,30 @@ pub const Tokenizer = struct {...@@ -496,6 +596,30 @@ pub const Tokenizer = struct {
496 break;596 break;
497 },597 },
498 },598 },
599 .LineComment => switch (c) {
600 '\n' => {
601 result.id = .LineComment;
602 self.index += 1;
603 break;
604 },
605 else => {},
606 },
607 .MultiLineComment => switch (c) {
608 '*' => {
609 state = .MultiLineCommentAsterisk;
610 },
611 else => {},
612 },
613 .MultiLineCommentAsterisk => switch (c) {
614 '/' => {
615 result.id = .MultiLineComment;
616 self.index += 1;
617 break;
618 },
619 else => {
620 state = .MultiLineComment;
621 },
622 },
499 .Zero => switch (c) {623 .Zero => switch (c) {
500 '0'...'9' => {624 '0'...'9' => {
501 state = .IntegerLiteralOct;625 state = .IntegerLiteralOct;
...@@ -531,7 +655,7 @@ pub const Tokenizer = struct {...@@ -531,7 +655,7 @@ pub const Tokenizer = struct {
531 state = .FloatFractionHex;655 state = .FloatFractionHex;
532 },656 },
533 'p', 'P' => {657 'p', 'P' => {
534 state = .FloatExponentUnsignedHex;658 state = .FloatExponent;
535 },659 },
536 else => {660 else => {
537 state = .IntegerSuffix;661 state = .IntegerSuffix;
...@@ -544,7 +668,7 @@ pub const Tokenizer = struct {...@@ -544,7 +668,7 @@ pub const Tokenizer = struct {
544 state = .FloatFraction;668 state = .FloatFraction;
545 },669 },
546 'e', 'E' => {670 'e', 'E' => {
547 state = .FloatExponentUnsigned;671 state = .FloatExponent;
548 },672 },
549 else => {673 else => {
550 state = .IntegerSuffix;674 state = .IntegerSuffix;
...@@ -615,18 +739,90 @@ pub const Tokenizer = struct {...@@ -615,18 +739,90 @@ pub const Tokenizer = struct {
615 break;739 break;
616 },740 },
617 },741 },
742 .FloatFraction => switch (c) {
743 '0'...'9' => {},
744 'e', 'E' => {
745 state = .FloatExponent;
746 },
747 else => {
748 self.index -= 1;
749 state = .FloatSuffix;
750 },
751 },
752 .FloatFractionHex => switch (c) {
753 '0'...'9', 'a'...'f', 'A'...'F' => {},
754 'p', 'P' => {
755 state = .FloatExponent;
756 },
757 else => {
758 result.id = .Invalid;
759 break;
760 },
761 },
762 .FloatExponent => switch (c) {
763 '+', '-' => {
764 state = .FloatExponentDigits;
765 },
766 else => {
767 self.index -= 1;
768 state = .FloatExponentDigits;
769 },
770 },
771 .FloatExponentDigits => switch (c) {
772 '0'...'9' => {
773 counter += 1;
774 },
775 else => {
776 if (counter == 0) {
777 result.id = .Invalid;
778 break;
779 }
780 state = .FloatSuffix;
781 },
782 },
783 .FloatSuffix => switch (c) {
784 'l', 'L' => {
785 result.id = .FloatLiteral;
786 result.num_suffix = .L;
787 self.index += 1;
788 break;
789 },
790 'f', 'F' => {
791 result.id = .FloatLiteral;
792 result.num_suffix = .F;
793 self.index += 1;
794 break;
795 },
796 else => {
797 result.id = .FloatLiteral;
798 break;
799 },
800 },
618 }801 }
619 } else if (self.index == self.source.buffer.len) {802 } else if (self.index == self.source.buffer.len) {
620 switch (state) {803 switch (state) {
804 .Start => {},
621 .Identifier => {805 .Identifier => {
622 result.id = .Identifier;806 result.id = .Identifier;
623 },807 },
624 .IntegerLiteralOct,808
625 .IntegerLiteralBinary,809 .Cr,
626 .IntegerLiteralHex,810 .Period2,
627 .IntegerLiteral,811 .StringLiteral,
628 .IntegerSuffix,812 .CharLiteral,
629 .Zero => result.id = .IntegerLiteral,813 .EscapeSequence,
814 .OctalEscape,
815 .HexEscape,
816 .UnicodeEscape,
817 .MultiLineComment,
818 .MultiLineCommentAsterisk,
819 .FloatFraction,
820 .FloatFractionHex,
821 .FloatExponent,
822 .FloatExponentDigits,
823 => result.id = .Invalid,
824
825 .IntegerLiteralOct, .IntegerLiteralBinary, .IntegerLiteralHex, .IntegerLiteral, .IntegerSuffix, .Zero => result.id = .IntegerLiteral,
630 .IntegerSuffixU => {826 .IntegerSuffixU => {
631 result.id = .IntegerLiteral;827 result.id = .IntegerLiteral;
632 result.num_suffix = .U;828 result.num_suffix = .U;
...@@ -641,16 +837,16 @@ pub const Tokenizer = struct {...@@ -641,16 +837,16 @@ pub const Tokenizer = struct {
641 },837 },
642 .IntegerSuffixUL => {838 .IntegerSuffixUL => {
643 result.id = .IntegerLiteral;839 result.id = .IntegerLiteral;
644 result.num_suffix = .Ul;840 result.num_suffix = .LU;
645 },841 },
646842
843 .FloatSuffix => result.id = .FloatLiteral,
647 .Equal => result.id = .Equal,844 .Equal => result.id = .Equal,
648 .Bang => result.id = .Bang,845 .Bang => result.id = .Bang,
649 .Minus => result.id = .Minus,846 .Minus => result.id = .Minus,
650 .Slash => result.id = .Slash,847 .Slash => result.id = .Slash,
651 .Ampersand => result.id = .Ampersand,848 .Ampersand => result.id = .Ampersand,
652 .Period => result.id = .Period,849 .Period => result.id = .Period,
653 .Period2 => result.id = .Invalid,
654 .Pipe => result.id = .Pipe,850 .Pipe => result.id = .Pipe,
655 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,851 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,
656 .AngleBracketRight => result.id = .AngleBracketRight,852 .AngleBracketRight => result.id = .AngleBracketRight,
...@@ -660,6 +856,7 @@ pub const Tokenizer = struct {...@@ -660,6 +856,7 @@ pub const Tokenizer = struct {
660 .Percent => result.id = .Percent,856 .Percent => result.id = .Percent,
661 .Caret => result.id = .Caret,857 .Caret => result.id = .Caret,
662 .Asterisk => result.id = .Asterisk,858 .Asterisk => result.id = .Asterisk,
859 .LineComment => result.id = .LineComment,
663 }860 }
664 }861 }
665862