authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 19:28:14+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:51+02:00
log795a5039995a1a23ba00d15488565f1a79d3f25b
treed4ad93a06dbe3b88c7e60615f870cf229b091ba1
parentf934f9b41938ee6208d7cdd8a26687bffbe171cf
signature Commit is signed but in an unrecognized format.

std-c tokenizer always add newline token

2 files changed, 117 insertions(+), 83 deletions(-)

lib/std/c/parse.zig+21-17
......@@ -797,38 +797,42 @@ const Parser = struct {
797797
798798 fn eatToken(parser: *Parser, id: @TagType(Token.Id)) ?TokenIndex {
799799 while (true) {
800 const next_tok = parser.it.next() orelse return null;
801 if (next_tok.id != .LineComment and next_tok.id != .MultiLineComment) {
802 if (next_tok.id == id) {
800 switch (parser.it.next() orelse return null) {
801 .LineComment, .MultiLineComment, .Nl => continue,
802 else => |next_id| if (next_id == id) {
803803 return parser.it.index;
804 }
805 _ = parser.it.prev();
806 return null;
804 } else {
805 _ = parser.it.prev();
806 return null;
807 },
807808 }
808809 }
809810 }
810811
811812 fn expectToken(parser: *Parser, id: @TagType(Token.Id)) Error!TokenIndex {
812813 while (true) {
813 const next_tok = parser.it.next() orelse return error.ParseError;
814 if (next_tok.id != .LineComment and next_tok.id != .MultiLineComment) {
815 if (next_tok.id != id) {
816 try parser.tree.errors.push(.{
814 switch (parser.it.next() orelse return null) {
815 .LineComment, .MultiLineComment, .Nl => continue,
816 else => |next_id| if (next_id != id) {
817 return parser.err(.{
817818 .ExpectedToken = .{ .token = parser.it.index, .expected_id = id },
818819 });
819 return error.ParseError;
820 }
821 return parser.it.index;
820 } else {
821 return parser.it.index;
822 },
822823 }
823824 }
824825 }
825826
826827 fn putBackToken(parser: *Parser, putting_back: TokenIndex) void {
827828 while (true) {
828 const prev_tok = parser.it.prev() orelse return;
829 if (prev_tok.id == .LineComment or prev_tok.id == .MultiLineComment) continue;
830 assert(parser.it.list.at(putting_back) == prev_tok);
831 return;
829 switch (parser.it.next() orelse return null) {
830 .LineComment, .MultiLineComment, .Nl => continue,
831 else => |next_id| {
832 assert(parser.it.list.at(putting_back) == prev_tok);
833 return;
834 },
835 }
832836 }
833837 }
834838
lib/std/c/tokenizer.zig+96-66
......@@ -449,20 +449,12 @@ pub const Tokenizer = struct {
449449 switch (state) {
450450 .Start => switch (c) {
451451 '\n' => {
452 if (!self.pp_directive) {
453 result.start = self.index + 1;
454 continue;
455 }
456452 self.pp_directive = false;
457453 result.id = .Nl;
458454 self.index += 1;
459455 break;
460456 },
461457 '\r' => {
462 if (!self.pp_directive) {
463 result.start = self.index + 1;
464 continue;
465 }
466458 state = .Cr;
467459 },
468460 '"' => {
......@@ -612,11 +604,14 @@ pub const Tokenizer = struct {
612604 },
613605 .BackSlash => switch (c) {
614606 '\n' => {
615 state = .Start;
607 state = if (string) .AfterStringLiteral else .Start;
616608 },
617609 '\r' => {
618610 state = .BackSlashCr;
619611 },
612 '\t', '\x0B', '\x0C', ' ' => {
613 // TODO warn
614 },
620615 else => {
621616 result.id = .Invalid;
622617 break;
......@@ -624,7 +619,7 @@ pub const Tokenizer = struct {
624619 },
625620 .BackSlashCr => switch (c) {
626621 '\n' => {
627 state = .Start;
622 state = if (string) .AfterStringLiteral else .Start;
628623 },
629624 else => {
630625 result.id = .Invalid;
......@@ -700,7 +695,14 @@ pub const Tokenizer = struct {
700695 '"' => {
701696 state = .StringLiteral;
702697 },
703 '\n'...'\r', ' ' => {},
698 '\\' => {
699 state = .BackSlash;
700 },
701 '\n', '\r' => {
702 if (self.pp_directive)
703 break;
704 },
705 '\t', '\x0B', '\x0C', ' ' => {},
704706 else => {
705707 break;
706708 },
......@@ -1314,60 +1316,64 @@ test "operators" {
13141316 \\ , & && &= ? < <= <<
13151317 \\ <<= > >= >> >>= ~ # ##
13161318 \\
1317 ,
1318 &[_]Token.Id{
1319 .Bang,
1320 .BangEqual,
1321 .Pipe,
1322 .PipePipe,
1323 .PipeEqual,
1324 .Equal,
1325 .EqualEqual,
1326 .LParen,
1327 .RParen,
1328 .LBrace,
1329 .RBrace,
1330 .LBracket,
1331 .RBracket,
1332 .Period,
1333 .Period,
1334 .Period,
1335 .Ellipsis,
1336 .Caret,
1337 .CaretEqual,
1338 .Plus,
1339 .PlusPlus,
1340 .PlusEqual,
1341 .Minus,
1342 .MinusMinus,
1343 .MinusEqual,
1344 .Asterisk,
1345 .AsteriskEqual,
1346 .Percent,
1347 .PercentEqual,
1348 .Arrow,
1349 .Colon,
1350 .Semicolon,
1351 .Slash,
1352 .SlashEqual,
1353 .Comma,
1354 .Ampersand,
1355 .AmpersandAmpersand,
1356 .AmpersandEqual,
1357 .QuestionMark,
1358 .AngleBracketLeft,
1359 .AngleBracketLeftEqual,
1360 .AngleBracketAngleBracketLeft,
1361 .AngleBracketAngleBracketLeftEqual,
1362 .AngleBracketRight,
1363 .AngleBracketRightEqual,
1364 .AngleBracketAngleBracketRight,
1365 .AngleBracketAngleBracketRightEqual,
1366 .Tilde,
1367 .Hash,
1368 .HashHash,
1369 },
1370 );
1319 , &[_]Token.Id{
1320 .Bang,
1321 .BangEqual,
1322 .Pipe,
1323 .PipePipe,
1324 .PipeEqual,
1325 .Equal,
1326 .EqualEqual,
1327 .Nl,
1328 .LParen,
1329 .RParen,
1330 .LBrace,
1331 .RBrace,
1332 .LBracket,
1333 .RBracket,
1334 .Period,
1335 .Period,
1336 .Period,
1337 .Ellipsis,
1338 .Nl,
1339 .Caret,
1340 .CaretEqual,
1341 .Plus,
1342 .PlusPlus,
1343 .PlusEqual,
1344 .Minus,
1345 .MinusMinus,
1346 .MinusEqual,
1347 .Nl,
1348 .Asterisk,
1349 .AsteriskEqual,
1350 .Percent,
1351 .PercentEqual,
1352 .Arrow,
1353 .Colon,
1354 .Semicolon,
1355 .Slash,
1356 .SlashEqual,
1357 .Nl,
1358 .Comma,
1359 .Ampersand,
1360 .AmpersandAmpersand,
1361 .AmpersandEqual,
1362 .QuestionMark,
1363 .AngleBracketLeft,
1364 .AngleBracketLeftEqual,
1365 .AngleBracketAngleBracketLeft,
1366 .Nl,
1367 .AngleBracketAngleBracketLeftEqual,
1368 .AngleBracketRight,
1369 .AngleBracketRightEqual,
1370 .AngleBracketAngleBracketRight,
1371 .AngleBracketAngleBracketRightEqual,
1372 .Tilde,
1373 .Hash,
1374 .HashHash,
1375 .Nl,
1376 });
13711377}
13721378
13731379test "keywords" {
......@@ -1388,6 +1394,7 @@ test "keywords" {
13881394 .Keyword_continue,
13891395 .Keyword_default,
13901396 .Keyword_do,
1397 .Nl,
13911398 .Keyword_double,
13921399 .Keyword_else,
13931400 .Keyword_enum,
......@@ -1397,6 +1404,7 @@ test "keywords" {
13971404 .Keyword_goto,
13981405 .Keyword_if,
13991406 .Keyword_int,
1407 .Nl,
14001408 .Keyword_long,
14011409 .Keyword_register,
14021410 .Keyword_return,
......@@ -1404,6 +1412,7 @@ test "keywords" {
14041412 .Keyword_signed,
14051413 .Keyword_sizeof,
14061414 .Keyword_static,
1415 .Nl,
14071416 .Keyword_struct,
14081417 .Keyword_switch,
14091418 .Keyword_typedef,
......@@ -1411,6 +1420,7 @@ test "keywords" {
14111420 .Keyword_unsigned,
14121421 .Keyword_void,
14131422 .Keyword_volatile,
1423 .Nl,
14141424 .Keyword_while,
14151425 .Keyword_bool,
14161426 .Keyword_complex,
......@@ -1418,12 +1428,14 @@ test "keywords" {
14181428 .Keyword_inline,
14191429 .Keyword_restrict,
14201430 .Keyword_alignas,
1431 .Nl,
14211432 .Keyword_alignof,
14221433 .Keyword_atomic,
14231434 .Keyword_generic,
14241435 .Keyword_noreturn,
14251436 .Keyword_static_assert,
14261437 .Keyword_thread_local,
1438 .Nl,
14271439 });
14281440}
14291441
......@@ -1469,7 +1481,10 @@ test "line continuation" {
14691481 \\ bar
14701482 \\"foo\
14711483 \\ bar"
1472 \\
1484 \\#define "foo"
1485 \\ "bar"
1486 \\#define "foo" \
1487 \\ "bar"
14731488 , &[_]Token.Id{
14741489 .Hash,
14751490 .Keyword_define,
......@@ -1477,6 +1492,14 @@ test "line continuation" {
14771492 .Identifier,
14781493 .Nl,
14791494 .{ .StringLiteral = .None },
1495 .Hash,
1496 .Keyword_define,
1497 .{ .StringLiteral = .None },
1498 .Nl,
1499 .{ .StringLiteral = .None },
1500 .Hash,
1501 .Keyword_define,
1502 .{ .StringLiteral = .None },
14801503 });
14811504}
14821505
......@@ -1499,9 +1522,13 @@ test "string prefix" {
14991522 .{ .StringLiteral = .Utf32 },
15001523 .{ .StringLiteral = .Wide },
15011524 .{ .CharLiteral = .None },
1525 .Nl,
15021526 .{ .CharLiteral = .Utf16 },
1527 .Nl,
15031528 .{ .CharLiteral = .Utf32 },
1529 .Nl,
15041530 .{ .CharLiteral = .Wide },
1531 .Nl,
15051532 });
15061533}
15071534
......@@ -1517,15 +1544,18 @@ test "num suffixes" {
15171544 .{ .FloatLiteral = .None },
15181545 .{ .FloatLiteral = .None },
15191546 .{ .FloatLiteral = .None },
1547 .Nl,
15201548 .{ .IntegerLiteral = .L },
15211549 .{ .IntegerLiteral = .LU },
15221550 .{ .IntegerLiteral = .LL },
15231551 .{ .IntegerLiteral = .LLU },
15241552 .{ .IntegerLiteral = .None },
1553 .Nl,
15251554 .{ .IntegerLiteral = .U },
15261555 .{ .IntegerLiteral = .LU },
15271556 .{ .IntegerLiteral = .LLU },
15281557 .{ .IntegerLiteral = .None },
1558 .Nl,
15291559 });
15301560}
15311561