authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 19:28:14+02:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-01-05 20:25:51+02:00
log795a5039995a1a23ba00d15488565f1a79d3f25b
treed4ad93a06dbe3b88c7e60615f870cf229b091ba1
parentf934f9b41938ee6208d7cdd8a26687bffbe171cf
signature Commit is signed but in an unrecognized format.

std-c tokenizer always add newline token

2 files changed, 117 insertions(+), 83 deletions(-)

lib/std/c/parse.zig+21-17
...@@ -797,38 +797,42 @@ const Parser = struct {...@@ -797,38 +797,42 @@ const Parser = struct {
797797
798 fn eatToken(parser: *Parser, id: @TagType(Token.Id)) ?TokenIndex {798 fn eatToken(parser: *Parser, id: @TagType(Token.Id)) ?TokenIndex {
799 while (true) {799 while (true) {
800 const next_tok = parser.it.next() orelse return null;800 switch (parser.it.next() orelse return null) {
801 if (next_tok.id != .LineComment and next_tok.id != .MultiLineComment) {801 .LineComment, .MultiLineComment, .Nl => continue,
802 if (next_tok.id == id) {802 else => |next_id| if (next_id == id) {
803 return parser.it.index;803 return parser.it.index;
804 }804 } else {
805 _ = parser.it.prev();805 _ = parser.it.prev();
806 return null;806 return null;
807 },
807 }808 }
808 }809 }
809 }810 }
810811
811 fn expectToken(parser: *Parser, id: @TagType(Token.Id)) Error!TokenIndex {812 fn expectToken(parser: *Parser, id: @TagType(Token.Id)) Error!TokenIndex {
812 while (true) {813 while (true) {
813 const next_tok = parser.it.next() orelse return error.ParseError;814 switch (parser.it.next() orelse return null) {
814 if (next_tok.id != .LineComment and next_tok.id != .MultiLineComment) {815 .LineComment, .MultiLineComment, .Nl => continue,
815 if (next_tok.id != id) {816 else => |next_id| if (next_id != id) {
816 try parser.tree.errors.push(.{817 return parser.err(.{
817 .ExpectedToken = .{ .token = parser.it.index, .expected_id = id },818 .ExpectedToken = .{ .token = parser.it.index, .expected_id = id },
818 });819 });
819 return error.ParseError;820 } else {
820 }821 return parser.it.index;
821 return parser.it.index;822 },
822 }823 }
823 }824 }
824 }825 }
825826
826 fn putBackToken(parser: *Parser, putting_back: TokenIndex) void {827 fn putBackToken(parser: *Parser, putting_back: TokenIndex) void {
827 while (true) {828 while (true) {
828 const prev_tok = parser.it.prev() orelse return;829 switch (parser.it.next() orelse return null) {
829 if (prev_tok.id == .LineComment or prev_tok.id == .MultiLineComment) continue;830 .LineComment, .MultiLineComment, .Nl => continue,
830 assert(parser.it.list.at(putting_back) == prev_tok);831 else => |next_id| {
831 return;832 assert(parser.it.list.at(putting_back) == prev_tok);
833 return;
834 },
835 }
832 }836 }
833 }837 }
834838
lib/std/c/tokenizer.zig+96-66
...@@ -449,20 +449,12 @@ pub const Tokenizer = struct {...@@ -449,20 +449,12 @@ pub const Tokenizer = struct {
449 switch (state) {449 switch (state) {
450 .Start => switch (c) {450 .Start => switch (c) {
451 '\n' => {451 '\n' => {
452 if (!self.pp_directive) {
453 result.start = self.index + 1;
454 continue;
455 }
456 self.pp_directive = false;452 self.pp_directive = false;
457 result.id = .Nl;453 result.id = .Nl;
458 self.index += 1;454 self.index += 1;
459 break;455 break;
460 },456 },
461 '\r' => {457 '\r' => {
462 if (!self.pp_directive) {
463 result.start = self.index + 1;
464 continue;
465 }
466 state = .Cr;458 state = .Cr;
467 },459 },
468 '"' => {460 '"' => {
...@@ -612,11 +604,14 @@ pub const Tokenizer = struct {...@@ -612,11 +604,14 @@ pub const Tokenizer = struct {
612 },604 },
613 .BackSlash => switch (c) {605 .BackSlash => switch (c) {
614 '\n' => {606 '\n' => {
615 state = .Start;607 state = if (string) .AfterStringLiteral else .Start;
616 },608 },
617 '\r' => {609 '\r' => {
618 state = .BackSlashCr;610 state = .BackSlashCr;
619 },611 },
612 '\t', '\x0B', '\x0C', ' ' => {
613 // TODO warn
614 },
620 else => {615 else => {
621 result.id = .Invalid;616 result.id = .Invalid;
622 break;617 break;
...@@ -624,7 +619,7 @@ pub const Tokenizer = struct {...@@ -624,7 +619,7 @@ pub const Tokenizer = struct {
624 },619 },
625 .BackSlashCr => switch (c) {620 .BackSlashCr => switch (c) {
626 '\n' => {621 '\n' => {
627 state = .Start;622 state = if (string) .AfterStringLiteral else .Start;
628 },623 },
629 else => {624 else => {
630 result.id = .Invalid;625 result.id = .Invalid;
...@@ -700,7 +695,14 @@ pub const Tokenizer = struct {...@@ -700,7 +695,14 @@ pub const Tokenizer = struct {
700 '"' => {695 '"' => {
701 state = .StringLiteral;696 state = .StringLiteral;
702 },697 },
703 '\n'...'\r', ' ' => {},698 '\\' => {
699 state = .BackSlash;
700 },
701 '\n', '\r' => {
702 if (self.pp_directive)
703 break;
704 },
705 '\t', '\x0B', '\x0C', ' ' => {},
704 else => {706 else => {
705 break;707 break;
706 },708 },
...@@ -1314,60 +1316,64 @@ test "operators" {...@@ -1314,60 +1316,64 @@ test "operators" {
1314 \\ , & && &= ? < <= <<1316 \\ , & && &= ? < <= <<
1315 \\ <<= > >= >> >>= ~ # ##1317 \\ <<= > >= >> >>= ~ # ##
1316 \\1318 \\
1317 ,1319 , &[_]Token.Id{
1318 &[_]Token.Id{1320 .Bang,
1319 .Bang,1321 .BangEqual,
1320 .BangEqual,1322 .Pipe,
1321 .Pipe,1323 .PipePipe,
1322 .PipePipe,1324 .PipeEqual,
1323 .PipeEqual,1325 .Equal,
1324 .Equal,1326 .EqualEqual,
1325 .EqualEqual,1327 .Nl,
1326 .LParen,1328 .LParen,
1327 .RParen,1329 .RParen,
1328 .LBrace,1330 .LBrace,
1329 .RBrace,1331 .RBrace,
1330 .LBracket,1332 .LBracket,
1331 .RBracket,1333 .RBracket,
1332 .Period,1334 .Period,
1333 .Period,1335 .Period,
1334 .Period,1336 .Period,
1335 .Ellipsis,1337 .Ellipsis,
1336 .Caret,1338 .Nl,
1337 .CaretEqual,1339 .Caret,
1338 .Plus,1340 .CaretEqual,
1339 .PlusPlus,1341 .Plus,
1340 .PlusEqual,1342 .PlusPlus,
1341 .Minus,1343 .PlusEqual,
1342 .MinusMinus,1344 .Minus,
1343 .MinusEqual,1345 .MinusMinus,
1344 .Asterisk,1346 .MinusEqual,
1345 .AsteriskEqual,1347 .Nl,
1346 .Percent,1348 .Asterisk,
1347 .PercentEqual,1349 .AsteriskEqual,
1348 .Arrow,1350 .Percent,
1349 .Colon,1351 .PercentEqual,
1350 .Semicolon,1352 .Arrow,
1351 .Slash,1353 .Colon,
1352 .SlashEqual,1354 .Semicolon,
1353 .Comma,1355 .Slash,
1354 .Ampersand,1356 .SlashEqual,
1355 .AmpersandAmpersand,1357 .Nl,
1356 .AmpersandEqual,1358 .Comma,
1357 .QuestionMark,1359 .Ampersand,
1358 .AngleBracketLeft,1360 .AmpersandAmpersand,
1359 .AngleBracketLeftEqual,1361 .AmpersandEqual,
1360 .AngleBracketAngleBracketLeft,1362 .QuestionMark,
1361 .AngleBracketAngleBracketLeftEqual,1363 .AngleBracketLeft,
1362 .AngleBracketRight,1364 .AngleBracketLeftEqual,
1363 .AngleBracketRightEqual,1365 .AngleBracketAngleBracketLeft,
1364 .AngleBracketAngleBracketRight,1366 .Nl,
1365 .AngleBracketAngleBracketRightEqual,1367 .AngleBracketAngleBracketLeftEqual,
1366 .Tilde,1368 .AngleBracketRight,
1367 .Hash,1369 .AngleBracketRightEqual,
1368 .HashHash,1370 .AngleBracketAngleBracketRight,
1369 },1371 .AngleBracketAngleBracketRightEqual,
1370 );1372 .Tilde,
1373 .Hash,
1374 .HashHash,
1375 .Nl,
1376 });
1371}1377}
13721378
1373test "keywords" {1379test "keywords" {
...@@ -1388,6 +1394,7 @@ test "keywords" {...@@ -1388,6 +1394,7 @@ test "keywords" {
1388 .Keyword_continue,1394 .Keyword_continue,
1389 .Keyword_default,1395 .Keyword_default,
1390 .Keyword_do,1396 .Keyword_do,
1397 .Nl,
1391 .Keyword_double,1398 .Keyword_double,
1392 .Keyword_else,1399 .Keyword_else,
1393 .Keyword_enum,1400 .Keyword_enum,
...@@ -1397,6 +1404,7 @@ test "keywords" {...@@ -1397,6 +1404,7 @@ test "keywords" {
1397 .Keyword_goto,1404 .Keyword_goto,
1398 .Keyword_if,1405 .Keyword_if,
1399 .Keyword_int,1406 .Keyword_int,
1407 .Nl,
1400 .Keyword_long,1408 .Keyword_long,
1401 .Keyword_register,1409 .Keyword_register,
1402 .Keyword_return,1410 .Keyword_return,
...@@ -1404,6 +1412,7 @@ test "keywords" {...@@ -1404,6 +1412,7 @@ test "keywords" {
1404 .Keyword_signed,1412 .Keyword_signed,
1405 .Keyword_sizeof,1413 .Keyword_sizeof,
1406 .Keyword_static,1414 .Keyword_static,
1415 .Nl,
1407 .Keyword_struct,1416 .Keyword_struct,
1408 .Keyword_switch,1417 .Keyword_switch,
1409 .Keyword_typedef,1418 .Keyword_typedef,
...@@ -1411,6 +1420,7 @@ test "keywords" {...@@ -1411,6 +1420,7 @@ test "keywords" {
1411 .Keyword_unsigned,1420 .Keyword_unsigned,
1412 .Keyword_void,1421 .Keyword_void,
1413 .Keyword_volatile,1422 .Keyword_volatile,
1423 .Nl,
1414 .Keyword_while,1424 .Keyword_while,
1415 .Keyword_bool,1425 .Keyword_bool,
1416 .Keyword_complex,1426 .Keyword_complex,
...@@ -1418,12 +1428,14 @@ test "keywords" {...@@ -1418,12 +1428,14 @@ test "keywords" {
1418 .Keyword_inline,1428 .Keyword_inline,
1419 .Keyword_restrict,1429 .Keyword_restrict,
1420 .Keyword_alignas,1430 .Keyword_alignas,
1431 .Nl,
1421 .Keyword_alignof,1432 .Keyword_alignof,
1422 .Keyword_atomic,1433 .Keyword_atomic,
1423 .Keyword_generic,1434 .Keyword_generic,
1424 .Keyword_noreturn,1435 .Keyword_noreturn,
1425 .Keyword_static_assert,1436 .Keyword_static_assert,
1426 .Keyword_thread_local,1437 .Keyword_thread_local,
1438 .Nl,
1427 });1439 });
1428}1440}
14291441
...@@ -1469,7 +1481,10 @@ test "line continuation" {...@@ -1469,7 +1481,10 @@ test "line continuation" {
1469 \\ bar1481 \\ bar
1470 \\"foo\1482 \\"foo\
1471 \\ bar"1483 \\ bar"
1472 \\1484 \\#define "foo"
1485 \\ "bar"
1486 \\#define "foo" \
1487 \\ "bar"
1473 , &[_]Token.Id{1488 , &[_]Token.Id{
1474 .Hash,1489 .Hash,
1475 .Keyword_define,1490 .Keyword_define,
...@@ -1477,6 +1492,14 @@ test "line continuation" {...@@ -1477,6 +1492,14 @@ test "line continuation" {
1477 .Identifier,1492 .Identifier,
1478 .Nl,1493 .Nl,
1479 .{ .StringLiteral = .None },1494 .{ .StringLiteral = .None },
1495 .Hash,
1496 .Keyword_define,
1497 .{ .StringLiteral = .None },
1498 .Nl,
1499 .{ .StringLiteral = .None },
1500 .Hash,
1501 .Keyword_define,
1502 .{ .StringLiteral = .None },
1480 });1503 });
1481}1504}
14821505
...@@ -1499,9 +1522,13 @@ test "string prefix" {...@@ -1499,9 +1522,13 @@ test "string prefix" {
1499 .{ .StringLiteral = .Utf32 },1522 .{ .StringLiteral = .Utf32 },
1500 .{ .StringLiteral = .Wide },1523 .{ .StringLiteral = .Wide },
1501 .{ .CharLiteral = .None },1524 .{ .CharLiteral = .None },
1525 .Nl,
1502 .{ .CharLiteral = .Utf16 },1526 .{ .CharLiteral = .Utf16 },
1527 .Nl,
1503 .{ .CharLiteral = .Utf32 },1528 .{ .CharLiteral = .Utf32 },
1529 .Nl,
1504 .{ .CharLiteral = .Wide },1530 .{ .CharLiteral = .Wide },
1531 .Nl,
1505 });1532 });
1506}1533}
15071534
...@@ -1517,15 +1544,18 @@ test "num suffixes" {...@@ -1517,15 +1544,18 @@ test "num suffixes" {
1517 .{ .FloatLiteral = .None },1544 .{ .FloatLiteral = .None },
1518 .{ .FloatLiteral = .None },1545 .{ .FloatLiteral = .None },
1519 .{ .FloatLiteral = .None },1546 .{ .FloatLiteral = .None },
1547 .Nl,
1520 .{ .IntegerLiteral = .L },1548 .{ .IntegerLiteral = .L },
1521 .{ .IntegerLiteral = .LU },1549 .{ .IntegerLiteral = .LU },
1522 .{ .IntegerLiteral = .LL },1550 .{ .IntegerLiteral = .LL },
1523 .{ .IntegerLiteral = .LLU },1551 .{ .IntegerLiteral = .LLU },
1524 .{ .IntegerLiteral = .None },1552 .{ .IntegerLiteral = .None },
1553 .Nl,
1525 .{ .IntegerLiteral = .U },1554 .{ .IntegerLiteral = .U },
1526 .{ .IntegerLiteral = .LU },1555 .{ .IntegerLiteral = .LU },
1527 .{ .IntegerLiteral = .LLU },1556 .{ .IntegerLiteral = .LLU },
1528 .{ .IntegerLiteral = .None },1557 .{ .IntegerLiteral = .None },
1558 .Nl,
1529 });1559 });
1530}1560}
15311561