authorgravatar for tssund93@gmail.comTravis <tssund93@gmail.com> 2020-10-28 21:45:58-05:00
committergravatar for tssund93@gmail.comTravis <tssund93@gmail.com> 2020-10-29 12:03:45-05:00
log960b5b518fa7ff7c6760d3e5f3c1256e553e6ba8
tree6f749eb05bb723d43de068c7142ca21d63f08355
parent3c7a49c4947b748c04e314dc89f2ce56562da29c

updated zig tokenizer to handle .*** and added tests

2 files changed, 49 insertions(+), 3 deletions(-)

lib/std/zig/tokenizer.zig+41-3
......@@ -403,6 +403,7 @@ pub const Tokenizer = struct {
403403 angle_bracket_angle_bracket_right,
404404 period,
405405 period_2,
406 period_asterisk,
406407 saw_at_sign,
407408 };
408409
......@@ -979,9 +980,7 @@ pub const Tokenizer = struct {
979980 state = .period_2;
980981 },
981982 '*' => {
982 result.id = .PeriodAsterisk;
983 self.index += 1;
984 break;
983 state = .period_asterisk;
985984 },
986985 else => {
987986 result.id = .Period;
......@@ -1001,6 +1000,17 @@ pub const Tokenizer = struct {
10011000 },
10021001 },
10031002
1003 .period_asterisk => switch (c) {
1004 '*' => {
1005 result.id = .Invalid;
1006 break;
1007 },
1008 else => {
1009 result.id = .PeriodAsterisk;
1010 break;
1011 }
1012 },
1013
10041014 .slash => switch (c) {
10051015 '/' => {
10061016 state = .line_comment_start;
......@@ -1376,6 +1386,9 @@ pub const Tokenizer = struct {
13761386 .period_2 => {
13771387 result.id = .Ellipsis2;
13781388 },
1389 .period_asterisk => {
1390 result.id = .PeriodAsterisk;
1391 },
13791392 .pipe => {
13801393 result.id = .Pipe;
13811394 },
......@@ -1762,6 +1775,31 @@ test "correctly parse pointer assignment" {
17621775 });
17631776}
17641777
1778test "correctly parse pointer dereference followed by asterisk" {
1779 testTokenize("\"b\".* ** 10", &[_]Token.Id{
1780 .StringLiteral,
1781 .PeriodAsterisk,
1782 .AsteriskAsterisk,
1783 .IntegerLiteral,
1784 });
1785
1786 testTokenize("(\"b\".*)** 10", &[_]Token.Id{
1787 .LParen,
1788 .StringLiteral,
1789 .PeriodAsterisk,
1790 .RParen,
1791 .AsteriskAsterisk,
1792 .IntegerLiteral,
1793 });
1794
1795 testTokenize("\"b\".*** 10", &[_]Token.Id{
1796 .StringLiteral,
1797 .Invalid,
1798 .AsteriskAsterisk,
1799 .IntegerLiteral,
1800 });
1801}
1802
17651803test "tokenizer - range literals" {
17661804 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
17671805 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
test/compile_errors.zig+8
......@@ -8195,4 +8195,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
81958195 , &[_][]const u8{
81968196 "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",
81978197 });
8198
8199 cases.add("Issue #6823: don't allow .* to be followed by **",
8200 \\fn foo() void {
8201 \\ var sequence = "repeat".*** 10;
8202 \\}
8203 , &[_][]const u8{
8204 "tmp.zig:2:30: error: `.*` can't be followed by `*`. Are you missing a space?",
8205 });
81988206}