From 069aee149546c9394733b128f4a5d5b3f6f6a984 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 28 Oct 2020 16:44:39 -0500 Subject: [PATCH 1/6] don't allow a token starting with an asterisk directly following .* --- src/stage1/tokenizer.cpp | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp index f597acb701e1a063701e1a4f147f4ea4d1ec1d2e..bfb70943d80aeca8558fe509f8bb34985a0621a5 100644 --- a/src/stage1/tokenizer.cpp +++ b/src/stage1/tokenizer.cpp @@ -223,6 +223,7 @@ enum TokenizeState { TokenizeStateSawGreaterThanGreaterThan, TokenizeStateSawDot, TokenizeStateSawDotDot, + TokenizeStateSawDotStar, TokenizeStateSawAtSign, TokenizeStateCharCode, TokenizeStateError, @@ -566,9 +567,8 @@ void tokenize(Buf *buf, Tokenization *out) { set_token_id(&t, t.cur_tok, TokenIdEllipsis2); break; case '*': - t.state = TokenizeStateStart; + t.state = TokenizeStateSawDotStar; set_token_id(&t, t.cur_tok, TokenIdDotStar); - end_token(&t); break; default: t.pos -= 1; @@ -591,6 +591,18 @@ void tokenize(Buf *buf, Tokenization *out) { continue; } break; + case TokenizeStateSawDotStar: + switch (c) { + case '*': + tokenize_error(&t, "`.*` can't be followed by `*`. Are you missing a space?"); + break; + default: + t.pos -= 1; + end_token(&t); + t.state = TokenizeStateStart; + continue; + } + break; case TokenizeStateSawGreaterThan: switch (c) { case '=': -- 2.54.0 From 3c7a49c4947b748c04e314dc89f2ce56562da29c Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 28 Oct 2020 18:19:05 -0500 Subject: [PATCH 2/6] add missing case for TokenizeStateSawDotStar at eof --- src/stage1/tokenizer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/stage1/tokenizer.cpp b/src/stage1/tokenizer.cpp index bfb70943d80aeca8558fe509f8bb34985a0621a5..a51f2c5c72e3653de75d1b98ce8f5958b3ae716c 100644 --- a/src/stage1/tokenizer.cpp +++ b/src/stage1/tokenizer.cpp @@ -1493,6 +1493,7 @@ void tokenize(Buf *buf, Tokenization *out) { case TokenizeStateSawGreaterThan: case TokenizeStateSawGreaterThanGreaterThan: case TokenizeStateSawDot: + case TokenizeStateSawDotStar: case TokenizeStateSawAtSign: case TokenizeStateSawStarPercent: case TokenizeStateSawPlusPercent: -- 2.54.0 From 960b5b518fa7ff7c6760d3e5f3c1256e553e6ba8 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 28 Oct 2020 21:45:58 -0500 Subject: [PATCH 3/6] updated zig tokenizer to handle .*** and added tests --- lib/std/zig/tokenizer.zig | 44 ++++++++++++++++++++++++++++++++++++--- test/compile_errors.zig | 8 +++++++ 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index c8f33dbfaaa4fcbf74cddd469a38138cbdb105dd..280ae5c1f8c790d6222c2ce8d857c6436c525f9e 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -403,6 +403,7 @@ pub const Tokenizer = struct { angle_bracket_angle_bracket_right, period, period_2, + period_asterisk, saw_at_sign, }; @@ -979,9 +980,7 @@ pub const Tokenizer = struct { state = .period_2; }, '*' => { - result.id = .PeriodAsterisk; - self.index += 1; - break; + state = .period_asterisk; }, else => { result.id = .Period; @@ -1001,6 +1000,17 @@ pub const Tokenizer = struct { }, }, + .period_asterisk => switch (c) { + '*' => { + result.id = .Invalid; + break; + }, + else => { + result.id = .PeriodAsterisk; + break; + } + }, + .slash => switch (c) { '/' => { state = .line_comment_start; @@ -1376,6 +1386,9 @@ pub const Tokenizer = struct { .period_2 => { result.id = .Ellipsis2; }, + .period_asterisk => { + result.id = .PeriodAsterisk; + }, .pipe => { result.id = .Pipe; }, @@ -1762,6 +1775,31 @@ test "correctly parse pointer assignment" { }); } +test "correctly parse pointer dereference followed by asterisk" { + testTokenize("\"b\".* ** 10", &[_]Token.Id{ + .StringLiteral, + .PeriodAsterisk, + .AsteriskAsterisk, + .IntegerLiteral, + }); + + testTokenize("(\"b\".*)** 10", &[_]Token.Id{ + .LParen, + .StringLiteral, + .PeriodAsterisk, + .RParen, + .AsteriskAsterisk, + .IntegerLiteral, + }); + + testTokenize("\"b\".*** 10", &[_]Token.Id{ + .StringLiteral, + .Invalid, + .AsteriskAsterisk, + .IntegerLiteral, + }); +} + test "tokenizer - range literals" { testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral }); testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral }); diff --git a/test/compile_errors.zig b/test/compile_errors.zig index eb058bd0e7e0054b4c471e80a3a5b8c070d73f59..8d5c4dc8a2721c5943eaed2c42cdea9ec1f0bcdf 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -8195,4 +8195,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void { , &[_][]const u8{ "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'", }); + + cases.add("Issue #6823: don't allow .* to be followed by **", + \\fn foo() void { + \\ var sequence = "repeat".*** 10; + \\} + , &[_][]const u8{ + "tmp.zig:2:30: error: `.*` can't be followed by `*`. Are you missing a space?", + }); } -- 2.54.0 From d7f9128b5d3c8c8f6a0a617c45717cdc7e3543fd Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 29 Oct 2020 11:04:50 -0500 Subject: [PATCH 4/6] add error message to zig side of tokenizing/parsing --- lib/std/zig/ast.zig | 4 ++++ lib/std/zig/parse.zig | 7 +++++++ lib/std/zig/parser_test.zig | 11 +++++++++++ lib/std/zig/tokenizer.zig | 8 +++++--- 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 0973877aa8d85940b291902965b34405410366b0..eb878fb889e41bac6fc6b33a471d86e2a0f2e02f 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -171,6 +171,7 @@ pub const Error = union(enum) { ExpectedBlockOrField: ExpectedBlockOrField, DeclBetweenFields: DeclBetweenFields, InvalidAnd: InvalidAnd, + AsteriskAfterPointerDereference: AsteriskAfterPointerDereference, pub fn render(self: *const Error, tokens: []const Token.Id, stream: anytype) !void { switch (self.*) { @@ -222,6 +223,7 @@ pub const Error = union(enum) { .ExpectedBlockOrField => |*x| return x.render(tokens, stream), .DeclBetweenFields => |*x| return x.render(tokens, stream), .InvalidAnd => |*x| return x.render(tokens, stream), + .AsteriskAfterPointerDereference => |*x| return x.render(tokens, stream), } } @@ -275,6 +277,7 @@ pub const Error = union(enum) { .ExpectedBlockOrField => |x| return x.token, .DeclBetweenFields => |x| return x.token, .InvalidAnd => |x| return x.token, + .AsteriskAfterPointerDereference => |x| return x.token, } } @@ -323,6 +326,7 @@ pub const Error = union(enum) { pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier"); pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields"); pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND."); + pub const AsteriskAfterPointerDereference = SimpleError("`.*` can't be followed by `*`. Are you missing a space?"); pub const ExpectedCall = struct { node: *Node, diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 467b06a5caaaaefeafb903573a047c12b7ab91bf..5e745dd424aac2ce7407fe9fa8bd7d3d81dea40b 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -2701,6 +2701,13 @@ const Parser = struct { return &node.base; } + if (p.token_ids[p.tok_i] == .Invalid_periodasterisks) { + try p.errors.append(p.gpa, .{ + .AsteriskAfterPointerDereference = .{ .token = p.tok_i }, + }); + return null; + } + if (p.eatToken(.Period)) |period| { if (try p.parseIdentifier()) |identifier| { const node = try p.arena.allocator.create(Node.SimpleInfixOp); diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 994ad6d5d1d3342f6e2c19b0d893a46b881a2216..9495f536d111e2957f3b68452c9d86854025b5c7 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -219,6 +219,17 @@ test "recovery: invalid global error set access" { }); } +test "recovery: invalid asterisk after pointer dereference" { + try testError( + \\test "" { + \\ var sequence = "repeat".*** 10; + \\} + , &[_]Error{ + .AsteriskAfterPointerDereference, + .ExpectedToken, + }); +} + test "recovery: missing semicolon after if, for, while stmt" { try testError( \\test "" { diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 280ae5c1f8c790d6222c2ce8d857c6436c525f9e..a0e2806e9a324049f051d04904b1c391c600c9ae 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -78,6 +78,7 @@ pub const Token = struct { pub const Id = enum { Invalid, Invalid_ampersands, + Invalid_periodasterisks, Identifier, StringLiteral, MultilineStringLiteralLine, @@ -201,6 +202,7 @@ pub const Token = struct { return switch (id) { .Invalid => "Invalid", .Invalid_ampersands => "&&", + .Invalid_periodasterisks => ".**", .Identifier => "Identifier", .StringLiteral => "StringLiteral", .MultilineStringLiteralLine => "MultilineStringLiteralLine", @@ -1002,13 +1004,13 @@ pub const Tokenizer = struct { .period_asterisk => switch (c) { '*' => { - result.id = .Invalid; + result.id = .Invalid_periodasterisks; break; }, else => { result.id = .PeriodAsterisk; break; - } + }, }, .slash => switch (c) { @@ -1794,7 +1796,7 @@ test "correctly parse pointer dereference followed by asterisk" { testTokenize("\"b\".*** 10", &[_]Token.Id{ .StringLiteral, - .Invalid, + .Invalid_periodasterisks, .AsteriskAsterisk, .IntegerLiteral, }); -- 2.54.0 From a63fd34c50b032e4ed5136dba8b098f780d5b9f8 Mon Sep 17 00:00:00 2001 From: Vexu Date: Thu, 29 Oct 2020 19:20:15 +0200 Subject: [PATCH 5/6] return a valid node even if invalid deref was used --- lib/std/zig/parse.zig | 12 +++++++++--- lib/std/zig/parser_test.zig | 9 ++++++++- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 5e745dd424aac2ce7407fe9fa8bd7d3d81dea40b..d36bf55ecd1efbe21bc065006b9c942c4e2ccfaa 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -2701,11 +2701,17 @@ const Parser = struct { return &node.base; } - if (p.token_ids[p.tok_i] == .Invalid_periodasterisks) { + if (p.eatToken(.Invalid_periodasterisks)) |period_asterisk| { try p.errors.append(p.gpa, .{ - .AsteriskAfterPointerDereference = .{ .token = p.tok_i }, + .AsteriskAfterPointerDereference = .{ .token = period_asterisk }, }); - return null; + const node = try p.arena.allocator.create(Node.SimpleSuffixOp); + node.* = .{ + .base = .{ .tag = .Deref }, + .lhs = lhs, + .rtoken = period_asterisk, + }; + return &node.base; } if (p.eatToken(.Period)) |period| { diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 9495f536d111e2957f3b68452c9d86854025b5c7..71642da4531d5d2d8e6a1061e1c26427949df70d 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -226,7 +226,14 @@ test "recovery: invalid asterisk after pointer dereference" { \\} , &[_]Error{ .AsteriskAfterPointerDereference, - .ExpectedToken, + }); + try testError( + \\test "" { + \\ var sequence = "repeat".** 10&&a; + \\} + , &[_]Error{ + .AsteriskAfterPointerDereference, + .InvalidAnd, }); } -- 2.54.0 From f54605ecc2c93b98b94c5e9e42aafa9e433006fc Mon Sep 17 00:00:00 2001 From: Travis Date: Thu, 29 Oct 2020 13:59:58 -0500 Subject: [PATCH 6/6] add missing Invalid_periodasterisks case in docgen --- doc/docgen.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 6916318eb2e5abea361ae1420d0297c1d788fa24..4a6942675568b2b4a08fe735d7ce6468a13e5612 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -972,7 +972,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token: .Tilde, => try writeEscaped(out, src[token.loc.start..token.loc.end]), - .Invalid, .Invalid_ampersands => return parseError( + .Invalid, .Invalid_ampersands, .Invalid_periodasterisks => return parseError( docgen_tokenizer, source_token, "syntax error", -- 2.54.0