authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-10-30 16:08:04+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-10-30 16:08:04+02:00
log80dd43213727b914418175e6e35c35de95204f7c
tree47ecbc59fdfdbe8645b1c07efb2b3cbc3e720bb8
parent72343ffd06dbbc95424d74c93188ba4a6aa74c49
parentf54605ecc2c93b98b94c5e9e42aafa9e433006fc
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6858 from travv0/no-star-after-dot-star

don't allow a token starting with an asterisk directly following .*

7 files changed, 102 insertions(+), 6 deletions(-)

doc/docgen.zig+1-1
......@@ -972,7 +972,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token:
972972 .Tilde,
973973 => try writeEscaped(out, src[token.loc.start..token.loc.end]),
974974
975 .Invalid, .Invalid_ampersands => return parseError(
975 .Invalid, .Invalid_ampersands, .Invalid_periodasterisks => return parseError(
976976 docgen_tokenizer,
977977 source_token,
978978 "syntax error",
lib/std/zig/ast.zig+4
......@@ -171,6 +171,7 @@ pub const Error = union(enum) {
171171 ExpectedBlockOrField: ExpectedBlockOrField,
172172 DeclBetweenFields: DeclBetweenFields,
173173 InvalidAnd: InvalidAnd,
174 AsteriskAfterPointerDereference: AsteriskAfterPointerDereference,
174175
175176 pub fn render(self: *const Error, tokens: []const Token.Id, stream: anytype) !void {
176177 switch (self.*) {
......@@ -222,6 +223,7 @@ pub const Error = union(enum) {
222223 .ExpectedBlockOrField => |*x| return x.render(tokens, stream),
223224 .DeclBetweenFields => |*x| return x.render(tokens, stream),
224225 .InvalidAnd => |*x| return x.render(tokens, stream),
226 .AsteriskAfterPointerDereference => |*x| return x.render(tokens, stream),
225227 }
226228 }
227229
......@@ -275,6 +277,7 @@ pub const Error = union(enum) {
275277 .ExpectedBlockOrField => |x| return x.token,
276278 .DeclBetweenFields => |x| return x.token,
277279 .InvalidAnd => |x| return x.token,
280 .AsteriskAfterPointerDereference => |x| return x.token,
278281 }
279282 }
280283
......@@ -323,6 +326,7 @@ pub const Error = union(enum) {
323326 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
324327 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
325328 pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
329 pub const AsteriskAfterPointerDereference = SimpleError("`.*` can't be followed by `*`. Are you missing a space?");
326330
327331 pub const ExpectedCall = struct {
328332 node: *Node,
lib/std/zig/parse.zig+13
......@@ -2701,6 +2701,19 @@ const Parser = struct {
27012701 return &node.base;
27022702 }
27032703
2704 if (p.eatToken(.Invalid_periodasterisks)) |period_asterisk| {
2705 try p.errors.append(p.gpa, .{
2706 .AsteriskAfterPointerDereference = .{ .token = period_asterisk },
2707 });
2708 const node = try p.arena.allocator.create(Node.SimpleSuffixOp);
2709 node.* = .{
2710 .base = .{ .tag = .Deref },
2711 .lhs = lhs,
2712 .rtoken = period_asterisk,
2713 };
2714 return &node.base;
2715 }
2716
27042717 if (p.eatToken(.Period)) |period| {
27052718 if (try p.parseIdentifier()) |identifier| {
27062719 const node = try p.arena.allocator.create(Node.SimpleInfixOp);
lib/std/zig/parser_test.zig+18
......@@ -219,6 +219,24 @@ test "recovery: invalid global error set access" {
219219 });
220220}
221221
222test "recovery: invalid asterisk after pointer dereference" {
223 try testError(
224 \\test "" {
225 \\ var sequence = "repeat".*** 10;
226 \\}
227 , &[_]Error{
228 .AsteriskAfterPointerDereference,
229 });
230 try testError(
231 \\test "" {
232 \\ var sequence = "repeat".** 10&&a;
233 \\}
234 , &[_]Error{
235 .AsteriskAfterPointerDereference,
236 .InvalidAnd,
237 });
238}
239
222240test "recovery: missing semicolon after if, for, while stmt" {
223241 try testError(
224242 \\test "" {
lib/std/zig/tokenizer.zig+43-3
......@@ -78,6 +78,7 @@ pub const Token = struct {
7878 pub const Id = enum {
7979 Invalid,
8080 Invalid_ampersands,
81 Invalid_periodasterisks,
8182 Identifier,
8283 StringLiteral,
8384 MultilineStringLiteralLine,
......@@ -201,6 +202,7 @@ pub const Token = struct {
201202 return switch (id) {
202203 .Invalid => "Invalid",
203204 .Invalid_ampersands => "&&",
205 .Invalid_periodasterisks => ".**",
204206 .Identifier => "Identifier",
205207 .StringLiteral => "StringLiteral",
206208 .MultilineStringLiteralLine => "MultilineStringLiteralLine",
......@@ -403,6 +405,7 @@ pub const Tokenizer = struct {
403405 angle_bracket_angle_bracket_right,
404406 period,
405407 period_2,
408 period_asterisk,
406409 saw_at_sign,
407410 };
408411
......@@ -979,9 +982,7 @@ pub const Tokenizer = struct {
979982 state = .period_2;
980983 },
981984 '*' => {
982 result.id = .PeriodAsterisk;
983 self.index += 1;
984 break;
985 state = .period_asterisk;
985986 },
986987 else => {
987988 result.id = .Period;
......@@ -1001,6 +1002,17 @@ pub const Tokenizer = struct {
10011002 },
10021003 },
10031004
1005 .period_asterisk => switch (c) {
1006 '*' => {
1007 result.id = .Invalid_periodasterisks;
1008 break;
1009 },
1010 else => {
1011 result.id = .PeriodAsterisk;
1012 break;
1013 },
1014 },
1015
10041016 .slash => switch (c) {
10051017 '/' => {
10061018 state = .line_comment_start;
......@@ -1376,6 +1388,9 @@ pub const Tokenizer = struct {
13761388 .period_2 => {
13771389 result.id = .Ellipsis2;
13781390 },
1391 .period_asterisk => {
1392 result.id = .PeriodAsterisk;
1393 },
13791394 .pipe => {
13801395 result.id = .Pipe;
13811396 },
......@@ -1762,6 +1777,31 @@ test "correctly parse pointer assignment" {
17621777 });
17631778}
17641779
1780test "correctly parse pointer dereference followed by asterisk" {
1781 testTokenize("\"b\".* ** 10", &[_]Token.Id{
1782 .StringLiteral,
1783 .PeriodAsterisk,
1784 .AsteriskAsterisk,
1785 .IntegerLiteral,
1786 });
1787
1788 testTokenize("(\"b\".*)** 10", &[_]Token.Id{
1789 .LParen,
1790 .StringLiteral,
1791 .PeriodAsterisk,
1792 .RParen,
1793 .AsteriskAsterisk,
1794 .IntegerLiteral,
1795 });
1796
1797 testTokenize("\"b\".*** 10", &[_]Token.Id{
1798 .StringLiteral,
1799 .Invalid_periodasterisks,
1800 .AsteriskAsterisk,
1801 .IntegerLiteral,
1802 });
1803}
1804
17651805test "tokenizer - range literals" {
17661806 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
17671807 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
src/stage1/tokenizer.cpp+15-2
......@@ -223,6 +223,7 @@ enum TokenizeState {
223223 TokenizeStateSawGreaterThanGreaterThan,
224224 TokenizeStateSawDot,
225225 TokenizeStateSawDotDot,
226 TokenizeStateSawDotStar,
226227 TokenizeStateSawAtSign,
227228 TokenizeStateCharCode,
228229 TokenizeStateError,
......@@ -566,9 +567,8 @@ void tokenize(Buf *buf, Tokenization *out) {
566567 set_token_id(&t, t.cur_tok, TokenIdEllipsis2);
567568 break;
568569 case '*':
569 t.state = TokenizeStateStart;
570 t.state = TokenizeStateSawDotStar;
570571 set_token_id(&t, t.cur_tok, TokenIdDotStar);
571 end_token(&t);
572572 break;
573573 default:
574574 t.pos -= 1;
......@@ -591,6 +591,18 @@ void tokenize(Buf *buf, Tokenization *out) {
591591 continue;
592592 }
593593 break;
594 case TokenizeStateSawDotStar:
595 switch (c) {
596 case '*':
597 tokenize_error(&t, "`.*` can't be followed by `*`. Are you missing a space?");
598 break;
599 default:
600 t.pos -= 1;
601 end_token(&t);
602 t.state = TokenizeStateStart;
603 continue;
604 }
605 break;
594606 case TokenizeStateSawGreaterThan:
595607 switch (c) {
596608 case '=':
......@@ -1481,6 +1493,7 @@ void tokenize(Buf *buf, Tokenization *out) {
14811493 case TokenizeStateSawGreaterThan:
14821494 case TokenizeStateSawGreaterThanGreaterThan:
14831495 case TokenizeStateSawDot:
1496 case TokenizeStateSawDotStar:
14841497 case TokenizeStateSawAtSign:
14851498 case TokenizeStateSawStarPercent:
14861499 case TokenizeStateSawPlusPercent:
test/compile_errors.zig+8
......@@ -8210,4 +8210,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
82108210 , &[_][]const u8{
82118211 "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",
82128212 });
8213
8214 cases.add("Issue #6823: don't allow .* to be followed by **",
8215 \\fn foo() void {
8216 \\ var sequence = "repeat".*** 10;
8217 \\}
8218 , &[_][]const u8{
8219 "tmp.zig:2:30: error: `.*` can't be followed by `*`. Are you missing a space?",
8220 });
82138221}