authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-10-30 16:08:04+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-10-30 16:08:04+02:00
log80dd43213727b914418175e6e35c35de95204f7c
tree47ecbc59fdfdbe8645b1c07efb2b3cbc3e720bb8
parent72343ffd06dbbc95424d74c93188ba4a6aa74c49
parentf54605ecc2c93b98b94c5e9e42aafa9e433006fc
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6858 from travv0/no-star-after-dot-star

don't allow a token starting with an asterisk directly following .*

7 files changed, 102 insertions(+), 6 deletions(-)

doc/docgen.zig+1-1
...@@ -972,7 +972,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token:...@@ -972,7 +972,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: anytype, source_token:
972 .Tilde,972 .Tilde,
973 => try writeEscaped(out, src[token.loc.start..token.loc.end]),973 => try writeEscaped(out, src[token.loc.start..token.loc.end]),
974974
975 .Invalid, .Invalid_ampersands => return parseError(975 .Invalid, .Invalid_ampersands, .Invalid_periodasterisks => return parseError(
976 docgen_tokenizer,976 docgen_tokenizer,
977 source_token,977 source_token,
978 "syntax error",978 "syntax error",
lib/std/zig/ast.zig+4
...@@ -171,6 +171,7 @@ pub const Error = union(enum) {...@@ -171,6 +171,7 @@ pub const Error = union(enum) {
171 ExpectedBlockOrField: ExpectedBlockOrField,171 ExpectedBlockOrField: ExpectedBlockOrField,
172 DeclBetweenFields: DeclBetweenFields,172 DeclBetweenFields: DeclBetweenFields,
173 InvalidAnd: InvalidAnd,173 InvalidAnd: InvalidAnd,
174 AsteriskAfterPointerDereference: AsteriskAfterPointerDereference,
174175
175 pub fn render(self: *const Error, tokens: []const Token.Id, stream: anytype) !void {176 pub fn render(self: *const Error, tokens: []const Token.Id, stream: anytype) !void {
176 switch (self.*) {177 switch (self.*) {
...@@ -222,6 +223,7 @@ pub const Error = union(enum) {...@@ -222,6 +223,7 @@ pub const Error = union(enum) {
222 .ExpectedBlockOrField => |*x| return x.render(tokens, stream),223 .ExpectedBlockOrField => |*x| return x.render(tokens, stream),
223 .DeclBetweenFields => |*x| return x.render(tokens, stream),224 .DeclBetweenFields => |*x| return x.render(tokens, stream),
224 .InvalidAnd => |*x| return x.render(tokens, stream),225 .InvalidAnd => |*x| return x.render(tokens, stream),
226 .AsteriskAfterPointerDereference => |*x| return x.render(tokens, stream),
225 }227 }
226 }228 }
227229
...@@ -275,6 +277,7 @@ pub const Error = union(enum) {...@@ -275,6 +277,7 @@ pub const Error = union(enum) {
275 .ExpectedBlockOrField => |x| return x.token,277 .ExpectedBlockOrField => |x| return x.token,
276 .DeclBetweenFields => |x| return x.token,278 .DeclBetweenFields => |x| return x.token,
277 .InvalidAnd => |x| return x.token,279 .InvalidAnd => |x| return x.token,
280 .AsteriskAfterPointerDereference => |x| return x.token,
278 }281 }
279 }282 }
280283
...@@ -323,6 +326,7 @@ pub const Error = union(enum) {...@@ -323,6 +326,7 @@ pub const Error = union(enum) {
323 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");326 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
324 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");327 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
325 pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");328 pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
329 pub const AsteriskAfterPointerDereference = SimpleError("`.*` can't be followed by `*`. Are you missing a space?");
326330
327 pub const ExpectedCall = struct {331 pub const ExpectedCall = struct {
328 node: *Node,332 node: *Node,
lib/std/zig/parse.zig+13
...@@ -2701,6 +2701,19 @@ const Parser = struct {...@@ -2701,6 +2701,19 @@ const Parser = struct {
2701 return &node.base;2701 return &node.base;
2702 }2702 }
27032703
2704 if (p.eatToken(.Invalid_periodasterisks)) |period_asterisk| {
2705 try p.errors.append(p.gpa, .{
2706 .AsteriskAfterPointerDereference = .{ .token = period_asterisk },
2707 });
2708 const node = try p.arena.allocator.create(Node.SimpleSuffixOp);
2709 node.* = .{
2710 .base = .{ .tag = .Deref },
2711 .lhs = lhs,
2712 .rtoken = period_asterisk,
2713 };
2714 return &node.base;
2715 }
2716
2704 if (p.eatToken(.Period)) |period| {2717 if (p.eatToken(.Period)) |period| {
2705 if (try p.parseIdentifier()) |identifier| {2718 if (try p.parseIdentifier()) |identifier| {
2706 const node = try p.arena.allocator.create(Node.SimpleInfixOp);2719 const node = try p.arena.allocator.create(Node.SimpleInfixOp);
lib/std/zig/parser_test.zig+18
...@@ -219,6 +219,24 @@ test "recovery: invalid global error set access" {...@@ -219,6 +219,24 @@ test "recovery: invalid global error set access" {
219 });219 });
220}220}
221221
222test "recovery: invalid asterisk after pointer dereference" {
223 try testError(
224 \\test "" {
225 \\ var sequence = "repeat".*** 10;
226 \\}
227 , &[_]Error{
228 .AsteriskAfterPointerDereference,
229 });
230 try testError(
231 \\test "" {
232 \\ var sequence = "repeat".** 10&&a;
233 \\}
234 , &[_]Error{
235 .AsteriskAfterPointerDereference,
236 .InvalidAnd,
237 });
238}
239
222test "recovery: missing semicolon after if, for, while stmt" {240test "recovery: missing semicolon after if, for, while stmt" {
223 try testError(241 try testError(
224 \\test "" {242 \\test "" {
lib/std/zig/tokenizer.zig+43-3
...@@ -78,6 +78,7 @@ pub const Token = struct {...@@ -78,6 +78,7 @@ pub const Token = struct {
78 pub const Id = enum {78 pub const Id = enum {
79 Invalid,79 Invalid,
80 Invalid_ampersands,80 Invalid_ampersands,
81 Invalid_periodasterisks,
81 Identifier,82 Identifier,
82 StringLiteral,83 StringLiteral,
83 MultilineStringLiteralLine,84 MultilineStringLiteralLine,
...@@ -201,6 +202,7 @@ pub const Token = struct {...@@ -201,6 +202,7 @@ pub const Token = struct {
201 return switch (id) {202 return switch (id) {
202 .Invalid => "Invalid",203 .Invalid => "Invalid",
203 .Invalid_ampersands => "&&",204 .Invalid_ampersands => "&&",
205 .Invalid_periodasterisks => ".**",
204 .Identifier => "Identifier",206 .Identifier => "Identifier",
205 .StringLiteral => "StringLiteral",207 .StringLiteral => "StringLiteral",
206 .MultilineStringLiteralLine => "MultilineStringLiteralLine",208 .MultilineStringLiteralLine => "MultilineStringLiteralLine",
...@@ -403,6 +405,7 @@ pub const Tokenizer = struct {...@@ -403,6 +405,7 @@ pub const Tokenizer = struct {
403 angle_bracket_angle_bracket_right,405 angle_bracket_angle_bracket_right,
404 period,406 period,
405 period_2,407 period_2,
408 period_asterisk,
406 saw_at_sign,409 saw_at_sign,
407 };410 };
408411
...@@ -979,9 +982,7 @@ pub const Tokenizer = struct {...@@ -979,9 +982,7 @@ pub const Tokenizer = struct {
979 state = .period_2;982 state = .period_2;
980 },983 },
981 '*' => {984 '*' => {
982 result.id = .PeriodAsterisk;985 state = .period_asterisk;
983 self.index += 1;
984 break;
985 },986 },
986 else => {987 else => {
987 result.id = .Period;988 result.id = .Period;
...@@ -1001,6 +1002,17 @@ pub const Tokenizer = struct {...@@ -1001,6 +1002,17 @@ pub const Tokenizer = struct {
1001 },1002 },
1002 },1003 },
10031004
1005 .period_asterisk => switch (c) {
1006 '*' => {
1007 result.id = .Invalid_periodasterisks;
1008 break;
1009 },
1010 else => {
1011 result.id = .PeriodAsterisk;
1012 break;
1013 },
1014 },
1015
1004 .slash => switch (c) {1016 .slash => switch (c) {
1005 '/' => {1017 '/' => {
1006 state = .line_comment_start;1018 state = .line_comment_start;
...@@ -1376,6 +1388,9 @@ pub const Tokenizer = struct {...@@ -1376,6 +1388,9 @@ pub const Tokenizer = struct {
1376 .period_2 => {1388 .period_2 => {
1377 result.id = .Ellipsis2;1389 result.id = .Ellipsis2;
1378 },1390 },
1391 .period_asterisk => {
1392 result.id = .PeriodAsterisk;
1393 },
1379 .pipe => {1394 .pipe => {
1380 result.id = .Pipe;1395 result.id = .Pipe;
1381 },1396 },
...@@ -1762,6 +1777,31 @@ test "correctly parse pointer assignment" {...@@ -1762,6 +1777,31 @@ test "correctly parse pointer assignment" {
1762 });1777 });
1763}1778}
17641779
1780test "correctly parse pointer dereference followed by asterisk" {
1781 testTokenize("\"b\".* ** 10", &[_]Token.Id{
1782 .StringLiteral,
1783 .PeriodAsterisk,
1784 .AsteriskAsterisk,
1785 .IntegerLiteral,
1786 });
1787
1788 testTokenize("(\"b\".*)** 10", &[_]Token.Id{
1789 .LParen,
1790 .StringLiteral,
1791 .PeriodAsterisk,
1792 .RParen,
1793 .AsteriskAsterisk,
1794 .IntegerLiteral,
1795 });
1796
1797 testTokenize("\"b\".*** 10", &[_]Token.Id{
1798 .StringLiteral,
1799 .Invalid_periodasterisks,
1800 .AsteriskAsterisk,
1801 .IntegerLiteral,
1802 });
1803}
1804
1765test "tokenizer - range literals" {1805test "tokenizer - range literals" {
1766 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });1806 testTokenize("0...9", &[_]Token.Id{ .IntegerLiteral, .Ellipsis3, .IntegerLiteral });
1767 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });1807 testTokenize("'0'...'9'", &[_]Token.Id{ .CharLiteral, .Ellipsis3, .CharLiteral });
src/stage1/tokenizer.cpp+15-2
...@@ -223,6 +223,7 @@ enum TokenizeState {...@@ -223,6 +223,7 @@ enum TokenizeState {
223 TokenizeStateSawGreaterThanGreaterThan,223 TokenizeStateSawGreaterThanGreaterThan,
224 TokenizeStateSawDot,224 TokenizeStateSawDot,
225 TokenizeStateSawDotDot,225 TokenizeStateSawDotDot,
226 TokenizeStateSawDotStar,
226 TokenizeStateSawAtSign,227 TokenizeStateSawAtSign,
227 TokenizeStateCharCode,228 TokenizeStateCharCode,
228 TokenizeStateError,229 TokenizeStateError,
...@@ -566,9 +567,8 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -566,9 +567,8 @@ void tokenize(Buf *buf, Tokenization *out) {
566 set_token_id(&t, t.cur_tok, TokenIdEllipsis2);567 set_token_id(&t, t.cur_tok, TokenIdEllipsis2);
567 break;568 break;
568 case '*':569 case '*':
569 t.state = TokenizeStateStart;570 t.state = TokenizeStateSawDotStar;
570 set_token_id(&t, t.cur_tok, TokenIdDotStar);571 set_token_id(&t, t.cur_tok, TokenIdDotStar);
571 end_token(&t);
572 break;572 break;
573 default:573 default:
574 t.pos -= 1;574 t.pos -= 1;
...@@ -591,6 +591,18 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -591,6 +591,18 @@ void tokenize(Buf *buf, Tokenization *out) {
591 continue;591 continue;
592 }592 }
593 break;593 break;
594 case TokenizeStateSawDotStar:
595 switch (c) {
596 case '*':
597 tokenize_error(&t, "`.*` can't be followed by `*`. Are you missing a space?");
598 break;
599 default:
600 t.pos -= 1;
601 end_token(&t);
602 t.state = TokenizeStateStart;
603 continue;
604 }
605 break;
594 case TokenizeStateSawGreaterThan:606 case TokenizeStateSawGreaterThan:
595 switch (c) {607 switch (c) {
596 case '=':608 case '=':
...@@ -1481,6 +1493,7 @@ void tokenize(Buf *buf, Tokenization *out) {...@@ -1481,6 +1493,7 @@ void tokenize(Buf *buf, Tokenization *out) {
1481 case TokenizeStateSawGreaterThan:1493 case TokenizeStateSawGreaterThan:
1482 case TokenizeStateSawGreaterThanGreaterThan:1494 case TokenizeStateSawGreaterThanGreaterThan:
1483 case TokenizeStateSawDot:1495 case TokenizeStateSawDot:
1496 case TokenizeStateSawDotStar:
1484 case TokenizeStateSawAtSign:1497 case TokenizeStateSawAtSign:
1485 case TokenizeStateSawStarPercent:1498 case TokenizeStateSawStarPercent:
1486 case TokenizeStateSawPlusPercent:1499 case TokenizeStateSawPlusPercent:
test/compile_errors.zig+8
...@@ -8210,4 +8210,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {...@@ -8210,4 +8210,12 @@ pub fn addCases(cases: *tests.CompileErrorContext) void {
8210 , &[_][]const u8{8210 , &[_][]const u8{
8211 "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",8211 "tmp.zig:4:9: error: expected type '*c_void', found '?*c_void'",
8212 });8212 });
8213
8214 cases.add("Issue #6823: don't allow .* to be followed by **",
8215 \\fn foo() void {
8216 \\ var sequence = "repeat".*** 10;
8217 \\}
8218 , &[_][]const u8{
8219 "tmp.zig:2:30: error: `.*` can't be followed by `*`. Are you missing a space?",
8220 });
8213}8221}