authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
loga32e240540895af536fdeb98db448ee745def59a
tree1472bddfdee4110ec8c01d2462d825209697e2e4
parent89f2923a8aad5119b1506c3b8e08464e132c228e
signature Commit is signed but in an unrecognized format.

improve recovery from invalid container members

Instead of trying to find the end of the block or the next comma/semicolon we no try to find the next token that can start a container member.

2 files changed, 99 insertions(+), 20 deletions(-)

lib/std/zig/parse.zig+78-20
...@@ -59,10 +59,13 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error...@@ -59,10 +59,13 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
59 node.* = .{59 node.* = .{
60 .decls = try parseContainerMembers(arena, it, tree),60 .decls = try parseContainerMembers(arena, it, tree),
61 .eof_token = eatToken(it, .Eof) orelse blk: {61 .eof_token = eatToken(it, .Eof) orelse blk: {
62 // parseContainerMembers will try to skip as much
63 // invalid tokens as it can so this can only be a '}'
64 const tok = eatToken(it, .RBrace).?;
62 try tree.errors.push(.{65 try tree.errors.push(.{
63 .ExpectedContainerMembers = .{ .token = it.index },66 .ExpectedContainerMembers = .{ .token = tok },
64 });67 });
65 break :blk undefined;68 break :blk tok;
66 },69 },
67 };70 };
68 return node;71 return node;
...@@ -101,7 +104,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -101,7 +104,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
101 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {104 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
102 error.OutOfMemory => return error.OutOfMemory,105 error.OutOfMemory => return error.OutOfMemory,
103 error.ParseError => {106 error.ParseError => {
104 findEndOfBlock(it);107 findNextContainerMember(it);
105 continue;108 continue;
106 },109 },
107 }) |node| {110 }) |node| {
...@@ -116,7 +119,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -116,7 +119,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
116 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {119 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
117 error.OutOfMemory => return error.OutOfMemory,120 error.OutOfMemory => return error.OutOfMemory,
118 error.ParseError => {121 error.ParseError => {
119 findEndOfBlock(it);122 findNextContainerMember(it);
120 continue;123 continue;
121 },124 },
122 }) |node| {125 }) |node| {
...@@ -178,8 +181,8 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -178,8 +181,8 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
178 if (parseContainerField(arena, it, tree) catch |err| switch (err) {181 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
179 error.OutOfMemory => return error.OutOfMemory,182 error.OutOfMemory => return error.OutOfMemory,
180 error.ParseError => {183 error.ParseError => {
181 // attempt to recover by finding a comma184 // attempt to recover
182 findToken(it, .Comma);185 findNextContainerMember(it);
183 continue;186 continue;
184 },187 },
185 }) |node| {188 }) |node| {
...@@ -198,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -198,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
198 const field = node.cast(Node.ContainerField).?;201 const field = node.cast(Node.ContainerField).?;
199 field.doc_comments = doc_comments;202 field.doc_comments = doc_comments;
200 try list.push(node);203 try list.push(node);
201 const comma = eatToken(it, .Comma) orelse break;204 const comma = eatToken(it, .Comma) orelse {
205 // try to continue parsing
206 const index = it.index;
207 findNextContainerMember(it);
208 switch (it.peek().?.id) {
209 .Eof, .RBrace => break,
210 else => {
211 // add error and continue
212 try tree.errors.push(.{
213 .ExpectedToken = .{ .token = index, .expected_id = .Comma },
214 });
215 continue;
216 }
217 }
218 };
202 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|219 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
203 field.doc_comments = appended_comment;220 field.doc_comments = appended_comment;
204 continue;221 continue;
...@@ -210,22 +227,63 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All...@@ -210,22 +227,63 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
210 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },227 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
211 });228 });
212 }229 }
213 break;230
231 switch (it.peek().?.id) {
232 .Eof, .RBrace => break,
233 else => {
234 // this was likely not supposed to end yet,
235 // try to find the next declaration
236 const index = it.index;
237 findNextContainerMember(it);
238 try tree.errors.push(.{
239 .ExpectedContainerMembers = .{ .token = index },
240 });
241 },
242 }
214 }243 }
215244
216 return list;245 return list;
217}246}
218247
219/// Attempts to find a closing brace.248fn findNextContainerMember(it: *TokenIterator) void {
220fn findEndOfBlock(it: *TokenIterator) void {249 var level: u32 = 0;
221 var count: u32 = 0;
222 while (true) {250 while (true) {
223 const tok = nextToken(it);251 const tok = nextToken(it);
224 switch (tok.ptr.id) {252 switch (tok.ptr.id) {
225 .LBrace => count += 1,253 // any of these can start a new top level declaration
226 .RBrace => {254 .Keyword_test,
227 if (count <= 1) return;255 .Keyword_comptime,
228 count -= 1;256 .Keyword_pub,
257 .Keyword_export,
258 .Keyword_extern,
259 .Keyword_inline,
260 .Keyword_noinline,
261 .Keyword_usingnamespace,
262 .Keyword_threadlocal,
263 .Keyword_const,
264 .Keyword_var,
265 .Keyword_fn,
266 .Identifier,
267 => {
268 if (level == 0) {
269 putBackToken(it, tok.index);
270 return;
271 }
272 },
273 .Comma, .Semicolon => {
274 // this decl was likely meant to end here
275 if (level == 0) {
276 return;
277 }
278 },
279 .LParen, .LBracket, .LBrace => level += 1,
280 .RParen, .RBracket, .RBrace => {
281 if (level == 0) {
282 // end of container, exit
283 putBackToken(it, tok.index);
284 return;
285 }
286 level -= 1;
229 },287 },
230 .Eof => {288 .Eof => {
231 putBackToken(it, tok.index);289 putBackToken(it, tok.index);
...@@ -338,9 +396,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?...@@ -338,9 +396,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
338 if (parseFnProto(arena, it, tree) catch |err| switch (err) {396 if (parseFnProto(arena, it, tree) catch |err| switch (err) {
339 error.OutOfMemory => return error.OutOfMemory,397 error.OutOfMemory => return error.OutOfMemory,
340 error.ParseError => {398 error.ParseError => {
341 // this fn will likely have a body so we399 findNextContainerMember(it);
342 // use findEndOfBlock instead of findToken.
343 findEndOfBlock(it);
344 return error.ParseError;400 return error.ParseError;
345 },401 },
346 }) |node| {402 }) |node| {
...@@ -381,7 +437,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?...@@ -381,7 +437,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
381 error.OutOfMemory => return error.OutOfMemory,437 error.OutOfMemory => return error.OutOfMemory,
382 error.ParseError => {438 error.ParseError => {
383 // try to skip to next decl439 // try to skip to next decl
384 findToken(it, .Semicolon);440 findNextContainerMember(it);
385 return error.ParseError;441 return error.ParseError;
386 },442 },
387 }) |node| {443 }) |node| {
...@@ -413,7 +469,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?...@@ -413,7 +469,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
413 error.OutOfMemory => return error.OutOfMemory,469 error.OutOfMemory => return error.OutOfMemory,
414 error.ParseError => {470 error.ParseError => {
415 // try to skip to next decl471 // try to skip to next decl
416 findToken(it, .Semicolon);472 findNextContainerMember(it);
417 return error.ParseError;473 return error.ParseError;
418 },474 },
419 };475 };
...@@ -3215,6 +3271,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {...@@ -3215,6 +3271,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
3215 try tree.errors.push(.{3271 try tree.errors.push(.{
3216 .ExpectedToken = .{ .token = token.index, .expected_id = id },3272 .ExpectedToken = .{ .token = token.index, .expected_id = id },
3217 });3273 });
3274 // go back so that we can recover properly
3275 putBackToken(it, token.index);
3218 return error.ParseError;3276 return error.ParseError;
3219 }3277 }
3220 return token.index;3278 return token.index;
lib/std/zig/parser_test.zig+21
...@@ -119,6 +119,25 @@ test "recovery: missing semicolon" {...@@ -119,6 +119,25 @@ test "recovery: missing semicolon" {
119 });119 });
120}120}
121121
122test "recovery: invalid container members" {
123 try testError(
124 \\usingnamespace;
125 \\foo+
126 \\bar@,
127 \\while (a == 2) { test "" {}}
128 \\test "" {
129 \\ a && b
130 \\}
131 , &[_]Error{
132 .ExpectedExpr,
133 .ExpectedToken,
134 .ExpectedToken,
135 .ExpectedContainerMembers,
136 .InvalidAnd,
137 .ExpectedToken,
138 });
139}
140
122test "zig fmt: top-level fields" {141test "zig fmt: top-level fields" {
123 try testCanonical(142 try testCanonical(
124 \\a: did_you_know,143 \\a: did_you_know,
...@@ -2953,6 +2972,8 @@ test "zig fmt: extern without container keyword returns error" {...@@ -2953,6 +2972,8 @@ test "zig fmt: extern without container keyword returns error" {
2953 \\2972 \\
2954 , &[_]Error{2973 , &[_]Error{
2955 .ExpectedExpr,2974 .ExpectedExpr,
2975 .ExpectedVarDeclOrFn,
2976 .ExpectedContainerMembers,
2956 });2977 });
2957}2978}
29582979