authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
committergravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-14 12:09:40+03:00
loga32e240540895af536fdeb98db448ee745def59a
tree1472bddfdee4110ec8c01d2462d825209697e2e4
parent89f2923a8aad5119b1506c3b8e08464e132c228e
signature Commit is signed but in an unrecognized format.

improve recovery from invalid container members

Instead of trying to find the end of the block or the next comma/semicolon we no try to find the next token that can start a container member.

2 files changed, 99 insertions(+), 20 deletions(-)

lib/std/zig/parse.zig+78-20
......@@ -59,10 +59,13 @@ fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error
5959 node.* = .{
6060 .decls = try parseContainerMembers(arena, it, tree),
6161 .eof_token = eatToken(it, .Eof) orelse blk: {
62 // parseContainerMembers will try to skip as much
63 // invalid tokens as it can so this can only be a '}'
64 const tok = eatToken(it, .RBrace).?;
6265 try tree.errors.push(.{
63 .ExpectedContainerMembers = .{ .token = it.index },
66 .ExpectedContainerMembers = .{ .token = tok },
6467 });
65 break :blk undefined;
68 break :blk tok;
6669 },
6770 };
6871 return node;
......@@ -101,7 +104,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
101104 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
102105 error.OutOfMemory => return error.OutOfMemory,
103106 error.ParseError => {
104 findEndOfBlock(it);
107 findNextContainerMember(it);
105108 continue;
106109 },
107110 }) |node| {
......@@ -116,7 +119,7 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
116119 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
117120 error.OutOfMemory => return error.OutOfMemory,
118121 error.ParseError => {
119 findEndOfBlock(it);
122 findNextContainerMember(it);
120123 continue;
121124 },
122125 }) |node| {
......@@ -178,8 +181,8 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
178181 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
179182 error.OutOfMemory => return error.OutOfMemory,
180183 error.ParseError => {
181 // attempt to recover by finding a comma
182 findToken(it, .Comma);
184 // attempt to recover
185 findNextContainerMember(it);
183186 continue;
184187 },
185188 }) |node| {
......@@ -198,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
198201 const field = node.cast(Node.ContainerField).?;
199202 field.doc_comments = doc_comments;
200203 try list.push(node);
201 const comma = eatToken(it, .Comma) orelse break;
204 const comma = eatToken(it, .Comma) orelse {
205 // try to continue parsing
206 const index = it.index;
207 findNextContainerMember(it);
208 switch (it.peek().?.id) {
209 .Eof, .RBrace => break,
210 else => {
211 // add error and continue
212 try tree.errors.push(.{
213 .ExpectedToken = .{ .token = index, .expected_id = .Comma },
214 });
215 continue;
216 }
217 }
218 };
202219 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
203220 field.doc_comments = appended_comment;
204221 continue;
......@@ -210,22 +227,63 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) All
210227 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
211228 });
212229 }
213 break;
230
231 switch (it.peek().?.id) {
232 .Eof, .RBrace => break,
233 else => {
234 // this was likely not supposed to end yet,
235 // try to find the next declaration
236 const index = it.index;
237 findNextContainerMember(it);
238 try tree.errors.push(.{
239 .ExpectedContainerMembers = .{ .token = index },
240 });
241 },
242 }
214243 }
215244
216245 return list;
217246}
218247
219/// Attempts to find a closing brace.
220fn findEndOfBlock(it: *TokenIterator) void {
221 var count: u32 = 0;
248fn findNextContainerMember(it: *TokenIterator) void {
249 var level: u32 = 0;
222250 while (true) {
223251 const tok = nextToken(it);
224252 switch (tok.ptr.id) {
225 .LBrace => count += 1,
226 .RBrace => {
227 if (count <= 1) return;
228 count -= 1;
253 // any of these can start a new top level declaration
254 .Keyword_test,
255 .Keyword_comptime,
256 .Keyword_pub,
257 .Keyword_export,
258 .Keyword_extern,
259 .Keyword_inline,
260 .Keyword_noinline,
261 .Keyword_usingnamespace,
262 .Keyword_threadlocal,
263 .Keyword_const,
264 .Keyword_var,
265 .Keyword_fn,
266 .Identifier,
267 => {
268 if (level == 0) {
269 putBackToken(it, tok.index);
270 return;
271 }
272 },
273 .Comma, .Semicolon => {
274 // this decl was likely meant to end here
275 if (level == 0) {
276 return;
277 }
278 },
279 .LParen, .LBracket, .LBrace => level += 1,
280 .RParen, .RBracket, .RBrace => {
281 if (level == 0) {
282 // end of container, exit
283 putBackToken(it, tok.index);
284 return;
285 }
286 level -= 1;
229287 },
230288 .Eof => {
231289 putBackToken(it, tok.index);
......@@ -338,9 +396,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
338396 if (parseFnProto(arena, it, tree) catch |err| switch (err) {
339397 error.OutOfMemory => return error.OutOfMemory,
340398 error.ParseError => {
341 // this fn will likely have a body so we
342 // use findEndOfBlock instead of findToken.
343 findEndOfBlock(it);
399 findNextContainerMember(it);
344400 return error.ParseError;
345401 },
346402 }) |node| {
......@@ -381,7 +437,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
381437 error.OutOfMemory => return error.OutOfMemory,
382438 error.ParseError => {
383439 // try to skip to next decl
384 findToken(it, .Semicolon);
440 findNextContainerMember(it);
385441 return error.ParseError;
386442 },
387443 }) |node| {
......@@ -413,7 +469,7 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?
413469 error.OutOfMemory => return error.OutOfMemory,
414470 error.ParseError => {
415471 // try to skip to next decl
416 findToken(it, .Semicolon);
472 findNextContainerMember(it);
417473 return error.ParseError;
418474 },
419475 };
......@@ -3215,6 +3271,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
32153271 try tree.errors.push(.{
32163272 .ExpectedToken = .{ .token = token.index, .expected_id = id },
32173273 });
3274 // go back so that we can recover properly
3275 putBackToken(it, token.index);
32183276 return error.ParseError;
32193277 }
32203278 return token.index;
lib/std/zig/parser_test.zig+21
......@@ -119,6 +119,25 @@ test "recovery: missing semicolon" {
119119 });
120120}
121121
122test "recovery: invalid container members" {
123 try testError(
124 \\usingnamespace;
125 \\foo+
126 \\bar@,
127 \\while (a == 2) { test "" {}}
128 \\test "" {
129 \\ a && b
130 \\}
131 , &[_]Error{
132 .ExpectedExpr,
133 .ExpectedToken,
134 .ExpectedToken,
135 .ExpectedContainerMembers,
136 .InvalidAnd,
137 .ExpectedToken,
138 });
139}
140
122141test "zig fmt: top-level fields" {
123142 try testCanonical(
124143 \\a: did_you_know,
......@@ -2953,6 +2972,8 @@ test "zig fmt: extern without container keyword returns error" {
29532972 \\
29542973 , &[_]Error{
29552974 .ExpectedExpr,
2975 .ExpectedVarDeclOrFn,
2976 .ExpectedContainerMembers,
29562977 });
29572978}
29582979