authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-01-13 13:38:31-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-01-13 13:38:31-05:00
logc774c9376a20c2fde7db4fed94bdea5ae94e1494
treead5ad8ee9f19a71667d79e9c2f65a0ad527cc9b1
parent2be12b24bcafd4b2785d61dcfbaa27f9e3cc705d
parent5880eb3a75b41d8ac26faa0185a83e001197a6e4
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3957 from xackus/stage2_parser_3799

stage2 parser: fix segfault on extern block

4 files changed, 56 insertions(+), 27 deletions(-)

lib/std/zig/ast.zig+2-1
......@@ -10,6 +10,7 @@ pub const TokenIndex = usize;
1010pub const Tree = struct {
1111 source: []const u8,
1212 tokens: TokenList,
13 /// undefined on parse error (errors not empty)
1314 root_node: *Node.Root,
1415 arena_allocator: std.heap.ArenaAllocator,
1516 errors: ErrorList,
......@@ -612,7 +613,7 @@ pub const Node = struct {
612613 visib_token: ?TokenIndex,
613614 thread_local_token: ?TokenIndex,
614615 name_token: TokenIndex,
615 eq_token: TokenIndex,
616 eq_token: ?TokenIndex,
616617 mut_token: TokenIndex,
617618 comptime_token: ?TokenIndex,
618619 extern_export_token: ?TokenIndex,
lib/std/zig/parse.zig+38-25
......@@ -13,7 +13,7 @@ pub const Error = error{ParseError} || Allocator.Error;
1313
1414/// Result should be freed with tree.deinit() when there are
1515/// no more references to any of the tokens or nodes.
16pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
16pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
1717 const tree = blk: {
1818 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists
1919 // from being initialized with a pointer to this `arena`, which is created on
......@@ -48,29 +48,32 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
4848
4949 while (it.peek().?.id == .LineComment) _ = it.next();
5050
51 tree.root_node = try parseRoot(arena, &it, tree);
51 tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {
52 switch (err) {
53 error.ParseError => {
54 assert(tree.errors.len != 0);
55 break :blk undefined;
56 },
57 error.OutOfMemory => {
58 return error.OutOfMemory;
59 },
60 }
61 };
62
5263 return tree;
5364}
5465
5566/// Root <- skip ContainerMembers eof
56fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
67fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {
5768 const node = try arena.create(Node.Root);
5869 node.* = Node.Root{
59 .decls = undefined,
60 .eof_token = undefined,
61 };
62 node.decls = parseContainerMembers(arena, it, tree) catch |err| {
63 // TODO: Switch on the error type
64 // https://github.com/ziglang/zig/issues/2473
65 if (err == error.ParseError) return node;
66 assert(err == Allocator.Error.OutOfMemory);
67 return Allocator.Error.OutOfMemory;
68 };
69 node.eof_token = eatToken(it, .Eof) orelse {
70 try tree.errors.push(AstError{
71 .ExpectedContainerMembers = AstError.ExpectedContainerMembers{ .token = it.index },
72 });
73 return node;
70 .decls = try parseContainerMembers(arena, it, tree),
71 .eof_token = eatToken(it, .Eof) orelse {
72 try tree.errors.push(AstError{
73 .ExpectedContainerMembers = .{ .token = it.index },
74 });
75 return error.ParseError;
76 },
7477 };
7578 return node;
7679}
......@@ -303,7 +306,17 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
303306fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
304307 const cc = parseFnCC(arena, it, tree);
305308 const fn_token = eatToken(it, .Keyword_fn) orelse {
306 if (cc == null) return null else return error.ParseError;
309 if (cc) |fnCC| {
310 if (fnCC == .Extern) {
311 putBackToken(it, fnCC.Extern); // 'extern' is also used in ContainerDecl
312 } else {
313 try tree.errors.push(AstError{
314 .ExpectedToken = .{ .token = it.index, .expected_id = .Keyword_fn },
315 });
316 return error.ParseError;
317 }
318 }
319 return null;
307320 };
308321 const name_token = eatToken(it, .Identifier);
309322 const lparen = try expectToken(it, tree, .LParen);
......@@ -390,7 +403,7 @@ fn parseVarDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
390403 .visib_token = null,
391404 .thread_local_token = null,
392405 .name_token = name_token,
393 .eq_token = eq_token orelse undefined,
406 .eq_token = eq_token,
394407 .mut_token = mut_token,
395408 .comptime_token = null,
396409 .extern_export_token = null,
......@@ -2177,7 +2190,7 @@ fn parsePrefixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
21772190 node.* = Node.PrefixOp{
21782191 .op_token = token.index,
21792192 .op = op,
2180 .rhs = undefined,
2193 .rhs = undefined, // set by caller
21812194 };
21822195 return &node.base;
21832196}
......@@ -2819,8 +2832,8 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
28192832 .doc_comments = null,
28202833 .visib_token = null,
28212834 .use_token = token,
2822 .expr = undefined,
2823 .semicolon_token = undefined,
2835 .expr = undefined, // set by caller
2836 .semicolon_token = undefined, // set by caller
28242837 };
28252838 return &node.base;
28262839}
......@@ -2979,9 +2992,9 @@ fn createInfixOp(arena: *Allocator, index: TokenIndex, op: Node.InfixOp.Op) !*No
29792992 const node = try arena.create(Node.InfixOp);
29802993 node.* = Node.InfixOp{
29812994 .op_token = index,
2982 .lhs = undefined,
2995 .lhs = undefined, // set by caller
29832996 .op = op,
2984 .rhs = undefined,
2997 .rhs = undefined, // set by caller
29852998 };
29862999 return &node.base;
29873000}
lib/std/zig/parser_test.zig+15
......@@ -2714,6 +2714,13 @@ test "zig fmt: top level doc comments" {
27142714 );
27152715}
27162716
2717test "zig fmt: extern without container keyword returns error" {
2718 try testError(
2719 \\const container = extern {};
2720 \\
2721 );
2722}
2723
27172724const std = @import("std");
27182725const mem = std.mem;
27192726const warn = std.debug.warn;
......@@ -2820,3 +2827,11 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
28202827fn testCanonical(source: []const u8) !void {
28212828 return testTransform(source, source);
28222829}
2830
2831fn testError(source: []const u8) !void {
2832 var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
2833 const tree = try std.zig.parse(&fixed_allocator.allocator, source);
2834 defer tree.deinit();
2835
2836 std.testing.expect(tree.errors.len != 0);
2837}
lib/std/zig/render.zig+1-1
......@@ -2020,7 +2020,7 @@ fn renderVarDecl(
20202020
20212021 if (var_decl.init_node) |init_node| {
20222022 const s = if (init_node.id == .MultilineStringLiteral) Space.None else Space.Space;
2023 try renderToken(tree, stream, var_decl.eq_token, indent, start_col, s); // =
2023 try renderToken(tree, stream, var_decl.eq_token.?, indent, start_col, s); // =
20242024 try renderExpression(allocator, stream, tree, indent, start_col, init_node, Space.None);
20252025 }
20262026