authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-01-13 13:38:31-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-01-13 13:38:31-05:00
logc774c9376a20c2fde7db4fed94bdea5ae94e1494
treead5ad8ee9f19a71667d79e9c2f65a0ad527cc9b1
parent2be12b24bcafd4b2785d61dcfbaa27f9e3cc705d
parent5880eb3a75b41d8ac26faa0185a83e001197a6e4
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #3957 from xackus/stage2_parser_3799

stage2 parser: fix segfault on extern block

4 files changed, 56 insertions(+), 27 deletions(-)

lib/std/zig/ast.zig+2-1
...@@ -10,6 +10,7 @@ pub const TokenIndex = usize;...@@ -10,6 +10,7 @@ pub const TokenIndex = usize;
10pub const Tree = struct {10pub const Tree = struct {
11 source: []const u8,11 source: []const u8,
12 tokens: TokenList,12 tokens: TokenList,
13 /// undefined on parse error (errors not empty)
13 root_node: *Node.Root,14 root_node: *Node.Root,
14 arena_allocator: std.heap.ArenaAllocator,15 arena_allocator: std.heap.ArenaAllocator,
15 errors: ErrorList,16 errors: ErrorList,
...@@ -612,7 +613,7 @@ pub const Node = struct {...@@ -612,7 +613,7 @@ pub const Node = struct {
612 visib_token: ?TokenIndex,613 visib_token: ?TokenIndex,
613 thread_local_token: ?TokenIndex,614 thread_local_token: ?TokenIndex,
614 name_token: TokenIndex,615 name_token: TokenIndex,
615 eq_token: TokenIndex,616 eq_token: ?TokenIndex,
616 mut_token: TokenIndex,617 mut_token: TokenIndex,
617 comptime_token: ?TokenIndex,618 comptime_token: ?TokenIndex,
618 extern_export_token: ?TokenIndex,619 extern_export_token: ?TokenIndex,
lib/std/zig/parse.zig+38-25
...@@ -13,7 +13,7 @@ pub const Error = error{ParseError} || Allocator.Error;...@@ -13,7 +13,7 @@ pub const Error = error{ParseError} || Allocator.Error;
1313
14/// Result should be freed with tree.deinit() when there are14/// Result should be freed with tree.deinit() when there are
15/// no more references to any of the tokens or nodes.15/// no more references to any of the tokens or nodes.
16pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {16pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
17 const tree = blk: {17 const tree = blk: {
18 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists18 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists
19 // from being initialized with a pointer to this `arena`, which is created on19 // from being initialized with a pointer to this `arena`, which is created on
...@@ -48,29 +48,32 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {...@@ -48,29 +48,32 @@ pub fn parse(allocator: *Allocator, source: []const u8) !*Tree {
4848
49 while (it.peek().?.id == .LineComment) _ = it.next();49 while (it.peek().?.id == .LineComment) _ = it.next();
5050
51 tree.root_node = try parseRoot(arena, &it, tree);51 tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {
52 switch (err) {
53 error.ParseError => {
54 assert(tree.errors.len != 0);
55 break :blk undefined;
56 },
57 error.OutOfMemory => {
58 return error.OutOfMemory;
59 },
60 }
61 };
62
52 return tree;63 return tree;
53}64}
5465
55/// Root <- skip ContainerMembers eof66/// Root <- skip ContainerMembers eof
56fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {67fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {
57 const node = try arena.create(Node.Root);68 const node = try arena.create(Node.Root);
58 node.* = Node.Root{69 node.* = Node.Root{
59 .decls = undefined,70 .decls = try parseContainerMembers(arena, it, tree),
60 .eof_token = undefined,71 .eof_token = eatToken(it, .Eof) orelse {
61 };72 try tree.errors.push(AstError{
62 node.decls = parseContainerMembers(arena, it, tree) catch |err| {73 .ExpectedContainerMembers = .{ .token = it.index },
63 // TODO: Switch on the error type74 });
64 // https://github.com/ziglang/zig/issues/247375 return error.ParseError;
65 if (err == error.ParseError) return node;76 },
66 assert(err == Allocator.Error.OutOfMemory);
67 return Allocator.Error.OutOfMemory;
68 };
69 node.eof_token = eatToken(it, .Eof) orelse {
70 try tree.errors.push(AstError{
71 .ExpectedContainerMembers = AstError.ExpectedContainerMembers{ .token = it.index },
72 });
73 return node;
74 };77 };
75 return node;78 return node;
76}79}
...@@ -303,7 +306,17 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -303,7 +306,17 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
303fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {306fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
304 const cc = parseFnCC(arena, it, tree);307 const cc = parseFnCC(arena, it, tree);
305 const fn_token = eatToken(it, .Keyword_fn) orelse {308 const fn_token = eatToken(it, .Keyword_fn) orelse {
306 if (cc == null) return null else return error.ParseError;309 if (cc) |fnCC| {
310 if (fnCC == .Extern) {
311 putBackToken(it, fnCC.Extern); // 'extern' is also used in ContainerDecl
312 } else {
313 try tree.errors.push(AstError{
314 .ExpectedToken = .{ .token = it.index, .expected_id = .Keyword_fn },
315 });
316 return error.ParseError;
317 }
318 }
319 return null;
307 };320 };
308 const name_token = eatToken(it, .Identifier);321 const name_token = eatToken(it, .Identifier);
309 const lparen = try expectToken(it, tree, .LParen);322 const lparen = try expectToken(it, tree, .LParen);
...@@ -390,7 +403,7 @@ fn parseVarDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -390,7 +403,7 @@ fn parseVarDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
390 .visib_token = null,403 .visib_token = null,
391 .thread_local_token = null,404 .thread_local_token = null,
392 .name_token = name_token,405 .name_token = name_token,
393 .eq_token = eq_token orelse undefined,406 .eq_token = eq_token,
394 .mut_token = mut_token,407 .mut_token = mut_token,
395 .comptime_token = null,408 .comptime_token = null,
396 .extern_export_token = null,409 .extern_export_token = null,
...@@ -2177,7 +2190,7 @@ fn parsePrefixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -2177,7 +2190,7 @@ fn parsePrefixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2177 node.* = Node.PrefixOp{2190 node.* = Node.PrefixOp{
2178 .op_token = token.index,2191 .op_token = token.index,
2179 .op = op,2192 .op = op,
2180 .rhs = undefined,2193 .rhs = undefined, // set by caller
2181 };2194 };
2182 return &node.base;2195 return &node.base;
2183}2196}
...@@ -2819,8 +2832,8 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -2819,8 +2832,8 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2819 .doc_comments = null,2832 .doc_comments = null,
2820 .visib_token = null,2833 .visib_token = null,
2821 .use_token = token,2834 .use_token = token,
2822 .expr = undefined,2835 .expr = undefined, // set by caller
2823 .semicolon_token = undefined,2836 .semicolon_token = undefined, // set by caller
2824 };2837 };
2825 return &node.base;2838 return &node.base;
2826}2839}
...@@ -2979,9 +2992,9 @@ fn createInfixOp(arena: *Allocator, index: TokenIndex, op: Node.InfixOp.Op) !*No...@@ -2979,9 +2992,9 @@ fn createInfixOp(arena: *Allocator, index: TokenIndex, op: Node.InfixOp.Op) !*No
2979 const node = try arena.create(Node.InfixOp);2992 const node = try arena.create(Node.InfixOp);
2980 node.* = Node.InfixOp{2993 node.* = Node.InfixOp{
2981 .op_token = index,2994 .op_token = index,
2982 .lhs = undefined,2995 .lhs = undefined, // set by caller
2983 .op = op,2996 .op = op,
2984 .rhs = undefined,2997 .rhs = undefined, // set by caller
2985 };2998 };
2986 return &node.base;2999 return &node.base;
2987}3000}
lib/std/zig/parser_test.zig+15
...@@ -2714,6 +2714,13 @@ test "zig fmt: top level doc comments" {...@@ -2714,6 +2714,13 @@ test "zig fmt: top level doc comments" {
2714 );2714 );
2715}2715}
27162716
2717test "zig fmt: extern without container keyword returns error" {
2718 try testError(
2719 \\const container = extern {};
2720 \\
2721 );
2722}
2723
2717const std = @import("std");2724const std = @import("std");
2718const mem = std.mem;2725const mem = std.mem;
2719const warn = std.debug.warn;2726const warn = std.debug.warn;
...@@ -2820,3 +2827,11 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {...@@ -2820,3 +2827,11 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
2820fn testCanonical(source: []const u8) !void {2827fn testCanonical(source: []const u8) !void {
2821 return testTransform(source, source);2828 return testTransform(source, source);
2822}2829}
2830
2831fn testError(source: []const u8) !void {
2832 var fixed_allocator = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
2833 const tree = try std.zig.parse(&fixed_allocator.allocator, source);
2834 defer tree.deinit();
2835
2836 std.testing.expect(tree.errors.len != 0);
2837}
lib/std/zig/render.zig+1-1
...@@ -2020,7 +2020,7 @@ fn renderVarDecl(...@@ -2020,7 +2020,7 @@ fn renderVarDecl(
20202020
2021 if (var_decl.init_node) |init_node| {2021 if (var_decl.init_node) |init_node| {
2022 const s = if (init_node.id == .MultilineStringLiteral) Space.None else Space.Space;2022 const s = if (init_node.id == .MultilineStringLiteral) Space.None else Space.Space;
2023 try renderToken(tree, stream, var_decl.eq_token, indent, start_col, s); // =2023 try renderToken(tree, stream, var_decl.eq_token.?, indent, start_col, s); // =
2024 try renderExpression(allocator, stream, tree, indent, start_col, init_node, Space.None);2024 try renderExpression(allocator, stream, tree, indent, start_col, init_node, Space.None);
2025 }2025 }
20262026