Commit 758de71d9769b3539f042aea8821c3ac18bed74c | zig

author	Veikka Tuominen <git@vexu.eu> 2020-05-16 15:56:03+03:00
committer	GitHub <noreply@github.com> 2020-05-16 15:56:03+03:00
log	758de71d9769b3539f042aea8821c3ac18bed74c
tree	9767474967f6944d28d1b06d3f01c2cc26ad886a
parent	092f726eeca1b73d29efb7074bc3302f974aba7d
parent	5ac684ec502096196f05a1c6ed1841b886d6db92
signature	Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5353 from Vexu/parser

Self-hosted parser fixes

7 files changed, 171 insertions(+), 115 deletions(-)

doc
- langref.html.in
lib
- std
  - zig
    ast.zig
    parse.zig
    parser_test.zig
    render.zig
src-self-hosted
- translate_c.zig
src
- parser.cpp

doc/langref.html.in+5-5

lib/std/zig/ast.zig+19-6

...	@@ -986,14 +986,22 @@ pub const Node = struct {	...	@@ -986,14 +986,22 @@ pub const Node = struct {
986	comptime_token: ?TokenIndex,	986	comptime_token: ?TokenIndex,
987	noalias_token: ?TokenIndex,	987	noalias_token: ?TokenIndex,
988	name_token: ?TokenIndex,	988	name_token: ?TokenIndex,
989	type_node: *Node,	989	param_type: ParamType,
990	var_args_token: ?TokenIndex,	990
		991	pub const ParamType = union(enum) {
		992	var_type: *Node,
		993	var_args: TokenIndex,
		994	type_expr: *Node,
		995	};
991		996
992	pub fn iterate(self: ParamDecl, index: usize) ?Node {	997	pub fn iterate(self: ParamDecl, index: usize) ?Node {
993	var i = index;	998	var i = index;
994		999
995	if (i < 1) {	1000	if (i < 1) {
996	return if (self.var_args_token == null) self.type_node else null;	1001	switch (self.param_type) {
		1002	.var_args => return null,
		1003	.var_type, .type_expr => \|node\| return node,
		1004	}
997	}	1005	}
998	i -= 1;	1006	i -= 1;
999		1007
...	@@ -1004,12 +1012,17 @@ pub const Node = struct {	...	@@ -1004,12 +1012,17 @@ pub const Node = struct {
1004	if (self.comptime_token) \|comptime_token\| return comptime_token;	1012	if (self.comptime_token) \|comptime_token\| return comptime_token;
1005	if (self.noalias_token) \|noalias_token\| return noalias_token;	1013	if (self.noalias_token) \|noalias_token\| return noalias_token;
1006	if (self.name_token) \|name_token\| return name_token;	1014	if (self.name_token) \|name_token\| return name_token;
1007	return self.type_node.firstToken();	1015	switch (self.param_type) {
		1016	.var_args => \|tok\| return tok,
		1017	.var_type, .type_expr => \|node\| return node.firstToken(),
		1018	}
1008	}	1019	}
1009		1020
1010	pub fn lastToken(self: *const ParamDecl) TokenIndex {	1021	pub fn lastToken(self: *const ParamDecl) TokenIndex {
1011	if (self.var_args_token) \|var_args_token\| return var_args_token;	1022	switch (self.param_type) {
1012	return self.type_node.lastToken();	1023	.var_args => \|tok\| return tok,
		1024	.var_type, .type_expr => \|node\| return node.lastToken(),
		1025	}
1013	}	1026	}
1014	};	1027	};
1015		1028

lib/std/zig/parse.zig+84-88

...	@@ -57,16 +57,10 @@ pub fn parse(allocator: Allocator, source: []const u8) Allocator.Error!Tree {	...	@@ -57,16 +57,10 @@ pub fn parse(allocator: Allocator, source: []const u8) Allocator.Error!Tree {
57	fn parseRoot(arena: Allocator, it: TokenIterator, tree: Tree) Allocator.Error!Node.Root {	57	fn parseRoot(arena: Allocator, it: TokenIterator, tree: Tree) Allocator.Error!Node.Root {
58	const node = try arena.create(Node.Root);	58	const node = try arena.create(Node.Root);
59	node.* = .{	59	node.* = .{
60	.decls = try parseContainerMembers(arena, it, tree),	60	.decls = try parseContainerMembers(arena, it, tree, true),
61	.eof_token = eatToken(it, .Eof) orelse blk: {	61	// parseContainerMembers will try to skip as much
62	// parseContainerMembers will try to skip as much	62	// invalid tokens as it can so this can only be the EOF
63	// invalid tokens as it can so this can only be a '}'	63	.eof_token = eatToken(it, .Eof).?,
64	const tok = eatToken(it, .RBrace).?;
65	try tree.errors.push(.{
66	.ExpectedContainerMembers = .{ .token = tok },
67	});
68	break :blk tok;
69	},
70	};	64	};
71	return node;	65	return node;
72	}	66	}
...	@@ -75,10 +69,10 @@ fn parseRoot(arena: Allocator, it: TokenIterator, tree: *Tree) Allocator.Error	...	@@ -75,10 +69,10 @@ fn parseRoot(arena: Allocator, it: TokenIterator, tree: *Tree) Allocator.Error
75	/// <- TestDecl ContainerMembers	69	/// <- TestDecl ContainerMembers
76	/// / TopLevelComptime ContainerMembers	70	/// / TopLevelComptime ContainerMembers
77	/// / KEYWORD_pub? TopLevelDecl ContainerMembers	71	/// / KEYWORD_pub? TopLevelDecl ContainerMembers
78	/// / KEYWORD_pub? ContainerField COMMA ContainerMembers	72	/// / ContainerField COMMA ContainerMembers
79	/// / KEYWORD_pub? ContainerField	73	/// / ContainerField
80	/// /	74	/// /
81	fn parseContainerMembers(arena: Allocator, it: TokenIterator, tree: *Tree) !Node.Root.DeclList {	75	fn parseContainerMembers(arena: Allocator, it: TokenIterator, tree: *Tree, top_level: bool) !Node.Root.DeclList {
82	var list = Node.Root.DeclList.init(arena);	76	var list = Node.Root.DeclList.init(arena);
83		77
84	var field_state: union(enum) {	78	var field_state: union(enum) {
...	@@ -205,9 +199,15 @@ fn parseContainerMembers(arena: Allocator, it: TokenIterator, tree: *Tree) !No	...	@@ -205,9 +199,15 @@ fn parseContainerMembers(arena: Allocator, it: TokenIterator, tree: *Tree) !No
205	// try to continue parsing	199	// try to continue parsing
206	const index = it.index;	200	const index = it.index;
207	findNextContainerMember(it);	201	findNextContainerMember(it);
208	switch (it.peek().?.id) {	202	const next = it.peek().?.id;
209	.Eof, .RBrace => break,	203	switch (next) {
		204	.Eof => break,
210	else => {	205	else => {
		206	if (next == .RBrace) {
		207	if (!top_level) break;
		208	_ = nextToken(it);
		209	}
		210
211	// add error and continue	211	// add error and continue
212	try tree.errors.push(.{	212	try tree.errors.push(.{
213	.ExpectedToken = .{ .token = index, .expected_id = .Comma },	213	.ExpectedToken = .{ .token = index, .expected_id = .Comma },
...	@@ -228,12 +228,18 @@ fn parseContainerMembers(arena: Allocator, it: TokenIterator, tree: *Tree) !No	...	@@ -228,12 +228,18 @@ fn parseContainerMembers(arena: Allocator, it: TokenIterator, tree: *Tree) !No
228	});	228	});
229	}	229	}
230		230
231	switch (it.peek().?.id) {	231	const next = it.peek().?.id;
232	.Eof, .RBrace => break,	232	switch (next) {
		233	.Eof => break,
233	else => {	234	else => {
		235	const index = it.index;
		236	if (next == .RBrace) {
		237	if (!top_level) break;
		238	_ = nextToken(it);
		239	}
		240
234	// this was likely not supposed to end yet,	241	// this was likely not supposed to end yet,
235	// try to find the next declaration	242	// try to find the next declaration
236	const index = it.index;
237	findNextContainerMember(it);	243	findNextContainerMember(it);
238	try tree.errors.push(.{	244	try tree.errors.push(.{
239	.ExpectedContainerMembers = .{ .token = index },	245	.ExpectedContainerMembers = .{ .token = index },
...	@@ -278,7 +284,10 @@ fn findNextContainerMember(it: *TokenIterator) void {	...	@@ -278,7 +284,10 @@ fn findNextContainerMember(it: *TokenIterator) void {
278	}	284	}
279	},	285	},
280	.LParen, .LBracket, .LBrace => level += 1,	286	.LParen, .LBracket, .LBrace => level += 1,
281	.RParen, .RBracket, .RBrace => {	287	.RParen, .RBracket => {
		288	if (level != 0) level -= 1;
		289	},
		290	.RBrace => {
282	if (level == 0) {	291	if (level == 0) {
283	// end of container, exit	292	// end of container, exit
284	putBackToken(it, tok.index);	293	putBackToken(it, tok.index);
...	@@ -402,20 +411,16 @@ fn parseTopLevelDecl(arena: Allocator, it: TokenIterator, tree: Tree) !?Node	...	@@ -402,20 +411,16 @@ fn parseTopLevelDecl(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
402	fn_node.*.extern_export_inline_token = extern_export_inline_token;	411	fn_node.*.extern_export_inline_token = extern_export_inline_token;
403	fn_node.*.lib_name = lib_name;	412	fn_node.*.lib_name = lib_name;
404	if (eatToken(it, .Semicolon)) \|_\| return node;	413	if (eatToken(it, .Semicolon)) \|_\| return node;
405	if (parseBlock(arena, it, tree) catch \|err\| switch (err) {	414
406	error.OutOfMemory => return error.OutOfMemory,	415	if (try expectNodeRecoverable(arena, it, tree, parseBlock, .{
407	// since parseBlock only return error.ParseError on	416	// since parseBlock only return error.ParseError on
408	// a missing '}' we can assume this function was	417	// a missing '}' we can assume this function was
409	// supposed to end here.	418	// supposed to end here.
410	error.ParseError => return node,	419	.ExpectedSemiOrLBrace = .{ .token = it.index },
411	}) \|body_node\| {	420	})) \|body_node\| {
412	fn_node.body_node = body_node;	421	fn_node.body_node = body_node;
413	return node;
414	}	422	}
415	try tree.errors.push(.{	423	return node;
416	.ExpectedSemiOrLBrace = .{ .token = it.index },
417	});
418	return error.ParseError;
419	}	424	}
420		425
421	if (extern_export_inline_token) \|token\| {	426	if (extern_export_inline_token) \|token\| {
...	@@ -490,14 +495,11 @@ fn parseFnProto(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	...	@@ -490,14 +495,11 @@ fn parseFnProto(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
490	const exclamation_token = eatToken(it, .Bang);	495	const exclamation_token = eatToken(it, .Bang);
491		496
492	const return_type_expr = (try parseVarType(arena, it, tree)) orelse	497	const return_type_expr = (try parseVarType(arena, it, tree)) orelse
493	(try parseTypeExpr(arena, it, tree)) orelse blk: {	498	try expectNodeRecoverable(arena, it, tree, parseTypeExpr, .{
494	try tree.errors.push(.{
495	.ExpectedReturnType = .{ .token = it.index },
496	});
497	// most likely the user forgot to specify the return type.	499	// most likely the user forgot to specify the return type.
498	// Mark return type as invalid and try to continue.	500	// Mark return type as invalid and try to continue.
499	break :blk null;	501	.ExpectedReturnType = .{ .token = it.index },
500	};	502	});
501		503
502	// TODO https://github.com/ziglang/zig/issues/3750	504	// TODO https://github.com/ziglang/zig/issues/3750
503	const R = Node.FnProto.ReturnType;	505	const R = Node.FnProto.ReturnType;
...	@@ -508,9 +510,10 @@ fn parseFnProto(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	...	@@ -508,9 +510,10 @@ fn parseFnProto(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
508	else	510	else
509	R{ .Explicit = return_type_expr.? };	511	R{ .Explicit = return_type_expr.? };
510		512
511	const var_args_token = if (params.len > 0)	513	const var_args_token = if (params.len > 0) blk: {
512	params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token	514	const param_type = params.at(params.len - 1).*.cast(Node.ParamDecl).?.param_type;
513	else	515	break :blk if (param_type == .var_args) param_type.var_args else null;
		516	} else
514	null;	517	null;
515		518
516	const fn_proto_node = try arena.create(Node.FnProto);	519	const fn_proto_node = try arena.create(Node.FnProto);
...	@@ -707,12 +710,7 @@ fn parseStatement(arena: Allocator, it: TokenIterator, tree: Tree) Error!?No	...	@@ -707,12 +710,7 @@ fn parseStatement(arena: Allocator, it: TokenIterator, tree: Tree) Error!?No
707	if (try parseLabeledStatement(arena, it, tree)) \|node\| return node;	710	if (try parseLabeledStatement(arena, it, tree)) \|node\| return node;
708	if (try parseSwitchExpr(arena, it, tree)) \|node\| return node;	711	if (try parseSwitchExpr(arena, it, tree)) \|node\| return node;
709	if (try parseAssignExpr(arena, it, tree)) \|node\| {	712	if (try parseAssignExpr(arena, it, tree)) \|node\| {
710	_ = eatToken(it, .Semicolon) orelse {	713	_ = try expectTokenRecoverable(it, tree, .Semicolon);
711	try tree.errors.push(.{
712	.ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
713	});
714	// pretend we saw a semicolon and continue parsing
715	};
716	return node;	714	return node;
717	}	715	}
718		716
...	@@ -727,16 +725,12 @@ fn parseIfStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node	...	@@ -727,16 +725,12 @@ fn parseIfStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
727	const if_prefix = if_node.cast(Node.If).?;	725	const if_prefix = if_node.cast(Node.If).?;
728		726
729	const block_expr = (try parseBlockExpr(arena, it, tree));	727	const block_expr = (try parseBlockExpr(arena, it, tree));
730	const assign_expr = if (block_expr == null) blk: {	728	const assign_expr = if (block_expr == null)
731	break :blk (try parseAssignExpr(arena, it, tree)) orelse null;	729	try expectNode(arena, it, tree, parseAdditionExpr, .{
732	} else null;
733
734	if (block_expr == null and assign_expr == null) {
735	try tree.errors.push(.{
736	.ExpectedBlockOrAssignment = .{ .token = it.index },	730	.ExpectedBlockOrAssignment = .{ .token = it.index },
737	});	731	})
738	return error.ParseError;	732	else
739	}	733	null;
740		734
741	const semicolon = if (assign_expr != null) eatToken(it, .Semicolon) else null;	735	const semicolon = if (assign_expr != null) eatToken(it, .Semicolon) else null;
742		736
...	@@ -773,10 +767,9 @@ fn parseIfStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node	...	@@ -773,10 +767,9 @@ fn parseIfStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
773	try tree.errors.push(.{	767	try tree.errors.push(.{
774	.ExpectedSemiOrElse = .{ .token = it.index },	768	.ExpectedSemiOrElse = .{ .token = it.index },
775	});	769	});
776	return error.ParseError;
777	}	770	}
778		771
779	unreachable;	772	return if_node;
780	}	773	}
781		774
782	/// LabeledStatement <- BlockLabel? (Block / LoopStatement)	775	/// LabeledStatement <- BlockLabel? (Block / LoopStatement)
...	@@ -882,7 +875,8 @@ fn parseForStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node	...	@@ -882,7 +875,8 @@ fn parseForStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
882	try tree.errors.push(.{	875	try tree.errors.push(.{
883	.ExpectedSemiOrElse = .{ .token = it.index },	876	.ExpectedSemiOrElse = .{ .token = it.index },
884	});	877	});
885	return null;	878
		879	return node;
886	}	880	}
887		881
888	return null;	882	return null;
...	@@ -944,7 +938,8 @@ fn parseWhileStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?No	...	@@ -944,7 +938,8 @@ fn parseWhileStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?No
944	try tree.errors.push(.{	938	try tree.errors.push(.{
945	.ExpectedSemiOrElse = .{ .token = it.index },	939	.ExpectedSemiOrElse = .{ .token = it.index },
946	});	940	});
947	return null;	941
		942	return node;
948	}	943	}
949		944
950	return null;	945	return null;
...	@@ -956,12 +951,7 @@ fn parseWhileStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?No	...	@@ -956,12 +951,7 @@ fn parseWhileStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?No
956	fn parseBlockExprStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	951	fn parseBlockExprStatement(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
957	if (try parseBlockExpr(arena, it, tree)) \|node\| return node;	952	if (try parseBlockExpr(arena, it, tree)) \|node\| return node;
958	if (try parseAssignExpr(arena, it, tree)) \|node\| {	953	if (try parseAssignExpr(arena, it, tree)) \|node\| {
959	_ = eatToken(it, .Semicolon) orelse {	954	_ = try expectTokenRecoverable(it, tree, .Semicolon);
960	try tree.errors.push(.{
961	.ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
962	});
963	// pretend we saw a semicolon and continue parsing
964	};
965	return node;	955	return node;
966	}	956	}
967	return null;	957	return null;
...	@@ -1478,17 +1468,19 @@ fn parsePrimaryTypeExpr(arena: Allocator, it: TokenIterator, tree: Tree) !?N	...	@@ -1478,17 +1468,19 @@ fn parsePrimaryTypeExpr(arena: Allocator, it: TokenIterator, tree: Tree) !?N
1478	return &node.base;	1468	return &node.base;
1479	}	1469	}
1480	if (eatToken(it, .Keyword_error)) \|token\| {	1470	if (eatToken(it, .Keyword_error)) \|token\| {
1481	const period = try expectToken(it, tree, .Period);	1471	const period = try expectTokenRecoverable(it, tree, .Period);
1482	const identifier = try expectNode(arena, it, tree, parseIdentifier, .{	1472	const identifier = try expectNodeRecoverable(arena, it, tree, parseIdentifier, .{
1483	.ExpectedIdentifier = .{ .token = it.index },	1473	.ExpectedIdentifier = .{ .token = it.index },
1484	});	1474	});
1485	const global_error_set = try createLiteral(arena, Node.ErrorType, token);	1475	const global_error_set = try createLiteral(arena, Node.ErrorType, token);
		1476	if (period == null or identifier == null) return global_error_set;
		1477
1486	const node = try arena.create(Node.InfixOp);	1478	const node = try arena.create(Node.InfixOp);
1487	node.* = .{	1479	node.* = .{
1488	.op_token = period,	1480	.op_token = period.?,
1489	.lhs = global_error_set,	1481	.lhs = global_error_set,
1490	.op = .Period,	1482	.op = .Period,
1491	.rhs = identifier,	1483	.rhs = identifier.?,
1492	};	1484	};
1493	return &node.base;	1485	return &node.base;
1494	}	1486	}
...	@@ -1948,15 +1940,8 @@ fn parseParamDecl(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	...	@@ -1948,15 +1940,8 @@ fn parseParamDecl(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
1948	.comptime_token = comptime_token,	1940	.comptime_token = comptime_token,
1949	.noalias_token = noalias_token,	1941	.noalias_token = noalias_token,
1950	.name_token = name_token,	1942	.name_token = name_token,
1951	// TODO: These should be squished into a ParamType enum	1943	.param_type = param_type,
1952	.type_node = undefined,
1953	.var_args_token = null,
1954	};	1944	};
1955	switch (param_type) {
1956	.VarType => \|node\| param_decl.type_node = node,
1957	.TypeExpr => \|node\| param_decl.type_node = node,
1958	.VarArgs => \|token\| param_decl.var_args_token = token,
1959	}
1960	return &param_decl.base;	1945	return &param_decl.base;
1961	}	1946	}
1962		1947
...	@@ -1964,20 +1949,15 @@ fn parseParamDecl(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	...	@@ -1964,20 +1949,15 @@ fn parseParamDecl(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
1964	/// <- KEYWORD_var	1949	/// <- KEYWORD_var
1965	/// / DOT3	1950	/// / DOT3
1966	/// / TypeExpr	1951	/// / TypeExpr
1967	fn parseParamType(arena: Allocator, it: TokenIterator, tree: *Tree) !?ParamType {	1952	fn parseParamType(arena: Allocator, it: TokenIterator, tree: *Tree) !?Node.ParamDecl.ParamType {
1968	if (try parseVarType(arena, it, tree)) \|node\| return ParamType{ .VarType = node };	1953	// TODO cast from tuple to error union is broken
1969	if (eatToken(it, .Ellipsis3)) \|token\| return ParamType{ .VarArgs = token };	1954	const P = Node.ParamDecl.ParamType;
1970	if (try parseTypeExpr(arena, it, tree)) \|node\| return ParamType{ .TypeExpr = node };	1955	if (try parseVarType(arena, it, tree)) \|node\| return P{ .var_type = node };
		1956	if (eatToken(it, .Ellipsis3)) \|token\| return P{ .var_args = token };
		1957	if (try parseTypeExpr(arena, it, tree)) \|node\| return P{ .type_expr = node };
1971	return null;	1958	return null;
1972	}	1959	}
1973		1960
1974	// TODO: Move to ast.Node.ParamDecl.ParamType
1975	const ParamType = union(enum) {
1976	VarType: *Node,
1977	VarArgs: TokenIndex,
1978	TypeExpr: *Node,
1979	};
1980
1981	/// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?	1961	/// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload?
1982	fn parseIfPrefix(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	1962	fn parseIfPrefix(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
1983	const if_token = eatToken(it, .Keyword_if) orelse return null;	1963	const if_token = eatToken(it, .Keyword_if) orelse return null;
...	@@ -2778,7 +2758,7 @@ fn parsePtrTypeStart(arena: Allocator, it: TokenIterator, tree: Tree) !?Node	...	@@ -2778,7 +2758,7 @@ fn parsePtrTypeStart(arena: Allocator, it: TokenIterator, tree: Tree) !?Node
2778	fn parseContainerDeclAuto(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {	2758	fn parseContainerDeclAuto(arena: Allocator, it: TokenIterator, tree: Tree) !?Node {
2779	const node = (try parseContainerDeclType(arena, it, tree)) orelse return null;	2759	const node = (try parseContainerDeclType(arena, it, tree)) orelse return null;
2780	const lbrace = try expectToken(it, tree, .LBrace);	2760	const lbrace = try expectToken(it, tree, .LBrace);
2781	const members = try parseContainerMembers(arena, it, tree);	2761	const members = try parseContainerMembers(arena, it, tree, false);
2782	const rbrace = try expectToken(it, tree, .RBrace);	2762	const rbrace = try expectToken(it, tree, .RBrace);
2783		2763
2784	const decl_type = node.cast(Node.ContainerDecl).?;	2764	const decl_type = node.cast(Node.ContainerDecl).?;
...	@@ -3250,6 +3230,11 @@ fn eatAnnotatedToken(it: *TokenIterator, id: Token.Id) ?AnnotatedToken {	...	@@ -3250,6 +3230,11 @@ fn eatAnnotatedToken(it: *TokenIterator, id: Token.Id) ?AnnotatedToken {
3250	}	3230	}
3251		3231
3252	fn expectToken(it: TokenIterator, tree: Tree, id: Token.Id) Error!TokenIndex {	3232	fn expectToken(it: TokenIterator, tree: Tree, id: Token.Id) Error!TokenIndex {
		3233	return (try expectTokenRecoverable(it, tree, id)) orelse
		3234	error.ParseError;
		3235	}
		3236
		3237	fn expectTokenRecoverable(it: TokenIterator, tree: Tree, id: Token.Id) !?TokenIndex {
3253	const token = nextToken(it);	3238	const token = nextToken(it);
3254	if (token.ptr.id != id) {	3239	if (token.ptr.id != id) {
3255	try tree.errors.push(.{	3240	try tree.errors.push(.{
...	@@ -3257,7 +3242,7 @@ fn expectToken(it: TokenIterator, tree: Tree, id: Token.Id) Error!TokenIndex {	...	@@ -3257,7 +3242,7 @@ fn expectToken(it: TokenIterator, tree: Tree, id: Token.Id) Error!TokenIndex {
3257	});	3242	});
3258	// go back so that we can recover properly	3243	// go back so that we can recover properly
3259	putBackToken(it, token.index);	3244	putBackToken(it, token.index);
3260	return error.ParseError;	3245	return null;
3261	}	3246	}
3262	return token.index;	3247	return token.index;
3263	}	3248	}
...	@@ -3297,9 +3282,20 @@ fn expectNode(	...	@@ -3297,9 +3282,20 @@ fn expectNode(
3297	parseFn: NodeParseFn,	3282	parseFn: NodeParseFn,
3298	err: AstError, // if parsing fails	3283	err: AstError, // if parsing fails
3299	) Error!*Node {	3284	) Error!*Node {
		3285	return (try expectNodeRecoverable(arena, it, tree, parseFn, err)) orelse
		3286	return error.ParseError;
		3287	}
		3288
		3289	fn expectNodeRecoverable(
		3290	arena: *Allocator,
		3291	it: *TokenIterator,
		3292	tree: *Tree,
		3293	parseFn: NodeParseFn,
		3294	err: AstError, // if parsing fails
		3295	) !?*Node {
3300	return (try parseFn(arena, it, tree)) orelse {	3296	return (try parseFn(arena, it, tree)) orelse {
3301	try tree.errors.push(err);	3297	try tree.errors.push(err);
3302	return error.ParseError;	3298	return null;
3303	};	3299	};
3304	}	3300	}
3305		3301

lib/std/zig/parser_test.zig+52

...	@@ -148,6 +148,58 @@ test "recovery: invalid parameter" {	...	@@ -148,6 +148,58 @@ test "recovery: invalid parameter" {
148	});	148	});
149	}	149	}
150		150
		151	test "recovery: extra '}' at top level" {
		152	try testError(
		153	\\}}}
		154	\\test "" {
		155	\\ a && b;
		156	\\}
		157	, &[_]Error{
		158	.ExpectedContainerMembers,
		159	.ExpectedContainerMembers,
		160	.ExpectedContainerMembers,
		161	.InvalidAnd,
		162	});
		163	}
		164
		165	test "recovery: mismatched bracket at top level" {
		166	try testError(
		167	\\const S = struct {
		168	\\ arr: 128]?G
		169	\\};
		170	, &[_]Error{
		171	.ExpectedToken,
		172	});
		173	}
		174
		175	test "recovery: invalid global error set access" {
		176	try testError(
		177	\\test "" {
		178	\\ error && foo;
		179	\\}
		180	, &[_]Error{
		181	.ExpectedToken,
		182	.ExpectedIdentifier,
		183	.InvalidAnd,
		184	});
		185	}
		186
		187	test "recovery: missing semicolon after if, for, while stmt" {
		188	try testError(
		189	\\test "" {
		190	\\ if (foo) bar
		191	\\ for (foo) \|a\| bar
		192	\\ while (foo) bar
		193	\\ a && b;
		194	\\}
		195	, &[_]Error{
		196	.ExpectedSemiOrElse,
		197	.ExpectedSemiOrElse,
		198	.ExpectedSemiOrElse,
		199	.InvalidAnd,
		200	});
		201	}
		202
151	test "zig fmt: top-level fields" {	203	test "zig fmt: top-level fields" {
152	try testCanonical(	204	try testCanonical(
153	\\a: did_you_know,	205	\\a: did_you_know,

lib/std/zig/render.zig+3-4

 @@ -2150,10 +2150,9 @@ fn renderParamDecl(
         try renderToken(tree, stream, name_token, indent, start_col, Space.None);
         try renderToken(tree, stream, tree.nextToken(name_token), indent, start_col, Space.Space); // :
     }
-    if (param_decl.var_args_token) |var_args_token| {
+    switch (param_decl.param_type) {
-        try renderToken(tree, stream, var_args_token, indent, start_col, space);
+        .var_args => |token|  try renderToken(tree, stream, token, indent, start_col, space),
-    } else {
+        .var_type, .type_expr => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, space),
-        try renderExpression(allocator, stream, tree, indent, start_col, param_decl.type_node, space);
     }
 }

src-self-hosted/translate_c.zig+5-9

...	@@ -501,7 +501,7 @@ fn visitFnDecl(c: Context, fn_decl: const ZigClangFunctionDecl) Error!void {	...	@@ -501,7 +501,7 @@ fn visitFnDecl(c: Context, fn_decl: const ZigClangFunctionDecl) Error!void {
501	const param = @fieldParentPtr(ast.Node.ParamDecl, "base", p.*);	501	const param = @fieldParentPtr(ast.Node.ParamDecl, "base", p.*);
502	const param_name = if (param.name_token) \|name_tok\|	502	const param_name = if (param.name_token) \|name_tok\|
503	tokenSlice(c, name_tok)	503	tokenSlice(c, name_tok)
504	else if (param.var_args_token != null) {	504	else if (param.param_type == .var_args) {
505	assert(it.next() == null);	505	assert(it.next() == null);
506	_ = proto_node.params.pop();	506	_ = proto_node.params.pop();
507	break;	507	break;
...	@@ -4103,8 +4103,7 @@ fn transCreateNodeMacroFn(c: Context, name: []const u8, ref: ast.Node, proto_a	...	@@ -4103,8 +4103,7 @@ fn transCreateNodeMacroFn(c: Context, name: []const u8, ref: ast.Node, proto_a
4103	.comptime_token = null,	4103	.comptime_token = null,
4104	.noalias_token = param.noalias_token,	4104	.noalias_token = param.noalias_token,
4105	.name_token = param_name_tok,	4105	.name_token = param_name_tok,
4106	.type_node = param.type_node,	4106	.param_type = param.param_type,
4107	.var_args_token = null,
4108	};	4107	};
4109	try fn_params.push(&param_node.base);	4108	try fn_params.push(&param_node.base);
4110	}	4109	}
...	@@ -4678,8 +4677,7 @@ fn finishTransFnProto(	...	@@ -4678,8 +4677,7 @@ fn finishTransFnProto(
4678	.comptime_token = null,	4677	.comptime_token = null,
4679	.noalias_token = noalias_tok,	4678	.noalias_token = noalias_tok,
4680	.name_token = param_name_tok,	4679	.name_token = param_name_tok,
4681	.type_node = type_node,	4680	.param_type = .{ .type_expr = type_node },
4682	.var_args_token = null,
4683	};	4681	};
4684	try fn_params.push(&param_node.base);	4682	try fn_params.push(&param_node.base);
4685		4683
...	@@ -4699,8 +4697,7 @@ fn finishTransFnProto(	...	@@ -4699,8 +4697,7 @@ fn finishTransFnProto(
4699	.comptime_token = null,	4697	.comptime_token = null,
4700	.noalias_token = null,	4698	.noalias_token = null,
4701	.name_token = null,	4699	.name_token = null,
4702	.type_node = undefined, // Note: Accessing this causes an access violation. Need to check .var_args_token first before trying this field	4700	.param_type = .{ .var_args = try appendToken(rp.c, .Ellipsis3, "...") }
4703	.var_args_token = try appendToken(rp.c, .Ellipsis3, "..."),
4704	};	4701	};
4705	try fn_params.push(&var_arg_node.base);	4702	try fn_params.push(&var_arg_node.base);
4706	}	4703	}
...	@@ -5108,8 +5105,7 @@ fn transMacroFnDefine(c: Context, it: CTokenList.Iterator, source: []const u8,	...	@@ -5108,8 +5105,7 @@ fn transMacroFnDefine(c: Context, it: CTokenList.Iterator, source: []const u8,
5108	.comptime_token = null,	5105	.comptime_token = null,
5109	.noalias_token = null,	5106	.noalias_token = null,
5110	.name_token = param_name_tok,	5107	.name_token = param_name_tok,
5111	.type_node = &identifier.base,	5108	.param_type = .{ .type_expr = &identifier.base },
5112	.var_args_token = null,
5113	};	5109	};
5114	try fn_params.push(&param_node.base);	5110	try fn_params.push(&param_node.base);
5115		5111

src/parser.cpp+3-3

...	@@ -538,8 +538,8 @@ enum ContainerFieldState {	...	@@ -538,8 +538,8 @@ enum ContainerFieldState {
538	// <- TestDecl ContainerMembers	538	// <- TestDecl ContainerMembers
539	// / TopLevelComptime ContainerMembers	539	// / TopLevelComptime ContainerMembers
540	// / KEYWORD_pub? TopLevelDecl ContainerMembers	540	// / KEYWORD_pub? TopLevelDecl ContainerMembers
541	// / KEYWORD_comptime? ContainerField COMMA ContainerMembers	541	// / ContainerField COMMA ContainerMembers
542	// / KEYWORD_comptime? ContainerField	542	// / ContainerField
543	// /	543	// /
544	static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {	544	static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) {
545	AstNodeContainerDecl res = {};	545	AstNodeContainerDecl res = {};
...	@@ -862,7 +862,7 @@ static AstNode ast_parse_var_decl(ParseContext pc) {	...	@@ -862,7 +862,7 @@ static AstNode ast_parse_var_decl(ParseContext pc) {
862	return res;	862	return res;
863	}	863	}
864		864
865	// ContainerField <- IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?	865	// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)?
866	static AstNode ast_parse_container_field(ParseContext pc) {	866	static AstNode ast_parse_container_field(ParseContext pc) {
867	Token *identifier = eat_token_if(pc, TokenIdSymbol);	867	Token *identifier = eat_token_if(pc, TokenIdSymbol);
868	if (identifier == nullptr)	868	if (identifier == nullptr)

...	@@ -10098,7 +10098,7 @@ FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? Link	...	@@ -10098,7 +10098,7 @@ FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? Link
10098		10098
10099	VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON	10099	VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON
10100		10100
10101	ContainerField <- IDENTIFIER (COLON TypeExpr)? (EQUAL Expr)?	10101	ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr)? (EQUAL Expr)?
10102		10102
10103	# * Block Level *	10103	# * Block Level *
10104	Statement	10104	Statement
...	@@ -10212,7 +10212,7 @@ PrimaryTypeExpr	...	@@ -10212,7 +10212,7 @@ PrimaryTypeExpr
10212	/ KEYWORD_error DOT IDENTIFIER	10212	/ KEYWORD_error DOT IDENTIFIER
10213	/ KEYWORD_false	10213	/ KEYWORD_false
10214	/ KEYWORD_null	10214	/ KEYWORD_null
10215	/ KEYWORD_promise	10215	/ KEYWORD_anyframe
10216	/ KEYWORD_true	10216	/ KEYWORD_true
10217	/ KEYWORD_undefined	10217	/ KEYWORD_undefined
10218	/ KEYWORD_unreachable	10218	/ KEYWORD_unreachable
...	@@ -10356,7 +10356,7 @@ PrefixOp	...	@@ -10356,7 +10356,7 @@ PrefixOp
10356		10356
10357	PrefixTypeOp	10357	PrefixTypeOp
10358	<- QUESTIONMARK	10358	<- QUESTIONMARK
10359	/ KEYWORD_promise MINUSRARROW	10359	/ KEYWORD_anyframe MINUSRARROW
10360	/ ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*	10360	/ ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
10361	/ PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*	10361	/ PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)*
10362		10362
...	@@ -10503,6 +10503,7 @@ end_of_word <- ![a-zA-Z0-9_] skip	...	@@ -10503,6 +10503,7 @@ end_of_word <- ![a-zA-Z0-9_] skip
10503	KEYWORD_align <- 'align' end_of_word	10503	KEYWORD_align <- 'align' end_of_word
10504	KEYWORD_allowzero <- 'allowzero' end_of_word	10504	KEYWORD_allowzero <- 'allowzero' end_of_word
10505	KEYWORD_and <- 'and' end_of_word	10505	KEYWORD_and <- 'and' end_of_word
		10506	KEYWORD_anyframe <- 'anyframe' end_of_word
10506	KEYWORD_asm <- 'asm' end_of_word	10507	KEYWORD_asm <- 'asm' end_of_word
10507	KEYWORD_async <- 'async' end_of_word	10508	KEYWORD_async <- 'async' end_of_word
10508	KEYWORD_await <- 'await' end_of_word	10509	KEYWORD_await <- 'await' end_of_word
...	@@ -10529,7 +10530,6 @@ KEYWORD_null <- 'null' end_of_word	...	@@ -10529,7 +10530,6 @@ KEYWORD_null <- 'null' end_of_word
10529	KEYWORD_or <- 'or' end_of_word	10530	KEYWORD_or <- 'or' end_of_word
10530	KEYWORD_orelse <- 'orelse' end_of_word	10531	KEYWORD_orelse <- 'orelse' end_of_word
10531	KEYWORD_packed <- 'packed' end_of_word	10532	KEYWORD_packed <- 'packed' end_of_word
10532	KEYWORD_promise <- 'promise' end_of_word
10533	KEYWORD_pub <- 'pub' end_of_word	10533	KEYWORD_pub <- 'pub' end_of_word
10534	KEYWORD_resume <- 'resume' end_of_word	10534	KEYWORD_resume <- 'resume' end_of_word
10535	KEYWORD_return <- 'return' end_of_word	10535	KEYWORD_return <- 'return' end_of_word
...	@@ -10556,7 +10556,7 @@ keyword <- KEYWORD_align / KEYWORD_and / KEYWORD_allowzero / KEYWORD_asm	...	@@ -10556,7 +10556,7 @@ keyword <- KEYWORD_align / KEYWORD_and / KEYWORD_allowzero / KEYWORD_asm
10556	/ KEYWORD_error / KEYWORD_export / KEYWORD_extern / KEYWORD_false	10556	/ KEYWORD_error / KEYWORD_export / KEYWORD_extern / KEYWORD_false
10557	/ KEYWORD_fn / KEYWORD_for / KEYWORD_if / KEYWORD_inline	10557	/ KEYWORD_fn / KEYWORD_for / KEYWORD_if / KEYWORD_inline
10558	/ KEYWORD_noalias / KEYWORD_null / KEYWORD_or	10558	/ KEYWORD_noalias / KEYWORD_null / KEYWORD_or
10559	/ KEYWORD_orelse / KEYWORD_packed / KEYWORD_promise / KEYWORD_pub	10559	/ KEYWORD_orelse / KEYWORD_packed / KEYWORD_anyframe / KEYWORD_pub
10560	/ KEYWORD_resume / KEYWORD_return / KEYWORD_linksection	10560	/ KEYWORD_resume / KEYWORD_return / KEYWORD_linksection
10561	/ KEYWORD_struct / KEYWORD_suspend	10561	/ KEYWORD_struct / KEYWORD_suspend
10562	/ KEYWORD_switch / KEYWORD_test / KEYWORD_threadlocal / KEYWORD_true / KEYWORD_try	10562	/ KEYWORD_switch / KEYWORD_test / KEYWORD_threadlocal / KEYWORD_true / KEYWORD_try