authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-15 14:15:30+03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-05-15 14:15:30+03:00
logf8b99331a2ca98f0e938c8caaf1cd232ad1e9fa3
treeaa74657a7023f839462bf2512b4ed4db4616243f
parent4b898893e21fc644c4e7a163232e5e98631640d6
parent440189a04ae4baa4a20114fe1d30f0eb585bacc4
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5336 from Vexu/parser

Make self-hosted parser more error tolerant

8 files changed, 949 insertions(+), 580 deletions(-)

doc/langref.html.in+3
......@@ -10104,6 +10104,7 @@ ContainerField &lt;- IDENTIFIER (COLON TypeExpr)? (EQUAL Expr)?
1010410104Statement
1010510105 &lt;- KEYWORD_comptime? VarDecl
1010610106 / KEYWORD_comptime BlockExprStatement
10107 / KEYWORD_nosuspend BlockExprStatement
1010710108 / KEYWORD_suspend (SEMICOLON / BlockExprStatement)
1010810109 / KEYWORD_defer BlockExprStatement
1010910110 / KEYWORD_errdefer BlockExprStatement
......@@ -10160,6 +10161,7 @@ PrimaryExpr
1016010161 / IfExpr
1016110162 / KEYWORD_break BreakLabel? Expr?
1016210163 / KEYWORD_comptime Expr
10164 / KEYWORD_nosuspend Expr
1016310165 / KEYWORD_continue BreakLabel?
1016410166 / KEYWORD_resume Expr
1016510167 / KEYWORD_return Expr?
......@@ -10522,6 +10524,7 @@ KEYWORD_for &lt;- 'for' end_of_word
1052210524KEYWORD_if &lt;- 'if' end_of_word
1052310525KEYWORD_inline &lt;- 'inline' end_of_word
1052410526KEYWORD_noalias &lt;- 'noalias' end_of_word
10527KEYWORD_nosuspend &lt;- 'nosuspend' end_of_word
1052510528KEYWORD_null &lt;- 'null' end_of_word
1052610529KEYWORD_or &lt;- 'or' end_of_word
1052710530KEYWORD_orelse &lt;- 'orelse' end_of_word
lib/std/zig/ast.zig+11-3
......@@ -129,6 +129,7 @@ pub const Error = union(enum) {
129129 ExpectedStatement: ExpectedStatement,
130130 ExpectedVarDeclOrFn: ExpectedVarDeclOrFn,
131131 ExpectedVarDecl: ExpectedVarDecl,
132 ExpectedFn: ExpectedFn,
132133 ExpectedReturnType: ExpectedReturnType,
133134 ExpectedAggregateKw: ExpectedAggregateKw,
134135 UnattachedDocComment: UnattachedDocComment,
......@@ -165,6 +166,7 @@ pub const Error = union(enum) {
165166 ExpectedDerefOrUnwrap: ExpectedDerefOrUnwrap,
166167 ExpectedSuffixOp: ExpectedSuffixOp,
167168 DeclBetweenFields: DeclBetweenFields,
169 InvalidAnd: InvalidAnd,
168170
169171 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
170172 switch (self.*) {
......@@ -177,6 +179,7 @@ pub const Error = union(enum) {
177179 .ExpectedStatement => |*x| return x.render(tokens, stream),
178180 .ExpectedVarDeclOrFn => |*x| return x.render(tokens, stream),
179181 .ExpectedVarDecl => |*x| return x.render(tokens, stream),
182 .ExpectedFn => |*x| return x.render(tokens, stream),
180183 .ExpectedReturnType => |*x| return x.render(tokens, stream),
181184 .ExpectedAggregateKw => |*x| return x.render(tokens, stream),
182185 .UnattachedDocComment => |*x| return x.render(tokens, stream),
......@@ -213,6 +216,7 @@ pub const Error = union(enum) {
213216 .ExpectedDerefOrUnwrap => |*x| return x.render(tokens, stream),
214217 .ExpectedSuffixOp => |*x| return x.render(tokens, stream),
215218 .DeclBetweenFields => |*x| return x.render(tokens, stream),
219 .InvalidAnd => |*x| return x.render(tokens, stream),
216220 }
217221 }
218222
......@@ -227,6 +231,7 @@ pub const Error = union(enum) {
227231 .ExpectedStatement => |x| return x.token,
228232 .ExpectedVarDeclOrFn => |x| return x.token,
229233 .ExpectedVarDecl => |x| return x.token,
234 .ExpectedFn => |x| return x.token,
230235 .ExpectedReturnType => |x| return x.token,
231236 .ExpectedAggregateKw => |x| return x.token,
232237 .UnattachedDocComment => |x| return x.token,
......@@ -263,6 +268,7 @@ pub const Error = union(enum) {
263268 .ExpectedDerefOrUnwrap => |x| return x.token,
264269 .ExpectedSuffixOp => |x| return x.token,
265270 .DeclBetweenFields => |x| return x.token,
271 .InvalidAnd => |x| return x.token,
266272 }
267273 }
268274
......@@ -274,6 +280,7 @@ pub const Error = union(enum) {
274280 pub const ExpectedStatement = SingleTokenError("Expected statement, found '{}'");
275281 pub const ExpectedVarDeclOrFn = SingleTokenError("Expected variable declaration or function, found '{}'");
276282 pub const ExpectedVarDecl = SingleTokenError("Expected variable declaration, found '{}'");
283 pub const ExpectedFn = SingleTokenError("Expected function, found '{}'");
277284 pub const ExpectedReturnType = SingleTokenError("Expected 'var' or return type expression, found '{}'");
278285 pub const ExpectedAggregateKw = SingleTokenError("Expected '" ++ Token.Id.Keyword_struct.symbol() ++ "', '" ++ Token.Id.Keyword_union.symbol() ++ "', or '" ++ Token.Id.Keyword_enum.symbol() ++ "', found '{}'");
279286 pub const ExpectedEqOrSemi = SingleTokenError("Expected '=' or ';', found '{}'");
......@@ -308,6 +315,7 @@ pub const Error = union(enum) {
308315 pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier");
309316 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
310317 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
318 pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
311319
312320 pub const ExpectedCall = struct {
313321 node: *Node,
......@@ -335,9 +343,6 @@ pub const Error = union(enum) {
335343 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
336344 const found_token = tokens.at(self.token);
337345 switch (found_token.id) {
338 .Invalid_ampersands => {
339 return stream.print("`&&` is invalid. Note that `and` is boolean AND.", .{});
340 },
341346 .Invalid => {
342347 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
343348 },
......@@ -888,6 +893,7 @@ pub const Node = struct {
888893 pub const ReturnType = union(enum) {
889894 Explicit: *Node,
890895 InferErrorSet: *Node,
896 Invalid: TokenIndex,
891897 };
892898
893899 pub fn iterate(self: *FnProto, index: usize) ?*Node {
......@@ -916,6 +922,7 @@ pub const Node = struct {
916922 if (i < 1) return node;
917923 i -= 1;
918924 },
925 .Invalid => {},
919926 }
920927
921928 if (self.body_node) |body_node| {
......@@ -937,6 +944,7 @@ pub const Node = struct {
937944 if (self.body_node) |body_node| return body_node.lastToken();
938945 switch (self.return_type) {
939946 .Explicit, .InferErrorSet => |node| return node.lastToken(),
947 .Invalid => |tok| return tok,
940948 }
941949 }
942950 };
lib/std/zig/parse.zig+281-82
......@@ -48,31 +48,24 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
4848
4949 while (it.peek().?.id == .LineComment) _ = it.next();
5050
51 tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {
52 switch (err) {
53 error.ParseError => {
54 assert(tree.errors.len != 0);
55 break :blk undefined;
56 },
57 error.OutOfMemory => {
58 return error.OutOfMemory;
59 },
60 }
61 };
51 tree.root_node = try parseRoot(arena, &it, tree);
6252
6353 return tree;
6454}
6555
6656/// Root <- skip ContainerMembers eof
67fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {
57fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
6858 const node = try arena.create(Node.Root);
6959 node.* = .{
7060 .decls = try parseContainerMembers(arena, it, tree),
71 .eof_token = eatToken(it, .Eof) orelse {
61 .eof_token = eatToken(it, .Eof) orelse blk: {
62 // parseContainerMembers will try to skip as much
63 // invalid tokens as it can so this can only be a '}'
64 const tok = eatToken(it, .RBrace).?;
7265 try tree.errors.push(.{
73 .ExpectedContainerMembers = .{ .token = it.index },
66 .ExpectedContainerMembers = .{ .token = tok },
7467 });
75 return error.ParseError;
68 break :blk tok;
7669 },
7770 };
7871 return node;
......@@ -108,7 +101,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
108101
109102 const doc_comments = try parseDocComment(arena, it, tree);
110103
111 if (try parseTestDecl(arena, it, tree)) |node| {
104 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
105 error.OutOfMemory => return error.OutOfMemory,
106 error.ParseError => {
107 findNextContainerMember(it);
108 continue;
109 },
110 }) |node| {
112111 if (field_state == .seen) {
113112 field_state = .{ .end = node.firstToken() };
114113 }
......@@ -117,7 +116,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
117116 continue;
118117 }
119118
120 if (try parseTopLevelComptime(arena, it, tree)) |node| {
119 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
120 error.OutOfMemory => return error.OutOfMemory,
121 error.ParseError => {
122 findNextContainerMember(it);
123 continue;
124 },
125 }) |node| {
121126 if (field_state == .seen) {
122127 field_state = .{ .end = node.firstToken() };
123128 }
......@@ -128,7 +133,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
128133
129134 const visib_token = eatToken(it, .Keyword_pub);
130135
131 if (try parseTopLevelDecl(arena, it, tree)) |node| {
136 if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {
137 error.OutOfMemory => return error.OutOfMemory,
138 error.ParseError => {
139 findNextContainerMember(it);
140 continue;
141 },
142 }) |node| {
132143 if (field_state == .seen) {
133144 field_state = .{ .end = visib_token orelse node.firstToken() };
134145 }
......@@ -163,10 +174,18 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
163174 try tree.errors.push(.{
164175 .ExpectedPubItem = .{ .token = it.index },
165176 });
166 return error.ParseError;
177 // ignore this pub
178 continue;
167179 }
168180
169 if (try parseContainerField(arena, it, tree)) |node| {
181 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
182 error.OutOfMemory => return error.OutOfMemory,
183 error.ParseError => {
184 // attempt to recover
185 findNextContainerMember(it);
186 continue;
187 },
188 }) |node| {
170189 switch (field_state) {
171190 .none => field_state = .seen,
172191 .err, .seen => {},
......@@ -182,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
182201 const field = node.cast(Node.ContainerField).?;
183202 field.doc_comments = doc_comments;
184203 try list.push(node);
185 const comma = eatToken(it, .Comma) orelse break;
204 const comma = eatToken(it, .Comma) orelse {
205 // try to continue parsing
206 const index = it.index;
207 findNextContainerMember(it);
208 switch (it.peek().?.id) {
209 .Eof, .RBrace => break,
210 else => {
211 // add error and continue
212 try tree.errors.push(.{
213 .ExpectedToken = .{ .token = index, .expected_id = .Comma },
214 });
215 continue;
216 },
217 }
218 };
186219 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
187220 field.doc_comments = appended_comment;
188221 continue;
......@@ -194,12 +227,102 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
194227 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
195228 });
196229 }
197 break;
230
231 switch (it.peek().?.id) {
232 .Eof, .RBrace => break,
233 else => {
234 // this was likely not supposed to end yet,
235 // try to find the next declaration
236 const index = it.index;
237 findNextContainerMember(it);
238 try tree.errors.push(.{
239 .ExpectedContainerMembers = .{ .token = index },
240 });
241 },
242 }
198243 }
199244
200245 return list;
201246}
202247
248/// Attempts to find next container member by searching for certain tokens
249fn findNextContainerMember(it: *TokenIterator) void {
250 var level: u32 = 0;
251 while (true) {
252 const tok = nextToken(it);
253 switch (tok.ptr.id) {
254 // any of these can start a new top level declaration
255 .Keyword_test,
256 .Keyword_comptime,
257 .Keyword_pub,
258 .Keyword_export,
259 .Keyword_extern,
260 .Keyword_inline,
261 .Keyword_noinline,
262 .Keyword_usingnamespace,
263 .Keyword_threadlocal,
264 .Keyword_const,
265 .Keyword_var,
266 .Keyword_fn,
267 .Identifier,
268 => {
269 if (level == 0) {
270 putBackToken(it, tok.index);
271 return;
272 }
273 },
274 .Comma, .Semicolon => {
275 // this decl was likely meant to end here
276 if (level == 0) {
277 return;
278 }
279 },
280 .LParen, .LBracket, .LBrace => level += 1,
281 .RParen, .RBracket, .RBrace => {
282 if (level == 0) {
283 // end of container, exit
284 putBackToken(it, tok.index);
285 return;
286 }
287 level -= 1;
288 },
289 .Eof => {
290 putBackToken(it, tok.index);
291 return;
292 },
293 else => {},
294 }
295 }
296}
297
298/// Attempts to find the next statement by searching for a semicolon
299fn findNextStmt(it: *TokenIterator) void {
300 var level: u32 = 0;
301 while (true) {
302 const tok = nextToken(it);
303 switch (tok.ptr.id) {
304 .LBrace => level += 1,
305 .RBrace => {
306 if (level == 0) {
307 putBackToken(it, tok.index);
308 return;
309 }
310 level -= 1;
311 },
312 .Semicolon => {
313 if (level == 0) {
314 return;
315 }
316 },
317 .Eof => {
318 putBackToken(it, tok.index);
319 return;
320 },
321 else => {},
322 }
323 }
324}
325
203326/// Eat a multiline container doc comment
204327fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
205328 var lines = Node.DocComment.LineList.init(arena);
......@@ -279,22 +402,30 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
279402 fn_node.*.extern_export_inline_token = extern_export_inline_token;
280403 fn_node.*.lib_name = lib_name;
281404 if (eatToken(it, .Semicolon)) |_| return node;
282 if (try parseBlock(arena, it, tree)) |body_node| {
405 if (parseBlock(arena, it, tree) catch |err| switch (err) {
406 error.OutOfMemory => return error.OutOfMemory,
407 // since parseBlock only return error.ParseError on
408 // a missing '}' we can assume this function was
409 // supposed to end here.
410 error.ParseError => return node,
411 }) |body_node| {
283412 fn_node.body_node = body_node;
284413 return node;
285414 }
286415 try tree.errors.push(.{
287416 .ExpectedSemiOrLBrace = .{ .token = it.index },
288417 });
289 return null;
418 return error.ParseError;
290419 }
291420
292421 if (extern_export_inline_token) |token| {
293422 if (tree.tokens.at(token).id == .Keyword_inline or
294423 tree.tokens.at(token).id == .Keyword_noinline)
295424 {
296 putBackToken(it, token);
297 return null;
425 try tree.errors.push(.{
426 .ExpectedFn = .{ .token = it.index },
427 });
428 return error.ParseError;
298429 }
299430 }
300431
......@@ -313,26 +444,19 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
313444 try tree.errors.push(.{
314445 .ExpectedVarDecl = .{ .token = it.index },
315446 });
447 // ignore this and try again;
316448 return error.ParseError;
317449 }
318450
319451 if (extern_export_inline_token) |token| {
320 if (lib_name) |string_literal_node|
321 putBackToken(it, string_literal_node.cast(Node.StringLiteral).?.token);
322 putBackToken(it, token);
323 return null;
452 try tree.errors.push(.{
453 .ExpectedVarDeclOrFn = .{ .token = it.index },
454 });
455 // ignore this and try again;
456 return error.ParseError;
324457 }
325458
326 const use_node = (try parseUse(arena, it, tree)) orelse return null;
327 const expr_node = try expectNode(arena, it, tree, parseExpr, .{
328 .ExpectedExpr = .{ .token = it.index },
329 });
330 const semicolon_token = try expectToken(it, tree, .Semicolon);
331 const use_node_raw = use_node.cast(Node.Use).?;
332 use_node_raw.*.expr = expr_node;
333 use_node_raw.*.semicolon_token = semicolon_token;
334
335 return use_node;
459 return try parseUse(arena, it, tree);
336460}
337461
338462/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
......@@ -366,18 +490,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
366490 const exclamation_token = eatToken(it, .Bang);
367491
368492 const return_type_expr = (try parseVarType(arena, it, tree)) orelse
369 try expectNode(arena, it, tree, parseTypeExpr, .{
370 .ExpectedReturnType = .{ .token = it.index },
371 });
493 (try parseTypeExpr(arena, it, tree)) orelse blk: {
494 try tree.errors.push(.{
495 .ExpectedReturnType = .{ .token = it.index },
496 });
497 // most likely the user forgot to specify the return type.
498 // Mark return type as invalid and try to continue.
499 break :blk null;
500 };
372501
373 const return_type: Node.FnProto.ReturnType = if (exclamation_token != null)
374 .{
375 .InferErrorSet = return_type_expr,
376 }
502 // TODO https://github.com/ziglang/zig/issues/3750
503 const R = Node.FnProto.ReturnType;
504 const return_type = if (return_type_expr == null)
505 R{ .Invalid = rparen }
506 else if (exclamation_token != null)
507 R{ .InferErrorSet = return_type_expr.? }
377508 else
378 .{
379 .Explicit = return_type_expr,
380 };
509 R{ .Explicit = return_type_expr.? };
381510
382511 const var_args_token = if (params.len > 0)
383512 params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token
......@@ -578,7 +707,12 @@ fn parseStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*No
578707 if (try parseLabeledStatement(arena, it, tree)) |node| return node;
579708 if (try parseSwitchExpr(arena, it, tree)) |node| return node;
580709 if (try parseAssignExpr(arena, it, tree)) |node| {
581 _ = try expectToken(it, tree, .Semicolon);
710 _ = eatToken(it, .Semicolon) orelse {
711 try tree.errors.push(.{
712 .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
713 });
714 // pretend we saw a semicolon and continue parsing
715 };
582716 return node;
583717 }
584718
......@@ -687,8 +821,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
687821 node.cast(Node.While).?.inline_token = inline_token;
688822 return node;
689823 }
824 if (inline_token == null) return null;
690825
691 return null;
826 // If we've seen "inline", there should have been a "for" or "while"
827 try tree.errors.push(.{
828 .ExpectedInlinable = .{ .token = it.index },
829 });
830 return error.ParseError;
692831}
693832
694833/// ForStatement
......@@ -817,7 +956,12 @@ fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
817956fn parseBlockExprStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
818957 if (try parseBlockExpr(arena, it, tree)) |node| return node;
819958 if (try parseAssignExpr(arena, it, tree)) |node| {
820 _ = try expectToken(it, tree, .Semicolon);
959 _ = eatToken(it, .Semicolon) orelse {
960 try tree.errors.push(.{
961 .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
962 });
963 // pretend we saw a semicolon and continue parsing
964 };
821965 return node;
822966 }
823967 return null;
......@@ -924,7 +1068,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
9241068 const node = try arena.create(Node.ControlFlowExpression);
9251069 node.* = .{
9261070 .ltoken = token,
927 .kind = Node.ControlFlowExpression.Kind{ .Break = label },
1071 .kind = .{ .Break = label },
9281072 .rhs = expr_node,
9291073 };
9301074 return &node.base;
......@@ -960,7 +1104,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
9601104 const node = try arena.create(Node.ControlFlowExpression);
9611105 node.* = .{
9621106 .ltoken = token,
963 .kind = Node.ControlFlowExpression.Kind{ .Continue = label },
1107 .kind = .{ .Continue = label },
9641108 .rhs = null,
9651109 };
9661110 return &node.base;
......@@ -984,7 +1128,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
9841128 const node = try arena.create(Node.ControlFlowExpression);
9851129 node.* = .{
9861130 .ltoken = token,
987 .kind = Node.ControlFlowExpression.Kind.Return,
1131 .kind = .Return,
9881132 .rhs = expr_node,
9891133 };
9901134 return &node.base;
......@@ -1022,7 +1166,14 @@ fn parseBlock(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
10221166
10231167 var statements = Node.Block.StatementList.init(arena);
10241168 while (true) {
1025 const statement = (try parseStatement(arena, it, tree)) orelse break;
1169 const statement = (parseStatement(arena, it, tree) catch |err| switch (err) {
1170 error.OutOfMemory => return error.OutOfMemory,
1171 error.ParseError => {
1172 // try to skip to the next statement
1173 findNextStmt(it);
1174 continue;
1175 },
1176 }) orelse break;
10261177 try statements.push(statement);
10271178 }
10281179
......@@ -1222,7 +1373,8 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
12221373 try tree.errors.push(.{
12231374 .ExpectedParamList = .{ .token = it.index },
12241375 });
1225 return null;
1376 // ignore this, continue parsing
1377 return res;
12261378 };
12271379 const node = try arena.create(Node.SuffixOp);
12281380 node.* = .{
......@@ -1287,7 +1439,6 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
12871439/// / IfTypeExpr
12881440/// / INTEGER
12891441/// / KEYWORD_comptime TypeExpr
1290/// / KEYWORD_nosuspend TypeExpr
12911442/// / KEYWORD_error DOT IDENTIFIER
12921443/// / KEYWORD_false
12931444/// / KEYWORD_null
......@@ -1326,15 +1477,6 @@ fn parsePrimaryTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*N
13261477 };
13271478 return &node.base;
13281479 }
1329 if (eatToken(it, .Keyword_nosuspend)) |token| {
1330 const expr = (try parseTypeExpr(arena, it, tree)) orelse return null;
1331 const node = try arena.create(Node.Nosuspend);
1332 node.* = .{
1333 .nosuspend_token = token,
1334 .expr = expr,
1335 };
1336 return &node.base;
1337 }
13381480 if (eatToken(it, .Keyword_error)) |token| {
13391481 const period = try expectToken(it, tree, .Period);
13401482 const identifier = try expectNode(arena, it, tree, parseIdentifier, .{
......@@ -2271,7 +2413,7 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
22712413 const node = try arena.create(Node.AnyFrameType);
22722414 node.* = .{
22732415 .anyframe_token = token,
2274 .result = Node.AnyFrameType.Result{
2416 .result = .{
22752417 .arrow_token = arrow,
22762418 .return_type = undefined, // set by caller
22772419 },
......@@ -2312,6 +2454,13 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
23122454 } else null;
23132455 _ = try expectToken(it, tree, .RParen);
23142456
2457 if (ptr_info.align_info != null) {
2458 try tree.errors.push(.{
2459 .ExtraAlignQualifier = .{ .token = it.index - 1 },
2460 });
2461 continue;
2462 }
2463
23152464 ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{
23162465 .node = expr_node,
23172466 .bit_range = bit_range,
......@@ -2320,14 +2469,32 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
23202469 continue;
23212470 }
23222471 if (eatToken(it, .Keyword_const)) |const_token| {
2472 if (ptr_info.const_token != null) {
2473 try tree.errors.push(.{
2474 .ExtraConstQualifier = .{ .token = it.index - 1 },
2475 });
2476 continue;
2477 }
23232478 ptr_info.const_token = const_token;
23242479 continue;
23252480 }
23262481 if (eatToken(it, .Keyword_volatile)) |volatile_token| {
2482 if (ptr_info.volatile_token != null) {
2483 try tree.errors.push(.{
2484 .ExtraVolatileQualifier = .{ .token = it.index - 1 },
2485 });
2486 continue;
2487 }
23272488 ptr_info.volatile_token = volatile_token;
23282489 continue;
23292490 }
23302491 if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
2492 if (ptr_info.allowzero_token != null) {
2493 try tree.errors.push(.{
2494 .ExtraAllowZeroQualifier = .{ .token = it.index - 1 },
2495 });
2496 continue;
2497 }
23312498 ptr_info.allowzero_token = allowzero_token;
23322499 continue;
23332500 }
......@@ -2346,9 +2513,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
23462513 if (try parseByteAlign(arena, it, tree)) |align_expr| {
23472514 if (slice_type.align_info != null) {
23482515 try tree.errors.push(.{
2349 .ExtraAlignQualifier = .{ .token = it.index },
2516 .ExtraAlignQualifier = .{ .token = it.index - 1 },
23502517 });
2351 return error.ParseError;
2518 continue;
23522519 }
23532520 slice_type.align_info = Node.PrefixOp.PtrInfo.Align{
23542521 .node = align_expr,
......@@ -2359,9 +2526,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
23592526 if (eatToken(it, .Keyword_const)) |const_token| {
23602527 if (slice_type.const_token != null) {
23612528 try tree.errors.push(.{
2362 .ExtraConstQualifier = .{ .token = it.index },
2529 .ExtraConstQualifier = .{ .token = it.index - 1 },
23632530 });
2364 return error.ParseError;
2531 continue;
23652532 }
23662533 slice_type.const_token = const_token;
23672534 continue;
......@@ -2369,9 +2536,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
23692536 if (eatToken(it, .Keyword_volatile)) |volatile_token| {
23702537 if (slice_type.volatile_token != null) {
23712538 try tree.errors.push(.{
2372 .ExtraVolatileQualifier = .{ .token = it.index },
2539 .ExtraVolatileQualifier = .{ .token = it.index - 1 },
23732540 });
2374 return error.ParseError;
2541 continue;
23752542 }
23762543 slice_type.volatile_token = volatile_token;
23772544 continue;
......@@ -2379,9 +2546,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
23792546 if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
23802547 if (slice_type.allowzero_token != null) {
23812548 try tree.errors.push(.{
2382 .ExtraAllowZeroQualifier = .{ .token = it.index },
2549 .ExtraAllowZeroQualifier = .{ .token = it.index - 1 },
23832550 });
2384 return error.ParseError;
2551 continue;
23852552 }
23862553 slice_type.allowzero_token = allowzero_token;
23872554 continue;
......@@ -2730,7 +2897,19 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
27302897 var list = L.init(arena);
27312898 while (try nodeParseFn(arena, it, tree)) |node| {
27322899 try list.push(node);
2733 if (eatToken(it, .Comma) == null) break;
2900
2901 switch (it.peek().?.id) {
2902 .Comma => _ = nextToken(it),
2903 // all possible delimiters
2904 .Colon, .RParen, .RBrace, .RBracket => break,
2905 else => {
2906 // this is likely just a missing comma,
2907 // continue parsing this list and give an error
2908 try tree.errors.push(.{
2909 .ExpectedToken = .{ .token = it.index, .expected_id = .Comma },
2910 });
2911 },
2912 }
27342913 }
27352914 return list;
27362915 }
......@@ -2740,7 +2919,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
27402919fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
27412920 return struct {
27422921 pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node {
2743 const op_token = eatToken(it, token) orelse return null;
2922 const op_token = if (token == .Keyword_and) switch (it.peek().?.id) {
2923 .Keyword_and => nextToken(it).index,
2924 .Invalid_ampersands => blk: {
2925 try tree.errors.push(.{
2926 .InvalidAnd = .{ .token = it.index },
2927 });
2928 break :blk nextToken(it).index;
2929 },
2930 else => return null,
2931 } else eatToken(it, token) orelse return null;
2932
27442933 const node = try arena.create(Node.InfixOp);
27452934 node.* = .{
27462935 .op_token = op_token,
......@@ -2761,7 +2950,13 @@ fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
27612950 try tree.errors.push(.{
27622951 .ExpectedParamList = .{ .token = it.index },
27632952 });
2764 return error.ParseError;
2953
2954 // lets pretend this was an identifier so we can continue parsing
2955 const node = try arena.create(Node.Identifier);
2956 node.* = .{
2957 .token = token,
2958 };
2959 return &node.base;
27652960 };
27662961 const node = try arena.create(Node.BuiltinCall);
27672962 node.* = .{
......@@ -2877,8 +3072,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
28773072 .doc_comments = null,
28783073 .visib_token = null,
28793074 .use_token = token,
2880 .expr = undefined, // set by caller
2881 .semicolon_token = undefined, // set by caller
3075 .expr = try expectNode(arena, it, tree, parseExpr, .{
3076 .ExpectedExpr = .{ .token = it.index },
3077 }),
3078 .semicolon_token = try expectToken(it, tree, .Semicolon),
28823079 };
28833080 return &node.base;
28843081}
......@@ -3058,6 +3255,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
30583255 try tree.errors.push(.{
30593256 .ExpectedToken = .{ .token = token.index, .expected_id = id },
30603257 });
3258 // go back so that we can recover properly
3259 putBackToken(it, token.index);
30613260 return error.ParseError;
30623261 }
30633262 return token.index;
lib/std/zig/parser_test.zig+167-4
......@@ -1,3 +1,153 @@
1test "recovery: top level" {
2 try testError(
3 \\test "" {inline}
4 \\test "" {inline}
5 , &[_]Error{
6 .ExpectedInlinable,
7 .ExpectedInlinable,
8 });
9}
10
11test "recovery: block statements" {
12 try testError(
13 \\test "" {
14 \\ foo + +;
15 \\ inline;
16 \\}
17 , &[_]Error{
18 .InvalidToken,
19 .ExpectedInlinable,
20 });
21}
22
23test "recovery: missing comma" {
24 try testError(
25 \\test "" {
26 \\ switch (foo) {
27 \\ 2 => {}
28 \\ 3 => {}
29 \\ else => {
30 \\ foo && bar +;
31 \\ }
32 \\ }
33 \\}
34 , &[_]Error{
35 .ExpectedToken,
36 .ExpectedToken,
37 .InvalidAnd,
38 .InvalidToken,
39 });
40}
41
42test "recovery: extra qualifier" {
43 try testError(
44 \\const a: *const const u8;
45 \\test ""
46 , &[_]Error{
47 .ExtraConstQualifier,
48 .ExpectedLBrace,
49 });
50}
51
52test "recovery: missing return type" {
53 try testError(
54 \\fn foo() {
55 \\ a && b;
56 \\}
57 \\test ""
58 , &[_]Error{
59 .ExpectedReturnType,
60 .InvalidAnd,
61 .ExpectedLBrace,
62 });
63}
64
65test "recovery: continue after invalid decl" {
66 try testError(
67 \\fn foo {
68 \\ inline;
69 \\}
70 \\pub test "" {
71 \\ async a && b;
72 \\}
73 , &[_]Error{
74 .ExpectedToken,
75 .ExpectedPubItem,
76 .ExpectedParamList,
77 .InvalidAnd,
78 });
79 try testError(
80 \\threadlocal test "" {
81 \\ @a && b;
82 \\}
83 , &[_]Error{
84 .ExpectedVarDecl,
85 .ExpectedParamList,
86 .InvalidAnd,
87 });
88}
89
90test "recovery: invalid extern/inline" {
91 try testError(
92 \\inline test "" { a && b; }
93 , &[_]Error{
94 .ExpectedFn,
95 .InvalidAnd,
96 });
97 try testError(
98 \\extern "" test "" { a && b; }
99 , &[_]Error{
100 .ExpectedVarDeclOrFn,
101 .InvalidAnd,
102 });
103}
104
105test "recovery: missing semicolon" {
106 try testError(
107 \\test "" {
108 \\ comptime a && b
109 \\ c && d
110 \\ @foo
111 \\}
112 , &[_]Error{
113 .InvalidAnd,
114 .ExpectedToken,
115 .InvalidAnd,
116 .ExpectedToken,
117 .ExpectedParamList,
118 .ExpectedToken,
119 });
120}
121
122test "recovery: invalid container members" {
123 try testError(
124 \\usingnamespace;
125 \\foo+
126 \\bar@,
127 \\while (a == 2) { test "" {}}
128 \\test "" {
129 \\ a && b
130 \\}
131 , &[_]Error{
132 .ExpectedExpr,
133 .ExpectedToken,
134 .ExpectedToken,
135 .ExpectedContainerMembers,
136 .InvalidAnd,
137 .ExpectedToken,
138 });
139}
140
141test "recovery: invalid parameter" {
142 try testError(
143 \\fn main() void {
144 \\ a(comptime T: type)
145 \\}
146 , &[_]Error{
147 .ExpectedToken,
148 });
149}
150
1151test "zig fmt: top-level fields" {
2152 try testCanonical(
3153 \\a: did_you_know,
......@@ -19,7 +169,9 @@ test "zig fmt: decl between fields" {
19169 \\ const baz1 = 2;
20170 \\ b: usize,
21171 \\};
22 );
172 , &[_]Error{
173 .DeclBetweenFields,
174 });
23175}
24176
25177test "zig fmt: errdefer with payload" {
......@@ -2828,7 +2980,10 @@ test "zig fmt: extern without container keyword returns error" {
28282980 try testError(
28292981 \\const container = extern {};
28302982 \\
2831 );
2983 , &[_]Error{
2984 .ExpectedExpr,
2985 .ExpectedVarDeclOrFn,
2986 });
28322987}
28332988
28342989test "zig fmt: integer literals with underscore separators" {
......@@ -3030,9 +3185,17 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
30303185fn testCanonical(source: []const u8) !void {
30313186 return testTransform(source, source);
30323187}
3033fn testError(source: []const u8) !void {
3188
3189const Error = @TagType(std.zig.ast.Error);
3190
3191fn testError(source: []const u8, expected_errors: []const Error) !void {
30343192 const tree = try std.zig.parse(std.testing.allocator, source);
30353193 defer tree.deinit();
30363194
3037 std.testing.expect(tree.errors.len != 0);
3195 std.testing.expect(tree.errors.len == expected_errors.len);
3196 for (expected_errors) |expected, i| {
3197 const err = tree.errors.at(i);
3198
3199 std.testing.expect(expected == err.*);
3200 }
30383201}
lib/std/zig/render.zig+7-2
......@@ -13,6 +13,9 @@ pub const Error = error{
1313
1414/// Returns whether anything changed
1515pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Error || Error)!bool {
16 // cannot render an invalid tree
17 std.debug.assert(tree.errors.len == 0);
18
1619 // make a passthrough stream that checks whether something changed
1720 const MyStream = struct {
1821 const MyStream = @This();
......@@ -1444,6 +1447,7 @@ fn renderExpression(
14441447 else switch (fn_proto.return_type) {
14451448 .Explicit => |node| node.firstToken(),
14461449 .InferErrorSet => |node| tree.prevToken(node.firstToken()),
1450 .Invalid => unreachable,
14471451 });
14481452 assert(tree.tokens.at(rparen).id == .RParen);
14491453
......@@ -1518,13 +1522,14 @@ fn renderExpression(
15181522 }
15191523
15201524 switch (fn_proto.return_type) {
1521 ast.Node.FnProto.ReturnType.Explicit => |node| {
1525 .Explicit => |node| {
15221526 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
15231527 },
1524 ast.Node.FnProto.ReturnType.InferErrorSet => |node| {
1528 .InferErrorSet => |node| {
15251529 try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !
15261530 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
15271531 },
1532 .Invalid => unreachable,
15281533 }
15291534 },
15301535
lib/std/zig/tokenizer.zig+477-477
......@@ -353,64 +353,64 @@ pub const Tokenizer = struct {
353353 }
354354
355355 const State = enum {
356 Start,
357 Identifier,
358 Builtin,
359 StringLiteral,
360 StringLiteralBackslash,
361 MultilineStringLiteralLine,
362 CharLiteral,
363 CharLiteralBackslash,
364 CharLiteralHexEscape,
365 CharLiteralUnicodeEscapeSawU,
366 CharLiteralUnicodeEscape,
367 CharLiteralUnicodeInvalid,
368 CharLiteralUnicode,
369 CharLiteralEnd,
370 Backslash,
371 Equal,
372 Bang,
373 Pipe,
374 Minus,
375 MinusPercent,
376 Asterisk,
377 AsteriskPercent,
378 Slash,
379 LineCommentStart,
380 LineComment,
381 DocCommentStart,
382 DocComment,
383 ContainerDocComment,
384 Zero,
385 IntegerLiteralDec,
386 IntegerLiteralDecNoUnderscore,
387 IntegerLiteralBin,
388 IntegerLiteralBinNoUnderscore,
389 IntegerLiteralOct,
390 IntegerLiteralOctNoUnderscore,
391 IntegerLiteralHex,
392 IntegerLiteralHexNoUnderscore,
393 NumberDotDec,
394 NumberDotHex,
395 FloatFractionDec,
396 FloatFractionDecNoUnderscore,
397 FloatFractionHex,
398 FloatFractionHexNoUnderscore,
399 FloatExponentUnsigned,
400 FloatExponentNumber,
401 FloatExponentNumberNoUnderscore,
402 Ampersand,
403 Caret,
404 Percent,
405 Plus,
406 PlusPercent,
407 AngleBracketLeft,
408 AngleBracketAngleBracketLeft,
409 AngleBracketRight,
410 AngleBracketAngleBracketRight,
411 Period,
412 Period2,
413 SawAtSign,
356 start,
357 identifier,
358 builtin,
359 string_literal,
360 string_literal_backslash,
361 multiline_string_literal_line,
362 char_literal,
363 char_literal_backslash,
364 char_literal_hex_escape,
365 char_literal_unicode_escape_saw_u,
366 char_literal_unicode_escape,
367 char_literal_unicode_invalid,
368 char_literal_unicode,
369 char_literal_end,
370 backslash,
371 equal,
372 bang,
373 pipe,
374 minus,
375 minus_percent,
376 asterisk,
377 asterisk_percent,
378 slash,
379 line_comment_start,
380 line_comment,
381 doc_comment_start,
382 doc_comment,
383 container_doc_comment,
384 zero,
385 int_literal_dec,
386 int_literal_dec_no_underscore,
387 int_literal_bin,
388 int_literal_bin_no_underscore,
389 int_literal_oct,
390 int_literal_oct_no_underscore,
391 int_literal_hex,
392 int_literal_hex_no_underscore,
393 num_dot_dec,
394 num_dot_hex,
395 float_fraction_dec,
396 float_fraction_dec_no_underscore,
397 float_fraction_hex,
398 float_fraction_hex_no_underscore,
399 float_exponent_unsigned,
400 float_exponent_num,
401 float_exponent_num_no_underscore,
402 ampersand,
403 caret,
404 percent,
405 plus,
406 plus_percent,
407 angle_bracket_left,
408 angle_bracket_angle_bracket_left,
409 angle_bracket_right,
410 angle_bracket_angle_bracket_right,
411 period,
412 period_2,
413 saw_at_sign,
414414 };
415415
416416 fn isIdentifierChar(char: u8) bool {
......@@ -423,9 +423,9 @@ pub const Tokenizer = struct {
423423 return token;
424424 }
425425 const start_index = self.index;
426 var state = State.Start;
426 var state: State = .start;
427427 var result = Token{
428 .id = Token.Id.Eof,
428 .id = .Eof,
429429 .start = self.index,
430430 .end = undefined,
431431 };
......@@ -434,40 +434,40 @@ pub const Tokenizer = struct {
434434 while (self.index < self.buffer.len) : (self.index += 1) {
435435 const c = self.buffer[self.index];
436436 switch (state) {
437 State.Start => switch (c) {
437 .start => switch (c) {
438438 ' ', '\n', '\t', '\r' => {
439439 result.start = self.index + 1;
440440 },
441441 '"' => {
442 state = State.StringLiteral;
443 result.id = Token.Id.StringLiteral;
442 state = .string_literal;
443 result.id = .StringLiteral;
444444 },
445445 '\'' => {
446 state = State.CharLiteral;
446 state = .char_literal;
447447 },
448448 'a'...'z', 'A'...'Z', '_' => {
449 state = State.Identifier;
450 result.id = Token.Id.Identifier;
449 state = .identifier;
450 result.id = .Identifier;
451451 },
452452 '@' => {
453 state = State.SawAtSign;
453 state = .saw_at_sign;
454454 },
455455 '=' => {
456 state = State.Equal;
456 state = .equal;
457457 },
458458 '!' => {
459 state = State.Bang;
459 state = .bang;
460460 },
461461 '|' => {
462 state = State.Pipe;
462 state = .pipe;
463463 },
464464 '(' => {
465 result.id = Token.Id.LParen;
465 result.id = .LParen;
466466 self.index += 1;
467467 break;
468468 },
469469 ')' => {
470 result.id = Token.Id.RParen;
470 result.id = .RParen;
471471 self.index += 1;
472472 break;
473473 },
......@@ -477,213 +477,213 @@ pub const Tokenizer = struct {
477477 break;
478478 },
479479 ']' => {
480 result.id = Token.Id.RBracket;
480 result.id = .RBracket;
481481 self.index += 1;
482482 break;
483483 },
484484 ';' => {
485 result.id = Token.Id.Semicolon;
485 result.id = .Semicolon;
486486 self.index += 1;
487487 break;
488488 },
489489 ',' => {
490 result.id = Token.Id.Comma;
490 result.id = .Comma;
491491 self.index += 1;
492492 break;
493493 },
494494 '?' => {
495 result.id = Token.Id.QuestionMark;
495 result.id = .QuestionMark;
496496 self.index += 1;
497497 break;
498498 },
499499 ':' => {
500 result.id = Token.Id.Colon;
500 result.id = .Colon;
501501 self.index += 1;
502502 break;
503503 },
504504 '%' => {
505 state = State.Percent;
505 state = .percent;
506506 },
507507 '*' => {
508 state = State.Asterisk;
508 state = .asterisk;
509509 },
510510 '+' => {
511 state = State.Plus;
511 state = .plus;
512512 },
513513 '<' => {
514 state = State.AngleBracketLeft;
514 state = .angle_bracket_left;
515515 },
516516 '>' => {
517 state = State.AngleBracketRight;
517 state = .angle_bracket_right;
518518 },
519519 '^' => {
520 state = State.Caret;
520 state = .caret;
521521 },
522522 '\\' => {
523 state = State.Backslash;
524 result.id = Token.Id.MultilineStringLiteralLine;
523 state = .backslash;
524 result.id = .MultilineStringLiteralLine;
525525 },
526526 '{' => {
527 result.id = Token.Id.LBrace;
527 result.id = .LBrace;
528528 self.index += 1;
529529 break;
530530 },
531531 '}' => {
532 result.id = Token.Id.RBrace;
532 result.id = .RBrace;
533533 self.index += 1;
534534 break;
535535 },
536536 '~' => {
537 result.id = Token.Id.Tilde;
537 result.id = .Tilde;
538538 self.index += 1;
539539 break;
540540 },
541541 '.' => {
542 state = State.Period;
542 state = .period;
543543 },
544544 '-' => {
545 state = State.Minus;
545 state = .minus;
546546 },
547547 '/' => {
548 state = State.Slash;
548 state = .slash;
549549 },
550550 '&' => {
551 state = State.Ampersand;
551 state = .ampersand;
552552 },
553553 '0' => {
554 state = State.Zero;
555 result.id = Token.Id.IntegerLiteral;
554 state = .zero;
555 result.id = .IntegerLiteral;
556556 },
557557 '1'...'9' => {
558 state = State.IntegerLiteralDec;
559 result.id = Token.Id.IntegerLiteral;
558 state = .int_literal_dec;
559 result.id = .IntegerLiteral;
560560 },
561561 else => {
562 result.id = Token.Id.Invalid;
562 result.id = .Invalid;
563563 self.index += 1;
564564 break;
565565 },
566566 },
567567
568 State.SawAtSign => switch (c) {
568 .saw_at_sign => switch (c) {
569569 '"' => {
570 result.id = Token.Id.Identifier;
571 state = State.StringLiteral;
570 result.id = .Identifier;
571 state = .string_literal;
572572 },
573573 else => {
574574 // reinterpret as a builtin
575575 self.index -= 1;
576 state = State.Builtin;
577 result.id = Token.Id.Builtin;
576 state = .builtin;
577 result.id = .Builtin;
578578 },
579579 },
580580
581 State.Ampersand => switch (c) {
581 .ampersand => switch (c) {
582582 '&' => {
583 result.id = Token.Id.Invalid_ampersands;
583 result.id = .Invalid_ampersands;
584584 self.index += 1;
585585 break;
586586 },
587587 '=' => {
588 result.id = Token.Id.AmpersandEqual;
588 result.id = .AmpersandEqual;
589589 self.index += 1;
590590 break;
591591 },
592592 else => {
593 result.id = Token.Id.Ampersand;
593 result.id = .Ampersand;
594594 break;
595595 },
596596 },
597597
598 State.Asterisk => switch (c) {
598 .asterisk => switch (c) {
599599 '=' => {
600 result.id = Token.Id.AsteriskEqual;
600 result.id = .AsteriskEqual;
601601 self.index += 1;
602602 break;
603603 },
604604 '*' => {
605 result.id = Token.Id.AsteriskAsterisk;
605 result.id = .AsteriskAsterisk;
606606 self.index += 1;
607607 break;
608608 },
609609 '%' => {
610 state = State.AsteriskPercent;
610 state = .asterisk_percent;
611611 },
612612 else => {
613 result.id = Token.Id.Asterisk;
613 result.id = .Asterisk;
614614 break;
615615 },
616616 },
617617
618 State.AsteriskPercent => switch (c) {
618 .asterisk_percent => switch (c) {
619619 '=' => {
620 result.id = Token.Id.AsteriskPercentEqual;
620 result.id = .AsteriskPercentEqual;
621621 self.index += 1;
622622 break;
623623 },
624624 else => {
625 result.id = Token.Id.AsteriskPercent;
625 result.id = .AsteriskPercent;
626626 break;
627627 },
628628 },
629629
630 State.Percent => switch (c) {
630 .percent => switch (c) {
631631 '=' => {
632 result.id = Token.Id.PercentEqual;
632 result.id = .PercentEqual;
633633 self.index += 1;
634634 break;
635635 },
636636 else => {
637 result.id = Token.Id.Percent;
637 result.id = .Percent;
638638 break;
639639 },
640640 },
641641
642 State.Plus => switch (c) {
642 .plus => switch (c) {
643643 '=' => {
644 result.id = Token.Id.PlusEqual;
644 result.id = .PlusEqual;
645645 self.index += 1;
646646 break;
647647 },
648648 '+' => {
649 result.id = Token.Id.PlusPlus;
649 result.id = .PlusPlus;
650650 self.index += 1;
651651 break;
652652 },
653653 '%' => {
654 state = State.PlusPercent;
654 state = .plus_percent;
655655 },
656656 else => {
657 result.id = Token.Id.Plus;
657 result.id = .Plus;
658658 break;
659659 },
660660 },
661661
662 State.PlusPercent => switch (c) {
662 .plus_percent => switch (c) {
663663 '=' => {
664 result.id = Token.Id.PlusPercentEqual;
664 result.id = .PlusPercentEqual;
665665 self.index += 1;
666666 break;
667667 },
668668 else => {
669 result.id = Token.Id.PlusPercent;
669 result.id = .PlusPercent;
670670 break;
671671 },
672672 },
673673
674 State.Caret => switch (c) {
674 .caret => switch (c) {
675675 '=' => {
676 result.id = Token.Id.CaretEqual;
676 result.id = .CaretEqual;
677677 self.index += 1;
678678 break;
679679 },
680680 else => {
681 result.id = Token.Id.Caret;
681 result.id = .Caret;
682682 break;
683683 },
684684 },
685685
686 State.Identifier => switch (c) {
686 .identifier => switch (c) {
687687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
688688 else => {
689689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
......@@ -692,19 +692,19 @@ pub const Tokenizer = struct {
692692 break;
693693 },
694694 },
695 State.Builtin => switch (c) {
695 .builtin => switch (c) {
696696 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
697697 else => break,
698698 },
699 State.Backslash => switch (c) {
699 .backslash => switch (c) {
700700 '\\' => {
701 state = State.MultilineStringLiteralLine;
701 state = .multiline_string_literal_line;
702702 },
703703 else => break,
704704 },
705 State.StringLiteral => switch (c) {
705 .string_literal => switch (c) {
706706 '\\' => {
707 state = State.StringLiteralBackslash;
707 state = .string_literal_backslash;
708708 },
709709 '"' => {
710710 self.index += 1;
......@@ -714,98 +714,98 @@ pub const Tokenizer = struct {
714714 else => self.checkLiteralCharacter(),
715715 },
716716
717 State.StringLiteralBackslash => switch (c) {
717 .string_literal_backslash => switch (c) {
718718 '\n', '\r' => break, // Look for this error later.
719719 else => {
720 state = State.StringLiteral;
720 state = .string_literal;
721721 },
722722 },
723723
724 State.CharLiteral => switch (c) {
724 .char_literal => switch (c) {
725725 '\\' => {
726 state = State.CharLiteralBackslash;
726 state = .char_literal_backslash;
727727 },
728728 '\'', 0x80...0xbf, 0xf8...0xff => {
729 result.id = Token.Id.Invalid;
729 result.id = .Invalid;
730730 break;
731731 },
732732 0xc0...0xdf => { // 110xxxxx
733733 remaining_code_units = 1;
734 state = State.CharLiteralUnicode;
734 state = .char_literal_unicode;
735735 },
736736 0xe0...0xef => { // 1110xxxx
737737 remaining_code_units = 2;
738 state = State.CharLiteralUnicode;
738 state = .char_literal_unicode;
739739 },
740740 0xf0...0xf7 => { // 11110xxx
741741 remaining_code_units = 3;
742 state = State.CharLiteralUnicode;
742 state = .char_literal_unicode;
743743 },
744744 else => {
745 state = State.CharLiteralEnd;
745 state = .char_literal_end;
746746 },
747747 },
748748
749 State.CharLiteralBackslash => switch (c) {
749 .char_literal_backslash => switch (c) {
750750 '\n' => {
751 result.id = Token.Id.Invalid;
751 result.id = .Invalid;
752752 break;
753753 },
754754 'x' => {
755 state = State.CharLiteralHexEscape;
755 state = .char_literal_hex_escape;
756756 seen_escape_digits = 0;
757757 },
758758 'u' => {
759 state = State.CharLiteralUnicodeEscapeSawU;
759 state = .char_literal_unicode_escape_saw_u;
760760 },
761761 else => {
762 state = State.CharLiteralEnd;
762 state = .char_literal_end;
763763 },
764764 },
765765
766 State.CharLiteralHexEscape => switch (c) {
766 .char_literal_hex_escape => switch (c) {
767767 '0'...'9', 'a'...'f', 'A'...'F' => {
768768 seen_escape_digits += 1;
769769 if (seen_escape_digits == 2) {
770 state = State.CharLiteralEnd;
770 state = .char_literal_end;
771771 }
772772 },
773773 else => {
774 result.id = Token.Id.Invalid;
774 result.id = .Invalid;
775775 break;
776776 },
777777 },
778778
779 State.CharLiteralUnicodeEscapeSawU => switch (c) {
779 .char_literal_unicode_escape_saw_u => switch (c) {
780780 '{' => {
781 state = State.CharLiteralUnicodeEscape;
781 state = .char_literal_unicode_escape;
782782 seen_escape_digits = 0;
783783 },
784784 else => {
785 result.id = Token.Id.Invalid;
786 state = State.CharLiteralUnicodeInvalid;
785 result.id = .Invalid;
786 state = .char_literal_unicode_invalid;
787787 },
788788 },
789789
790 State.CharLiteralUnicodeEscape => switch (c) {
790 .char_literal_unicode_escape => switch (c) {
791791 '0'...'9', 'a'...'f', 'A'...'F' => {
792792 seen_escape_digits += 1;
793793 },
794794 '}' => {
795795 if (seen_escape_digits == 0) {
796 result.id = Token.Id.Invalid;
797 state = State.CharLiteralUnicodeInvalid;
796 result.id = .Invalid;
797 state = .char_literal_unicode_invalid;
798798 } else {
799 state = State.CharLiteralEnd;
799 state = .char_literal_end;
800800 }
801801 },
802802 else => {
803 result.id = Token.Id.Invalid;
804 state = State.CharLiteralUnicodeInvalid;
803 result.id = .Invalid;
804 state = .char_literal_unicode_invalid;
805805 },
806806 },
807807
808 State.CharLiteralUnicodeInvalid => switch (c) {
808 .char_literal_unicode_invalid => switch (c) {
809809 // Keep consuming characters until an obvious stopping point.
810810 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
811811 // instead of creating the tokens `u{0ab1`, `Q`, `}`
......@@ -813,32 +813,32 @@ pub const Tokenizer = struct {
813813 else => break,
814814 },
815815
816 State.CharLiteralEnd => switch (c) {
816 .char_literal_end => switch (c) {
817817 '\'' => {
818 result.id = Token.Id.CharLiteral;
818 result.id = .CharLiteral;
819819 self.index += 1;
820820 break;
821821 },
822822 else => {
823 result.id = Token.Id.Invalid;
823 result.id = .Invalid;
824824 break;
825825 },
826826 },
827827
828 State.CharLiteralUnicode => switch (c) {
828 .char_literal_unicode => switch (c) {
829829 0x80...0xbf => {
830830 remaining_code_units -= 1;
831831 if (remaining_code_units == 0) {
832 state = State.CharLiteralEnd;
832 state = .char_literal_end;
833833 }
834834 },
835835 else => {
836 result.id = Token.Id.Invalid;
836 result.id = .Invalid;
837837 break;
838838 },
839839 },
840840
841 State.MultilineStringLiteralLine => switch (c) {
841 .multiline_string_literal_line => switch (c) {
842842 '\n' => {
843843 self.index += 1;
844844 break;
......@@ -847,449 +847,449 @@ pub const Tokenizer = struct {
847847 else => self.checkLiteralCharacter(),
848848 },
849849
850 State.Bang => switch (c) {
850 .bang => switch (c) {
851851 '=' => {
852 result.id = Token.Id.BangEqual;
852 result.id = .BangEqual;
853853 self.index += 1;
854854 break;
855855 },
856856 else => {
857 result.id = Token.Id.Bang;
857 result.id = .Bang;
858858 break;
859859 },
860860 },
861861
862 State.Pipe => switch (c) {
862 .pipe => switch (c) {
863863 '=' => {
864 result.id = Token.Id.PipeEqual;
864 result.id = .PipeEqual;
865865 self.index += 1;
866866 break;
867867 },
868868 '|' => {
869 result.id = Token.Id.PipePipe;
869 result.id = .PipePipe;
870870 self.index += 1;
871871 break;
872872 },
873873 else => {
874 result.id = Token.Id.Pipe;
874 result.id = .Pipe;
875875 break;
876876 },
877877 },
878878
879 State.Equal => switch (c) {
879 .equal => switch (c) {
880880 '=' => {
881 result.id = Token.Id.EqualEqual;
881 result.id = .EqualEqual;
882882 self.index += 1;
883883 break;
884884 },
885885 '>' => {
886 result.id = Token.Id.EqualAngleBracketRight;
886 result.id = .EqualAngleBracketRight;
887887 self.index += 1;
888888 break;
889889 },
890890 else => {
891 result.id = Token.Id.Equal;
891 result.id = .Equal;
892892 break;
893893 },
894894 },
895895
896 State.Minus => switch (c) {
896 .minus => switch (c) {
897897 '>' => {
898 result.id = Token.Id.Arrow;
898 result.id = .Arrow;
899899 self.index += 1;
900900 break;
901901 },
902902 '=' => {
903 result.id = Token.Id.MinusEqual;
903 result.id = .MinusEqual;
904904 self.index += 1;
905905 break;
906906 },
907907 '%' => {
908 state = State.MinusPercent;
908 state = .minus_percent;
909909 },
910910 else => {
911 result.id = Token.Id.Minus;
911 result.id = .Minus;
912912 break;
913913 },
914914 },
915915
916 State.MinusPercent => switch (c) {
916 .minus_percent => switch (c) {
917917 '=' => {
918 result.id = Token.Id.MinusPercentEqual;
918 result.id = .MinusPercentEqual;
919919 self.index += 1;
920920 break;
921921 },
922922 else => {
923 result.id = Token.Id.MinusPercent;
923 result.id = .MinusPercent;
924924 break;
925925 },
926926 },
927927
928 State.AngleBracketLeft => switch (c) {
928 .angle_bracket_left => switch (c) {
929929 '<' => {
930 state = State.AngleBracketAngleBracketLeft;
930 state = .angle_bracket_angle_bracket_left;
931931 },
932932 '=' => {
933 result.id = Token.Id.AngleBracketLeftEqual;
933 result.id = .AngleBracketLeftEqual;
934934 self.index += 1;
935935 break;
936936 },
937937 else => {
938 result.id = Token.Id.AngleBracketLeft;
938 result.id = .AngleBracketLeft;
939939 break;
940940 },
941941 },
942942
943 State.AngleBracketAngleBracketLeft => switch (c) {
943 .angle_bracket_angle_bracket_left => switch (c) {
944944 '=' => {
945 result.id = Token.Id.AngleBracketAngleBracketLeftEqual;
945 result.id = .AngleBracketAngleBracketLeftEqual;
946946 self.index += 1;
947947 break;
948948 },
949949 else => {
950 result.id = Token.Id.AngleBracketAngleBracketLeft;
950 result.id = .AngleBracketAngleBracketLeft;
951951 break;
952952 },
953953 },
954954
955 State.AngleBracketRight => switch (c) {
955 .angle_bracket_right => switch (c) {
956956 '>' => {
957 state = State.AngleBracketAngleBracketRight;
957 state = .angle_bracket_angle_bracket_right;
958958 },
959959 '=' => {
960 result.id = Token.Id.AngleBracketRightEqual;
960 result.id = .AngleBracketRightEqual;
961961 self.index += 1;
962962 break;
963963 },
964964 else => {
965 result.id = Token.Id.AngleBracketRight;
965 result.id = .AngleBracketRight;
966966 break;
967967 },
968968 },
969969
970 State.AngleBracketAngleBracketRight => switch (c) {
970 .angle_bracket_angle_bracket_right => switch (c) {
971971 '=' => {
972 result.id = Token.Id.AngleBracketAngleBracketRightEqual;
972 result.id = .AngleBracketAngleBracketRightEqual;
973973 self.index += 1;
974974 break;
975975 },
976976 else => {
977 result.id = Token.Id.AngleBracketAngleBracketRight;
977 result.id = .AngleBracketAngleBracketRight;
978978 break;
979979 },
980980 },
981981
982 State.Period => switch (c) {
982 .period => switch (c) {
983983 '.' => {
984 state = State.Period2;
984 state = .period_2;
985985 },
986986 '*' => {
987 result.id = Token.Id.PeriodAsterisk;
987 result.id = .PeriodAsterisk;
988988 self.index += 1;
989989 break;
990990 },
991991 else => {
992 result.id = Token.Id.Period;
992 result.id = .Period;
993993 break;
994994 },
995995 },
996996
997 State.Period2 => switch (c) {
997 .period_2 => switch (c) {
998998 '.' => {
999 result.id = Token.Id.Ellipsis3;
999 result.id = .Ellipsis3;
10001000 self.index += 1;
10011001 break;
10021002 },
10031003 else => {
1004 result.id = Token.Id.Ellipsis2;
1004 result.id = .Ellipsis2;
10051005 break;
10061006 },
10071007 },
10081008
1009 State.Slash => switch (c) {
1009 .slash => switch (c) {
10101010 '/' => {
1011 state = State.LineCommentStart;
1012 result.id = Token.Id.LineComment;
1011 state = .line_comment_start;
1012 result.id = .LineComment;
10131013 },
10141014 '=' => {
1015 result.id = Token.Id.SlashEqual;
1015 result.id = .SlashEqual;
10161016 self.index += 1;
10171017 break;
10181018 },
10191019 else => {
1020 result.id = Token.Id.Slash;
1020 result.id = .Slash;
10211021 break;
10221022 },
10231023 },
1024 State.LineCommentStart => switch (c) {
1024 .line_comment_start => switch (c) {
10251025 '/' => {
1026 state = State.DocCommentStart;
1026 state = .doc_comment_start;
10271027 },
10281028 '!' => {
1029 result.id = Token.Id.ContainerDocComment;
1030 state = State.ContainerDocComment;
1029 result.id = .ContainerDocComment;
1030 state = .container_doc_comment;
10311031 },
10321032 '\n' => break,
10331033 else => {
1034 state = State.LineComment;
1034 state = .line_comment;
10351035 self.checkLiteralCharacter();
10361036 },
10371037 },
1038 State.DocCommentStart => switch (c) {
1038 .doc_comment_start => switch (c) {
10391039 '/' => {
1040 state = State.LineComment;
1040 state = .line_comment;
10411041 },
10421042 '\n' => {
1043 result.id = Token.Id.DocComment;
1043 result.id = .DocComment;
10441044 break;
10451045 },
10461046 else => {
1047 state = State.DocComment;
1048 result.id = Token.Id.DocComment;
1047 state = .doc_comment;
1048 result.id = .DocComment;
10491049 self.checkLiteralCharacter();
10501050 },
10511051 },
1052 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {
1052 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
10531053 '\n' => break,
10541054 else => self.checkLiteralCharacter(),
10551055 },
1056 State.Zero => switch (c) {
1056 .zero => switch (c) {
10571057 'b' => {
1058 state = State.IntegerLiteralBinNoUnderscore;
1058 state = .int_literal_bin_no_underscore;
10591059 },
10601060 'o' => {
1061 state = State.IntegerLiteralOctNoUnderscore;
1061 state = .int_literal_oct_no_underscore;
10621062 },
10631063 'x' => {
1064 state = State.IntegerLiteralHexNoUnderscore;
1064 state = .int_literal_hex_no_underscore;
10651065 },
10661066 '0'...'9', '_', '.', 'e', 'E' => {
10671067 // reinterpret as a decimal number
10681068 self.index -= 1;
1069 state = State.IntegerLiteralDec;
1069 state = .int_literal_dec;
10701070 },
10711071 else => {
10721072 if (isIdentifierChar(c)) {
1073 result.id = Token.Id.Invalid;
1073 result.id = .Invalid;
10741074 }
10751075 break;
10761076 },
10771077 },
1078 State.IntegerLiteralBinNoUnderscore => switch (c) {
1078 .int_literal_bin_no_underscore => switch (c) {
10791079 '0'...'1' => {
1080 state = State.IntegerLiteralBin;
1080 state = .int_literal_bin;
10811081 },
10821082 else => {
1083 result.id = Token.Id.Invalid;
1083 result.id = .Invalid;
10841084 break;
10851085 },
10861086 },
1087 State.IntegerLiteralBin => switch (c) {
1087 .int_literal_bin => switch (c) {
10881088 '_' => {
1089 state = State.IntegerLiteralBinNoUnderscore;
1089 state = .int_literal_bin_no_underscore;
10901090 },
10911091 '0'...'1' => {},
10921092 else => {
10931093 if (isIdentifierChar(c)) {
1094 result.id = Token.Id.Invalid;
1094 result.id = .Invalid;
10951095 }
10961096 break;
10971097 },
10981098 },
1099 State.IntegerLiteralOctNoUnderscore => switch (c) {
1099 .int_literal_oct_no_underscore => switch (c) {
11001100 '0'...'7' => {
1101 state = State.IntegerLiteralOct;
1101 state = .int_literal_oct;
11021102 },
11031103 else => {
1104 result.id = Token.Id.Invalid;
1104 result.id = .Invalid;
11051105 break;
11061106 },
11071107 },
1108 State.IntegerLiteralOct => switch (c) {
1108 .int_literal_oct => switch (c) {
11091109 '_' => {
1110 state = State.IntegerLiteralOctNoUnderscore;
1110 state = .int_literal_oct_no_underscore;
11111111 },
11121112 '0'...'7' => {},
11131113 else => {
11141114 if (isIdentifierChar(c)) {
1115 result.id = Token.Id.Invalid;
1115 result.id = .Invalid;
11161116 }
11171117 break;
11181118 },
11191119 },
1120 State.IntegerLiteralDecNoUnderscore => switch (c) {
1120 .int_literal_dec_no_underscore => switch (c) {
11211121 '0'...'9' => {
1122 state = State.IntegerLiteralDec;
1122 state = .int_literal_dec;
11231123 },
11241124 else => {
1125 result.id = Token.Id.Invalid;
1125 result.id = .Invalid;
11261126 break;
11271127 },
11281128 },
1129 State.IntegerLiteralDec => switch (c) {
1129 .int_literal_dec => switch (c) {
11301130 '_' => {
1131 state = State.IntegerLiteralDecNoUnderscore;
1131 state = .int_literal_dec_no_underscore;
11321132 },
11331133 '.' => {
1134 state = State.NumberDotDec;
1135 result.id = Token.Id.FloatLiteral;
1134 state = .num_dot_dec;
1135 result.id = .FloatLiteral;
11361136 },
11371137 'e', 'E' => {
1138 state = State.FloatExponentUnsigned;
1139 result.id = Token.Id.FloatLiteral;
1138 state = .float_exponent_unsigned;
1139 result.id = .FloatLiteral;
11401140 },
11411141 '0'...'9' => {},
11421142 else => {
11431143 if (isIdentifierChar(c)) {
1144 result.id = Token.Id.Invalid;
1144 result.id = .Invalid;
11451145 }
11461146 break;
11471147 },
11481148 },
1149 State.IntegerLiteralHexNoUnderscore => switch (c) {
1149 .int_literal_hex_no_underscore => switch (c) {
11501150 '0'...'9', 'a'...'f', 'A'...'F' => {
1151 state = State.IntegerLiteralHex;
1151 state = .int_literal_hex;
11521152 },
11531153 else => {
1154 result.id = Token.Id.Invalid;
1154 result.id = .Invalid;
11551155 break;
11561156 },
11571157 },
1158 State.IntegerLiteralHex => switch (c) {
1158 .int_literal_hex => switch (c) {
11591159 '_' => {
1160 state = State.IntegerLiteralHexNoUnderscore;
1160 state = .int_literal_hex_no_underscore;
11611161 },
11621162 '.' => {
1163 state = State.NumberDotHex;
1164 result.id = Token.Id.FloatLiteral;
1163 state = .num_dot_hex;
1164 result.id = .FloatLiteral;
11651165 },
11661166 'p', 'P' => {
1167 state = State.FloatExponentUnsigned;
1168 result.id = Token.Id.FloatLiteral;
1167 state = .float_exponent_unsigned;
1168 result.id = .FloatLiteral;
11691169 },
11701170 '0'...'9', 'a'...'f', 'A'...'F' => {},
11711171 else => {
11721172 if (isIdentifierChar(c)) {
1173 result.id = Token.Id.Invalid;
1173 result.id = .Invalid;
11741174 }
11751175 break;
11761176 },
11771177 },
1178 State.NumberDotDec => switch (c) {
1178 .num_dot_dec => switch (c) {
11791179 '.' => {
11801180 self.index -= 1;
1181 state = State.Start;
1181 state = .start;
11821182 break;
11831183 },
11841184 'e', 'E' => {
1185 state = State.FloatExponentUnsigned;
1185 state = .float_exponent_unsigned;
11861186 },
11871187 '0'...'9' => {
1188 result.id = Token.Id.FloatLiteral;
1189 state = State.FloatFractionDec;
1188 result.id = .FloatLiteral;
1189 state = .float_fraction_dec;
11901190 },
11911191 else => {
11921192 if (isIdentifierChar(c)) {
1193 result.id = Token.Id.Invalid;
1193 result.id = .Invalid;
11941194 }
11951195 break;
11961196 },
11971197 },
1198 State.NumberDotHex => switch (c) {
1198 .num_dot_hex => switch (c) {
11991199 '.' => {
12001200 self.index -= 1;
1201 state = State.Start;
1201 state = .start;
12021202 break;
12031203 },
12041204 'p', 'P' => {
1205 state = State.FloatExponentUnsigned;
1205 state = .float_exponent_unsigned;
12061206 },
12071207 '0'...'9', 'a'...'f', 'A'...'F' => {
1208 result.id = Token.Id.FloatLiteral;
1209 state = State.FloatFractionHex;
1208 result.id = .FloatLiteral;
1209 state = .float_fraction_hex;
12101210 },
12111211 else => {
12121212 if (isIdentifierChar(c)) {
1213 result.id = Token.Id.Invalid;
1213 result.id = .Invalid;
12141214 }
12151215 break;
12161216 },
12171217 },
1218 State.FloatFractionDecNoUnderscore => switch (c) {
1218 .float_fraction_dec_no_underscore => switch (c) {
12191219 '0'...'9' => {
1220 state = State.FloatFractionDec;
1220 state = .float_fraction_dec;
12211221 },
12221222 else => {
1223 result.id = Token.Id.Invalid;
1223 result.id = .Invalid;
12241224 break;
12251225 },
12261226 },
1227 State.FloatFractionDec => switch (c) {
1227 .float_fraction_dec => switch (c) {
12281228 '_' => {
1229 state = State.FloatFractionDecNoUnderscore;
1229 state = .float_fraction_dec_no_underscore;
12301230 },
12311231 'e', 'E' => {
1232 state = State.FloatExponentUnsigned;
1232 state = .float_exponent_unsigned;
12331233 },
12341234 '0'...'9' => {},
12351235 else => {
12361236 if (isIdentifierChar(c)) {
1237 result.id = Token.Id.Invalid;
1237 result.id = .Invalid;
12381238 }
12391239 break;
12401240 },
12411241 },
1242 State.FloatFractionHexNoUnderscore => switch (c) {
1242 .float_fraction_hex_no_underscore => switch (c) {
12431243 '0'...'9', 'a'...'f', 'A'...'F' => {
1244 state = State.FloatFractionHex;
1244 state = .float_fraction_hex;
12451245 },
12461246 else => {
1247 result.id = Token.Id.Invalid;
1247 result.id = .Invalid;
12481248 break;
12491249 },
12501250 },
1251 State.FloatFractionHex => switch (c) {
1251 .float_fraction_hex => switch (c) {
12521252 '_' => {
1253 state = State.FloatFractionHexNoUnderscore;
1253 state = .float_fraction_hex_no_underscore;
12541254 },
12551255 'p', 'P' => {
1256 state = State.FloatExponentUnsigned;
1256 state = .float_exponent_unsigned;
12571257 },
12581258 '0'...'9', 'a'...'f', 'A'...'F' => {},
12591259 else => {
12601260 if (isIdentifierChar(c)) {
1261 result.id = Token.Id.Invalid;
1261 result.id = .Invalid;
12621262 }
12631263 break;
12641264 },
12651265 },
1266 State.FloatExponentUnsigned => switch (c) {
1266 .float_exponent_unsigned => switch (c) {
12671267 '+', '-' => {
1268 state = State.FloatExponentNumberNoUnderscore;
1268 state = .float_exponent_num_no_underscore;
12691269 },
12701270 else => {
12711271 // reinterpret as a normal exponent number
12721272 self.index -= 1;
1273 state = State.FloatExponentNumberNoUnderscore;
1273 state = .float_exponent_num_no_underscore;
12741274 },
12751275 },
1276 State.FloatExponentNumberNoUnderscore => switch (c) {
1276 .float_exponent_num_no_underscore => switch (c) {
12771277 '0'...'9' => {
1278 state = State.FloatExponentNumber;
1278 state = .float_exponent_num;
12791279 },
12801280 else => {
1281 result.id = Token.Id.Invalid;
1281 result.id = .Invalid;
12821282 break;
12831283 },
12841284 },
1285 State.FloatExponentNumber => switch (c) {
1285 .float_exponent_num => switch (c) {
12861286 '_' => {
1287 state = State.FloatExponentNumberNoUnderscore;
1287 state = .float_exponent_num_no_underscore;
12881288 },
12891289 '0'...'9' => {},
12901290 else => {
12911291 if (isIdentifierChar(c)) {
1292 result.id = Token.Id.Invalid;
1292 result.id = .Invalid;
12931293 }
12941294 break;
12951295 },
......@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {
12971297 }
12981298 } else if (self.index == self.buffer.len) {
12991299 switch (state) {
1300 State.Start,
1301 State.IntegerLiteralDec,
1302 State.IntegerLiteralBin,
1303 State.IntegerLiteralOct,
1304 State.IntegerLiteralHex,
1305 State.NumberDotDec,
1306 State.NumberDotHex,
1307 State.FloatFractionDec,
1308 State.FloatFractionHex,
1309 State.FloatExponentNumber,
1310 State.StringLiteral, // find this error later
1311 State.MultilineStringLiteralLine,
1312 State.Builtin,
1300 .start,
1301 .int_literal_dec,
1302 .int_literal_bin,
1303 .int_literal_oct,
1304 .int_literal_hex,
1305 .num_dot_dec,
1306 .num_dot_hex,
1307 .float_fraction_dec,
1308 .float_fraction_hex,
1309 .float_exponent_num,
1310 .string_literal, // find this error later
1311 .multiline_string_literal_line,
1312 .builtin,
13131313 => {},
13141314
1315 State.Identifier => {
1315 .identifier => {
13161316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
13171317 result.id = id;
13181318 }
13191319 },
1320 State.LineCommentStart, State.LineComment => {
1321 result.id = Token.Id.LineComment;
1322 },
1323 State.DocComment, State.DocCommentStart => {
1324 result.id = Token.Id.DocComment;
1325 },
1326 State.ContainerDocComment => {
1327 result.id = Token.Id.ContainerDocComment;
1328 },
1329
1330 State.IntegerLiteralDecNoUnderscore,
1331 State.IntegerLiteralBinNoUnderscore,
1332 State.IntegerLiteralOctNoUnderscore,
1333 State.IntegerLiteralHexNoUnderscore,
1334 State.FloatFractionDecNoUnderscore,
1335 State.FloatFractionHexNoUnderscore,
1336 State.FloatExponentNumberNoUnderscore,
1337 State.FloatExponentUnsigned,
1338 State.SawAtSign,
1339 State.Backslash,
1340 State.CharLiteral,
1341 State.CharLiteralBackslash,
1342 State.CharLiteralHexEscape,
1343 State.CharLiteralUnicodeEscapeSawU,
1344 State.CharLiteralUnicodeEscape,
1345 State.CharLiteralUnicodeInvalid,
1346 State.CharLiteralEnd,
1347 State.CharLiteralUnicode,
1348 State.StringLiteralBackslash,
1320 .line_comment, .line_comment_start => {
1321 result.id = .LineComment;
1322 },
1323 .doc_comment, .doc_comment_start => {
1324 result.id = .DocComment;
1325 },
1326 .container_doc_comment => {
1327 result.id = .ContainerDocComment;
1328 },
1329
1330 .int_literal_dec_no_underscore,
1331 .int_literal_bin_no_underscore,
1332 .int_literal_oct_no_underscore,
1333 .int_literal_hex_no_underscore,
1334 .float_fraction_dec_no_underscore,
1335 .float_fraction_hex_no_underscore,
1336 .float_exponent_num_no_underscore,
1337 .float_exponent_unsigned,
1338 .saw_at_sign,
1339 .backslash,
1340 .char_literal,
1341 .char_literal_backslash,
1342 .char_literal_hex_escape,
1343 .char_literal_unicode_escape_saw_u,
1344 .char_literal_unicode_escape,
1345 .char_literal_unicode_invalid,
1346 .char_literal_end,
1347 .char_literal_unicode,
1348 .string_literal_backslash,
13491349 => {
1350 result.id = Token.Id.Invalid;
1350 result.id = .Invalid;
13511351 },
13521352
1353 State.Equal => {
1354 result.id = Token.Id.Equal;
1353 .equal => {
1354 result.id = .Equal;
13551355 },
1356 State.Bang => {
1357 result.id = Token.Id.Bang;
1356 .bang => {
1357 result.id = .Bang;
13581358 },
1359 State.Minus => {
1360 result.id = Token.Id.Minus;
1359 .minus => {
1360 result.id = .Minus;
13611361 },
1362 State.Slash => {
1363 result.id = Token.Id.Slash;
1362 .slash => {
1363 result.id = .Slash;
13641364 },
1365 State.Zero => {
1366 result.id = Token.Id.IntegerLiteral;
1365 .zero => {
1366 result.id = .IntegerLiteral;
13671367 },
1368 State.Ampersand => {
1369 result.id = Token.Id.Ampersand;
1368 .ampersand => {
1369 result.id = .Ampersand;
13701370 },
1371 State.Period => {
1372 result.id = Token.Id.Period;
1371 .period => {
1372 result.id = .Period;
13731373 },
1374 State.Period2 => {
1375 result.id = Token.Id.Ellipsis2;
1374 .period_2 => {
1375 result.id = .Ellipsis2;
13761376 },
1377 State.Pipe => {
1378 result.id = Token.Id.Pipe;
1377 .pipe => {
1378 result.id = .Pipe;
13791379 },
1380 State.AngleBracketAngleBracketRight => {
1381 result.id = Token.Id.AngleBracketAngleBracketRight;
1380 .angle_bracket_angle_bracket_right => {
1381 result.id = .AngleBracketAngleBracketRight;
13821382 },
1383 State.AngleBracketRight => {
1384 result.id = Token.Id.AngleBracketRight;
1383 .angle_bracket_right => {
1384 result.id = .AngleBracketRight;
13851385 },
1386 State.AngleBracketAngleBracketLeft => {
1387 result.id = Token.Id.AngleBracketAngleBracketLeft;
1386 .angle_bracket_angle_bracket_left => {
1387 result.id = .AngleBracketAngleBracketLeft;
13881388 },
1389 State.AngleBracketLeft => {
1390 result.id = Token.Id.AngleBracketLeft;
1389 .angle_bracket_left => {
1390 result.id = .AngleBracketLeft;
13911391 },
1392 State.PlusPercent => {
1393 result.id = Token.Id.PlusPercent;
1392 .plus_percent => {
1393 result.id = .PlusPercent;
13941394 },
1395 State.Plus => {
1396 result.id = Token.Id.Plus;
1395 .plus => {
1396 result.id = .Plus;
13971397 },
1398 State.Percent => {
1399 result.id = Token.Id.Percent;
1398 .percent => {
1399 result.id = .Percent;
14001400 },
1401 State.Caret => {
1402 result.id = Token.Id.Caret;
1401 .caret => {
1402 result.id = .Caret;
14031403 },
1404 State.AsteriskPercent => {
1405 result.id = Token.Id.AsteriskPercent;
1404 .asterisk_percent => {
1405 result.id = .AsteriskPercent;
14061406 },
1407 State.Asterisk => {
1408 result.id = Token.Id.Asterisk;
1407 .asterisk => {
1408 result.id = .Asterisk;
14091409 },
1410 State.MinusPercent => {
1411 result.id = Token.Id.MinusPercent;
1410 .minus_percent => {
1411 result.id = .MinusPercent;
14121412 },
14131413 }
14141414 }
14151415
1416 if (result.id == Token.Id.Eof) {
1416 if (result.id == .Eof) {
14171417 if (self.pending_invalid_token) |token| {
14181418 self.pending_invalid_token = null;
14191419 return token;
......@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {
14281428 if (self.pending_invalid_token != null) return;
14291429 const invalid_length = self.getInvalidCharacterLength();
14301430 if (invalid_length == 0) return;
1431 self.pending_invalid_token = Token{
1432 .id = Token.Id.Invalid,
1431 self.pending_invalid_token = .{
1432 .id = .Invalid,
14331433 .start = self.index,
14341434 .end = self.index + invalid_length,
14351435 };
......@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {
14741474};
14751475
14761476test "tokenizer" {
1477 testTokenize("test", &[_]Token.Id{Token.Id.Keyword_test});
1477 testTokenize("test", &[_]Token.Id{.Keyword_test});
14781478}
14791479
14801480test "tokenizer - unknown length pointer and then c pointer" {
......@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {
14821482 \\[*]u8
14831483 \\[*c]u8
14841484 , &[_]Token.Id{
1485 Token.Id.LBracket,
1486 Token.Id.Asterisk,
1487 Token.Id.RBracket,
1488 Token.Id.Identifier,
1489 Token.Id.LBracket,
1490 Token.Id.Asterisk,
1491 Token.Id.Identifier,
1492 Token.Id.RBracket,
1493 Token.Id.Identifier,
1485 .LBracket,
1486 .Asterisk,
1487 .RBracket,
1488 .Identifier,
1489 .LBracket,
1490 .Asterisk,
1491 .Identifier,
1492 .RBracket,
1493 .Identifier,
14941494 });
14951495}
14961496
......@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {
15611561
15621562test "tokenizer - float literal e exponent" {
15631563 testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{
1564 Token.Id.Identifier,
1565 Token.Id.Equal,
1566 Token.Id.FloatLiteral,
1567 Token.Id.Semicolon,
1564 .Identifier,
1565 .Equal,
1566 .FloatLiteral,
1567 .Semicolon,
15681568 });
15691569}
15701570
15711571test "tokenizer - float literal p exponent" {
15721572 testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{
1573 Token.Id.Identifier,
1574 Token.Id.Equal,
1575 Token.Id.FloatLiteral,
1576 Token.Id.Semicolon,
1573 .Identifier,
1574 .Equal,
1575 .FloatLiteral,
1576 .Semicolon,
15771577 });
15781578}
15791579
15801580test "tokenizer - chars" {
1581 testTokenize("'c'", &[_]Token.Id{Token.Id.CharLiteral});
1581 testTokenize("'c'", &[_]Token.Id{.CharLiteral});
15821582}
15831583
15841584test "tokenizer - invalid token characters" {
1585 testTokenize("#", &[_]Token.Id{Token.Id.Invalid});
1586 testTokenize("`", &[_]Token.Id{Token.Id.Invalid});
1587 testTokenize("'c", &[_]Token.Id{Token.Id.Invalid});
1588 testTokenize("'", &[_]Token.Id{Token.Id.Invalid});
1589 testTokenize("''", &[_]Token.Id{ Token.Id.Invalid, Token.Id.Invalid });
1585 testTokenize("#", &[_]Token.Id{.Invalid});
1586 testTokenize("`", &[_]Token.Id{.Invalid});
1587 testTokenize("'c", &[_]Token.Id{.Invalid});
1588 testTokenize("'", &[_]Token.Id{.Invalid});
1589 testTokenize("''", &[_]Token.Id{ .Invalid, .Invalid });
15901590}
15911591
15921592test "tokenizer - invalid literal/comment characters" {
15931593 testTokenize("\"\x00\"", &[_]Token.Id{
1594 Token.Id.StringLiteral,
1595 Token.Id.Invalid,
1594 .StringLiteral,
1595 .Invalid,
15961596 });
15971597 testTokenize("//\x00", &[_]Token.Id{
1598 Token.Id.LineComment,
1599 Token.Id.Invalid,
1598 .LineComment,
1599 .Invalid,
16001600 });
16011601 testTokenize("//\x1f", &[_]Token.Id{
1602 Token.Id.LineComment,
1603 Token.Id.Invalid,
1602 .LineComment,
1603 .Invalid,
16041604 });
16051605 testTokenize("//\x7f", &[_]Token.Id{
1606 Token.Id.LineComment,
1607 Token.Id.Invalid,
1606 .LineComment,
1607 .Invalid,
16081608 });
16091609}
16101610
16111611test "tokenizer - utf8" {
1612 testTokenize("//\xc2\x80", &[_]Token.Id{Token.Id.LineComment});
1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{Token.Id.LineComment});
1612 testTokenize("//\xc2\x80", &[_]Token.Id{.LineComment});
1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{.LineComment});
16141614}
16151615
16161616test "tokenizer - invalid utf8" {
16171617 testTokenize("//\x80", &[_]Token.Id{
1618 Token.Id.LineComment,
1619 Token.Id.Invalid,
1618 .LineComment,
1619 .Invalid,
16201620 });
16211621 testTokenize("//\xbf", &[_]Token.Id{
1622 Token.Id.LineComment,
1623 Token.Id.Invalid,
1622 .LineComment,
1623 .Invalid,
16241624 });
16251625 testTokenize("//\xf8", &[_]Token.Id{
1626 Token.Id.LineComment,
1627 Token.Id.Invalid,
1626 .LineComment,
1627 .Invalid,
16281628 });
16291629 testTokenize("//\xff", &[_]Token.Id{
1630 Token.Id.LineComment,
1631 Token.Id.Invalid,
1630 .LineComment,
1631 .Invalid,
16321632 });
16331633 testTokenize("//\xc2\xc0", &[_]Token.Id{
1634 Token.Id.LineComment,
1635 Token.Id.Invalid,
1634 .LineComment,
1635 .Invalid,
16361636 });
16371637 testTokenize("//\xe0", &[_]Token.Id{
1638 Token.Id.LineComment,
1639 Token.Id.Invalid,
1638 .LineComment,
1639 .Invalid,
16401640 });
16411641 testTokenize("//\xf0", &[_]Token.Id{
1642 Token.Id.LineComment,
1643 Token.Id.Invalid,
1642 .LineComment,
1643 .Invalid,
16441644 });
16451645 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{
1646 Token.Id.LineComment,
1647 Token.Id.Invalid,
1646 .LineComment,
1647 .Invalid,
16481648 });
16491649}
16501650
16511651test "tokenizer - illegal unicode codepoints" {
16521652 // unicode newline characters.U+0085, U+2028, U+2029
1653 testTokenize("//\xc2\x84", &[_]Token.Id{Token.Id.LineComment});
1653 testTokenize("//\xc2\x84", &[_]Token.Id{.LineComment});
16541654 testTokenize("//\xc2\x85", &[_]Token.Id{
1655 Token.Id.LineComment,
1656 Token.Id.Invalid,
1655 .LineComment,
1656 .Invalid,
16571657 });
1658 testTokenize("//\xc2\x86", &[_]Token.Id{Token.Id.LineComment});
1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{Token.Id.LineComment});
1658 testTokenize("//\xc2\x86", &[_]Token.Id{.LineComment});
1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{.LineComment});
16601660 testTokenize("//\xe2\x80\xa8", &[_]Token.Id{
1661 Token.Id.LineComment,
1662 Token.Id.Invalid,
1661 .LineComment,
1662 .Invalid,
16631663 });
16641664 testTokenize("//\xe2\x80\xa9", &[_]Token.Id{
1665 Token.Id.LineComment,
1666 Token.Id.Invalid,
1665 .LineComment,
1666 .Invalid,
16671667 });
1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{Token.Id.LineComment});
1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{.LineComment});
16691669}
16701670
16711671test "tokenizer - string identifier and builtin fns" {
16721672 testTokenize(
16731673 \\const @"if" = @import("std");
16741674 , &[_]Token.Id{
1675 Token.Id.Keyword_const,
1676 Token.Id.Identifier,
1677 Token.Id.Equal,
1678 Token.Id.Builtin,
1679 Token.Id.LParen,
1680 Token.Id.StringLiteral,
1681 Token.Id.RParen,
1682 Token.Id.Semicolon,
1675 .Keyword_const,
1676 .Identifier,
1677 .Equal,
1678 .Builtin,
1679 .LParen,
1680 .StringLiteral,
1681 .RParen,
1682 .Semicolon,
16831683 });
16841684}
16851685
......@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {
16871687 testTokenize(
16881688 \\\\foo bar
16891689 , &[_]Token.Id{
1690 Token.Id.MultilineStringLiteralLine,
1690 .MultilineStringLiteralLine,
16911691 });
16921692}
16931693
16941694test "tokenizer - pipe and then invalid" {
16951695 testTokenize("||=", &[_]Token.Id{
1696 Token.Id.PipePipe,
1697 Token.Id.Equal,
1696 .PipePipe,
1697 .Equal,
16981698 });
16991699}
17001700
17011701test "tokenizer - line comment and doc comment" {
1702 testTokenize("//", &[_]Token.Id{Token.Id.LineComment});
1703 testTokenize("// a / b", &[_]Token.Id{Token.Id.LineComment});
1704 testTokenize("// /", &[_]Token.Id{Token.Id.LineComment});
1705 testTokenize("/// a", &[_]Token.Id{Token.Id.DocComment});
1706 testTokenize("///", &[_]Token.Id{Token.Id.DocComment});
1707 testTokenize("////", &[_]Token.Id{Token.Id.LineComment});
1708 testTokenize("//!", &[_]Token.Id{Token.Id.ContainerDocComment});
1709 testTokenize("//!!", &[_]Token.Id{Token.Id.ContainerDocComment});
1702 testTokenize("//", &[_]Token.Id{.LineComment});
1703 testTokenize("// a / b", &[_]Token.Id{.LineComment});
1704 testTokenize("// /", &[_]Token.Id{.LineComment});
1705 testTokenize("/// a", &[_]Token.Id{.DocComment});
1706 testTokenize("///", &[_]Token.Id{.DocComment});
1707 testTokenize("////", &[_]Token.Id{.LineComment});
1708 testTokenize("//!", &[_]Token.Id{.ContainerDocComment});
1709 testTokenize("//!!", &[_]Token.Id{.ContainerDocComment});
17101710}
17111711
17121712test "tokenizer - line comment followed by identifier" {
......@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {
17151715 \\ // another
17161716 \\ Another,
17171717 , &[_]Token.Id{
1718 Token.Id.Identifier,
1719 Token.Id.Comma,
1720 Token.Id.LineComment,
1721 Token.Id.Identifier,
1722 Token.Id.Comma,
1718 .Identifier,
1719 .Comma,
1720 .LineComment,
1721 .Identifier,
1722 .Comma,
17231723 });
17241724}
17251725
17261726test "tokenizer - UTF-8 BOM is recognized and skipped" {
17271727 testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{
1728 Token.Id.Identifier,
1729 Token.Id.Semicolon,
1728 .Identifier,
1729 .Semicolon,
17301730 });
17311731}
17321732
17331733test "correctly parse pointer assignment" {
17341734 testTokenize("b.*=3;\n", &[_]Token.Id{
1735 Token.Id.Identifier,
1736 Token.Id.PeriodAsterisk,
1737 Token.Id.Equal,
1738 Token.Id.IntegerLiteral,
1739 Token.Id.Semicolon,
1735 .Identifier,
1736 .PeriodAsterisk,
1737 .Equal,
1738 .IntegerLiteral,
1739 .Semicolon,
17401740 });
17411741}
17421742
......@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
19791979 }
19801980 }
19811981 const last_token = tokenizer.next();
1982 std.testing.expect(last_token.id == Token.Id.Eof);
1982 std.testing.expect(last_token.id == .Eof);
19831983}
src-self-hosted/translate_c.zig+3-3
......@@ -668,7 +668,7 @@ fn transTypeDefAsBuiltin(c: *Context, typedef_decl: *const ZigClangTypedefNameDe
668668 return transCreateNodeIdentifier(c, builtin_name);
669669}
670670
671fn checkForBuiltinTypedef(checked_name: []const u8) !?[]const u8 {
671fn checkForBuiltinTypedef(checked_name: []const u8) ?[]const u8 {
672672 const table = [_][2][]const u8{
673673 .{ "uint8_t", "u8" },
674674 .{ "int8_t", "i8" },
......@@ -703,7 +703,7 @@ fn transTypeDef(c: *Context, typedef_decl: *const ZigClangTypedefNameDecl, top_l
703703 // TODO https://github.com/ziglang/zig/issues/3756
704704 // TODO https://github.com/ziglang/zig/issues/1802
705705 const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.a(), "{}_{}", .{ typedef_name, c.getMangle() }) else typedef_name;
706 if (try checkForBuiltinTypedef(checked_name)) |builtin| {
706 if (checkForBuiltinTypedef(checked_name)) |builtin| {
707707 return transTypeDefAsBuiltin(c, typedef_decl, builtin);
708708 }
709709
......@@ -1411,7 +1411,7 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt)
14111411 const underlying_type = ZigClangQualType_getTypePtr(underlying_qual);
14121412
14131413 const mangled_name = try block_scope.makeMangledName(c, name);
1414 if (try checkForBuiltinTypedef(name)) |builtin| {
1414 if (checkForBuiltinTypedef(name)) |builtin| {
14151415 try block_scope.variables.push(.{
14161416 .alias = builtin,
14171417 .name = mangled_name,
src/parser.cpp-9
......@@ -1609,7 +1609,6 @@ static AstNode *ast_parse_suffix_expr(ParseContext *pc) {
16091609// / IfTypeExpr
16101610// / INTEGER
16111611// / KEYWORD_comptime TypeExpr
1612// / KEYWORD_nosuspend TypeExpr
16131612// / KEYWORD_error DOT IDENTIFIER
16141613// / KEYWORD_false
16151614// / KEYWORD_null
......@@ -1711,14 +1710,6 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {
17111710 return res;
17121711 }
17131712
1714 Token *nosuspend = eat_token_if(pc, TokenIdKeywordNoSuspend);
1715 if (nosuspend != nullptr) {
1716 AstNode *expr = ast_expect(pc, ast_parse_type_expr);
1717 AstNode *res = ast_create_node(pc, NodeTypeNoSuspend, nosuspend);
1718 res->data.nosuspend_expr.expr = expr;
1719 return res;
1720 }
1721
17221713 Token *error = eat_token_if(pc, TokenIdKeywordError);
17231714 if (error != nullptr) {
17241715 Token *dot = expect_token(pc, TokenIdDot);