authorgravatar for git@vexu.euVeikka Tuominen <git@vexu.eu> 2020-05-15 14:15:30+03:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-05-15 14:15:30+03:00
logf8b99331a2ca98f0e938c8caaf1cd232ad1e9fa3
treeaa74657a7023f839462bf2512b4ed4db4616243f
parent4b898893e21fc644c4e7a163232e5e98631640d6
parent440189a04ae4baa4a20114fe1d30f0eb585bacc4
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5336 from Vexu/parser

Make self-hosted parser more error tolerant

8 files changed, 949 insertions(+), 580 deletions(-)

doc/langref.html.in+3
...@@ -10104,6 +10104,7 @@ ContainerField &lt;- IDENTIFIER (COLON TypeExpr)? (EQUAL Expr)?...@@ -10104,6 +10104,7 @@ ContainerField &lt;- IDENTIFIER (COLON TypeExpr)? (EQUAL Expr)?
10104Statement10104Statement
10105 &lt;- KEYWORD_comptime? VarDecl10105 &lt;- KEYWORD_comptime? VarDecl
10106 / KEYWORD_comptime BlockExprStatement10106 / KEYWORD_comptime BlockExprStatement
10107 / KEYWORD_nosuspend BlockExprStatement
10107 / KEYWORD_suspend (SEMICOLON / BlockExprStatement)10108 / KEYWORD_suspend (SEMICOLON / BlockExprStatement)
10108 / KEYWORD_defer BlockExprStatement10109 / KEYWORD_defer BlockExprStatement
10109 / KEYWORD_errdefer BlockExprStatement10110 / KEYWORD_errdefer BlockExprStatement
...@@ -10160,6 +10161,7 @@ PrimaryExpr...@@ -10160,6 +10161,7 @@ PrimaryExpr
10160 / IfExpr10161 / IfExpr
10161 / KEYWORD_break BreakLabel? Expr?10162 / KEYWORD_break BreakLabel? Expr?
10162 / KEYWORD_comptime Expr10163 / KEYWORD_comptime Expr
10164 / KEYWORD_nosuspend Expr
10163 / KEYWORD_continue BreakLabel?10165 / KEYWORD_continue BreakLabel?
10164 / KEYWORD_resume Expr10166 / KEYWORD_resume Expr
10165 / KEYWORD_return Expr?10167 / KEYWORD_return Expr?
...@@ -10522,6 +10524,7 @@ KEYWORD_for &lt;- 'for' end_of_word...@@ -10522,6 +10524,7 @@ KEYWORD_for &lt;- 'for' end_of_word
10522KEYWORD_if &lt;- 'if' end_of_word10524KEYWORD_if &lt;- 'if' end_of_word
10523KEYWORD_inline &lt;- 'inline' end_of_word10525KEYWORD_inline &lt;- 'inline' end_of_word
10524KEYWORD_noalias &lt;- 'noalias' end_of_word10526KEYWORD_noalias &lt;- 'noalias' end_of_word
10527KEYWORD_nosuspend &lt;- 'nosuspend' end_of_word
10525KEYWORD_null &lt;- 'null' end_of_word10528KEYWORD_null &lt;- 'null' end_of_word
10526KEYWORD_or &lt;- 'or' end_of_word10529KEYWORD_or &lt;- 'or' end_of_word
10527KEYWORD_orelse &lt;- 'orelse' end_of_word10530KEYWORD_orelse &lt;- 'orelse' end_of_word
lib/std/zig/ast.zig+11-3
...@@ -129,6 +129,7 @@ pub const Error = union(enum) {...@@ -129,6 +129,7 @@ pub const Error = union(enum) {
129 ExpectedStatement: ExpectedStatement,129 ExpectedStatement: ExpectedStatement,
130 ExpectedVarDeclOrFn: ExpectedVarDeclOrFn,130 ExpectedVarDeclOrFn: ExpectedVarDeclOrFn,
131 ExpectedVarDecl: ExpectedVarDecl,131 ExpectedVarDecl: ExpectedVarDecl,
132 ExpectedFn: ExpectedFn,
132 ExpectedReturnType: ExpectedReturnType,133 ExpectedReturnType: ExpectedReturnType,
133 ExpectedAggregateKw: ExpectedAggregateKw,134 ExpectedAggregateKw: ExpectedAggregateKw,
134 UnattachedDocComment: UnattachedDocComment,135 UnattachedDocComment: UnattachedDocComment,
...@@ -165,6 +166,7 @@ pub const Error = union(enum) {...@@ -165,6 +166,7 @@ pub const Error = union(enum) {
165 ExpectedDerefOrUnwrap: ExpectedDerefOrUnwrap,166 ExpectedDerefOrUnwrap: ExpectedDerefOrUnwrap,
166 ExpectedSuffixOp: ExpectedSuffixOp,167 ExpectedSuffixOp: ExpectedSuffixOp,
167 DeclBetweenFields: DeclBetweenFields,168 DeclBetweenFields: DeclBetweenFields,
169 InvalidAnd: InvalidAnd,
168170
169 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {171 pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void {
170 switch (self.*) {172 switch (self.*) {
...@@ -177,6 +179,7 @@ pub const Error = union(enum) {...@@ -177,6 +179,7 @@ pub const Error = union(enum) {
177 .ExpectedStatement => |*x| return x.render(tokens, stream),179 .ExpectedStatement => |*x| return x.render(tokens, stream),
178 .ExpectedVarDeclOrFn => |*x| return x.render(tokens, stream),180 .ExpectedVarDeclOrFn => |*x| return x.render(tokens, stream),
179 .ExpectedVarDecl => |*x| return x.render(tokens, stream),181 .ExpectedVarDecl => |*x| return x.render(tokens, stream),
182 .ExpectedFn => |*x| return x.render(tokens, stream),
180 .ExpectedReturnType => |*x| return x.render(tokens, stream),183 .ExpectedReturnType => |*x| return x.render(tokens, stream),
181 .ExpectedAggregateKw => |*x| return x.render(tokens, stream),184 .ExpectedAggregateKw => |*x| return x.render(tokens, stream),
182 .UnattachedDocComment => |*x| return x.render(tokens, stream),185 .UnattachedDocComment => |*x| return x.render(tokens, stream),
...@@ -213,6 +216,7 @@ pub const Error = union(enum) {...@@ -213,6 +216,7 @@ pub const Error = union(enum) {
213 .ExpectedDerefOrUnwrap => |*x| return x.render(tokens, stream),216 .ExpectedDerefOrUnwrap => |*x| return x.render(tokens, stream),
214 .ExpectedSuffixOp => |*x| return x.render(tokens, stream),217 .ExpectedSuffixOp => |*x| return x.render(tokens, stream),
215 .DeclBetweenFields => |*x| return x.render(tokens, stream),218 .DeclBetweenFields => |*x| return x.render(tokens, stream),
219 .InvalidAnd => |*x| return x.render(tokens, stream),
216 }220 }
217 }221 }
218222
...@@ -227,6 +231,7 @@ pub const Error = union(enum) {...@@ -227,6 +231,7 @@ pub const Error = union(enum) {
227 .ExpectedStatement => |x| return x.token,231 .ExpectedStatement => |x| return x.token,
228 .ExpectedVarDeclOrFn => |x| return x.token,232 .ExpectedVarDeclOrFn => |x| return x.token,
229 .ExpectedVarDecl => |x| return x.token,233 .ExpectedVarDecl => |x| return x.token,
234 .ExpectedFn => |x| return x.token,
230 .ExpectedReturnType => |x| return x.token,235 .ExpectedReturnType => |x| return x.token,
231 .ExpectedAggregateKw => |x| return x.token,236 .ExpectedAggregateKw => |x| return x.token,
232 .UnattachedDocComment => |x| return x.token,237 .UnattachedDocComment => |x| return x.token,
...@@ -263,6 +268,7 @@ pub const Error = union(enum) {...@@ -263,6 +268,7 @@ pub const Error = union(enum) {
263 .ExpectedDerefOrUnwrap => |x| return x.token,268 .ExpectedDerefOrUnwrap => |x| return x.token,
264 .ExpectedSuffixOp => |x| return x.token,269 .ExpectedSuffixOp => |x| return x.token,
265 .DeclBetweenFields => |x| return x.token,270 .DeclBetweenFields => |x| return x.token,
271 .InvalidAnd => |x| return x.token,
266 }272 }
267 }273 }
268274
...@@ -274,6 +280,7 @@ pub const Error = union(enum) {...@@ -274,6 +280,7 @@ pub const Error = union(enum) {
274 pub const ExpectedStatement = SingleTokenError("Expected statement, found '{}'");280 pub const ExpectedStatement = SingleTokenError("Expected statement, found '{}'");
275 pub const ExpectedVarDeclOrFn = SingleTokenError("Expected variable declaration or function, found '{}'");281 pub const ExpectedVarDeclOrFn = SingleTokenError("Expected variable declaration or function, found '{}'");
276 pub const ExpectedVarDecl = SingleTokenError("Expected variable declaration, found '{}'");282 pub const ExpectedVarDecl = SingleTokenError("Expected variable declaration, found '{}'");
283 pub const ExpectedFn = SingleTokenError("Expected function, found '{}'");
277 pub const ExpectedReturnType = SingleTokenError("Expected 'var' or return type expression, found '{}'");284 pub const ExpectedReturnType = SingleTokenError("Expected 'var' or return type expression, found '{}'");
278 pub const ExpectedAggregateKw = SingleTokenError("Expected '" ++ Token.Id.Keyword_struct.symbol() ++ "', '" ++ Token.Id.Keyword_union.symbol() ++ "', or '" ++ Token.Id.Keyword_enum.symbol() ++ "', found '{}'");285 pub const ExpectedAggregateKw = SingleTokenError("Expected '" ++ Token.Id.Keyword_struct.symbol() ++ "', '" ++ Token.Id.Keyword_union.symbol() ++ "', or '" ++ Token.Id.Keyword_enum.symbol() ++ "', found '{}'");
279 pub const ExpectedEqOrSemi = SingleTokenError("Expected '=' or ';', found '{}'");286 pub const ExpectedEqOrSemi = SingleTokenError("Expected '=' or ';', found '{}'");
...@@ -308,6 +315,7 @@ pub const Error = union(enum) {...@@ -308,6 +315,7 @@ pub const Error = union(enum) {
308 pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier");315 pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier");
309 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");316 pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier");
310 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");317 pub const DeclBetweenFields = SimpleError("Declarations are not allowed between container fields");
318 pub const InvalidAnd = SimpleError("`&&` is invalid. Note that `and` is boolean AND.");
311319
312 pub const ExpectedCall = struct {320 pub const ExpectedCall = struct {
313 node: *Node,321 node: *Node,
...@@ -335,9 +343,6 @@ pub const Error = union(enum) {...@@ -335,9 +343,6 @@ pub const Error = union(enum) {
335 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {343 pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void {
336 const found_token = tokens.at(self.token);344 const found_token = tokens.at(self.token);
337 switch (found_token.id) {345 switch (found_token.id) {
338 .Invalid_ampersands => {
339 return stream.print("`&&` is invalid. Note that `and` is boolean AND.", .{});
340 },
341 .Invalid => {346 .Invalid => {
342 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});347 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
343 },348 },
...@@ -888,6 +893,7 @@ pub const Node = struct {...@@ -888,6 +893,7 @@ pub const Node = struct {
888 pub const ReturnType = union(enum) {893 pub const ReturnType = union(enum) {
889 Explicit: *Node,894 Explicit: *Node,
890 InferErrorSet: *Node,895 InferErrorSet: *Node,
896 Invalid: TokenIndex,
891 };897 };
892898
893 pub fn iterate(self: *FnProto, index: usize) ?*Node {899 pub fn iterate(self: *FnProto, index: usize) ?*Node {
...@@ -916,6 +922,7 @@ pub const Node = struct {...@@ -916,6 +922,7 @@ pub const Node = struct {
916 if (i < 1) return node;922 if (i < 1) return node;
917 i -= 1;923 i -= 1;
918 },924 },
925 .Invalid => {},
919 }926 }
920927
921 if (self.body_node) |body_node| {928 if (self.body_node) |body_node| {
...@@ -937,6 +944,7 @@ pub const Node = struct {...@@ -937,6 +944,7 @@ pub const Node = struct {
937 if (self.body_node) |body_node| return body_node.lastToken();944 if (self.body_node) |body_node| return body_node.lastToken();
938 switch (self.return_type) {945 switch (self.return_type) {
939 .Explicit, .InferErrorSet => |node| return node.lastToken(),946 .Explicit, .InferErrorSet => |node| return node.lastToken(),
947 .Invalid => |tok| return tok,
940 }948 }
941 }949 }
942 };950 };
lib/std/zig/parse.zig+281-82
...@@ -48,31 +48,24 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {...@@ -48,31 +48,24 @@ pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree {
4848
49 while (it.peek().?.id == .LineComment) _ = it.next();49 while (it.peek().?.id == .LineComment) _ = it.next();
5050
51 tree.root_node = parseRoot(arena, &it, tree) catch |err| blk: {51 tree.root_node = try parseRoot(arena, &it, tree);
52 switch (err) {
53 error.ParseError => {
54 assert(tree.errors.len != 0);
55 break :blk undefined;
56 },
57 error.OutOfMemory => {
58 return error.OutOfMemory;
59 },
60 }
61 };
6252
63 return tree;53 return tree;
64}54}
6555
66/// Root <- skip ContainerMembers eof56/// Root <- skip ContainerMembers eof
67fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!*Node.Root {57fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root {
68 const node = try arena.create(Node.Root);58 const node = try arena.create(Node.Root);
69 node.* = .{59 node.* = .{
70 .decls = try parseContainerMembers(arena, it, tree),60 .decls = try parseContainerMembers(arena, it, tree),
71 .eof_token = eatToken(it, .Eof) orelse {61 .eof_token = eatToken(it, .Eof) orelse blk: {
62 // parseContainerMembers will try to skip as much
63 // invalid tokens as it can so this can only be a '}'
64 const tok = eatToken(it, .RBrace).?;
72 try tree.errors.push(.{65 try tree.errors.push(.{
73 .ExpectedContainerMembers = .{ .token = it.index },66 .ExpectedContainerMembers = .{ .token = tok },
74 });67 });
75 return error.ParseError;68 break :blk tok;
76 },69 },
77 };70 };
78 return node;71 return node;
...@@ -108,7 +101,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -108,7 +101,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
108101
109 const doc_comments = try parseDocComment(arena, it, tree);102 const doc_comments = try parseDocComment(arena, it, tree);
110103
111 if (try parseTestDecl(arena, it, tree)) |node| {104 if (parseTestDecl(arena, it, tree) catch |err| switch (err) {
105 error.OutOfMemory => return error.OutOfMemory,
106 error.ParseError => {
107 findNextContainerMember(it);
108 continue;
109 },
110 }) |node| {
112 if (field_state == .seen) {111 if (field_state == .seen) {
113 field_state = .{ .end = node.firstToken() };112 field_state = .{ .end = node.firstToken() };
114 }113 }
...@@ -117,7 +116,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -117,7 +116,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
117 continue;116 continue;
118 }117 }
119118
120 if (try parseTopLevelComptime(arena, it, tree)) |node| {119 if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) {
120 error.OutOfMemory => return error.OutOfMemory,
121 error.ParseError => {
122 findNextContainerMember(it);
123 continue;
124 },
125 }) |node| {
121 if (field_state == .seen) {126 if (field_state == .seen) {
122 field_state = .{ .end = node.firstToken() };127 field_state = .{ .end = node.firstToken() };
123 }128 }
...@@ -128,7 +133,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -128,7 +133,13 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
128133
129 const visib_token = eatToken(it, .Keyword_pub);134 const visib_token = eatToken(it, .Keyword_pub);
130135
131 if (try parseTopLevelDecl(arena, it, tree)) |node| {136 if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) {
137 error.OutOfMemory => return error.OutOfMemory,
138 error.ParseError => {
139 findNextContainerMember(it);
140 continue;
141 },
142 }) |node| {
132 if (field_state == .seen) {143 if (field_state == .seen) {
133 field_state = .{ .end = visib_token orelse node.firstToken() };144 field_state = .{ .end = visib_token orelse node.firstToken() };
134 }145 }
...@@ -163,10 +174,18 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -163,10 +174,18 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
163 try tree.errors.push(.{174 try tree.errors.push(.{
164 .ExpectedPubItem = .{ .token = it.index },175 .ExpectedPubItem = .{ .token = it.index },
165 });176 });
166 return error.ParseError;177 // ignore this pub
178 continue;
167 }179 }
168180
169 if (try parseContainerField(arena, it, tree)) |node| {181 if (parseContainerField(arena, it, tree) catch |err| switch (err) {
182 error.OutOfMemory => return error.OutOfMemory,
183 error.ParseError => {
184 // attempt to recover
185 findNextContainerMember(it);
186 continue;
187 },
188 }) |node| {
170 switch (field_state) {189 switch (field_state) {
171 .none => field_state = .seen,190 .none => field_state = .seen,
172 .err, .seen => {},191 .err, .seen => {},
...@@ -182,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -182,7 +201,21 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
182 const field = node.cast(Node.ContainerField).?;201 const field = node.cast(Node.ContainerField).?;
183 field.doc_comments = doc_comments;202 field.doc_comments = doc_comments;
184 try list.push(node);203 try list.push(node);
185 const comma = eatToken(it, .Comma) orelse break;204 const comma = eatToken(it, .Comma) orelse {
205 // try to continue parsing
206 const index = it.index;
207 findNextContainerMember(it);
208 switch (it.peek().?.id) {
209 .Eof, .RBrace => break,
210 else => {
211 // add error and continue
212 try tree.errors.push(.{
213 .ExpectedToken = .{ .token = index, .expected_id = .Comma },
214 });
215 continue;
216 },
217 }
218 };
186 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|219 if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment|
187 field.doc_comments = appended_comment;220 field.doc_comments = appended_comment;
188 continue;221 continue;
...@@ -194,12 +227,102 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No...@@ -194,12 +227,102 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No
194 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },227 .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() },
195 });228 });
196 }229 }
197 break;230
231 switch (it.peek().?.id) {
232 .Eof, .RBrace => break,
233 else => {
234 // this was likely not supposed to end yet,
235 // try to find the next declaration
236 const index = it.index;
237 findNextContainerMember(it);
238 try tree.errors.push(.{
239 .ExpectedContainerMembers = .{ .token = index },
240 });
241 },
242 }
198 }243 }
199244
200 return list;245 return list;
201}246}
202247
248/// Attempts to find next container member by searching for certain tokens
249fn findNextContainerMember(it: *TokenIterator) void {
250 var level: u32 = 0;
251 while (true) {
252 const tok = nextToken(it);
253 switch (tok.ptr.id) {
254 // any of these can start a new top level declaration
255 .Keyword_test,
256 .Keyword_comptime,
257 .Keyword_pub,
258 .Keyword_export,
259 .Keyword_extern,
260 .Keyword_inline,
261 .Keyword_noinline,
262 .Keyword_usingnamespace,
263 .Keyword_threadlocal,
264 .Keyword_const,
265 .Keyword_var,
266 .Keyword_fn,
267 .Identifier,
268 => {
269 if (level == 0) {
270 putBackToken(it, tok.index);
271 return;
272 }
273 },
274 .Comma, .Semicolon => {
275 // this decl was likely meant to end here
276 if (level == 0) {
277 return;
278 }
279 },
280 .LParen, .LBracket, .LBrace => level += 1,
281 .RParen, .RBracket, .RBrace => {
282 if (level == 0) {
283 // end of container, exit
284 putBackToken(it, tok.index);
285 return;
286 }
287 level -= 1;
288 },
289 .Eof => {
290 putBackToken(it, tok.index);
291 return;
292 },
293 else => {},
294 }
295 }
296}
297
298/// Attempts to find the next statement by searching for a semicolon
299fn findNextStmt(it: *TokenIterator) void {
300 var level: u32 = 0;
301 while (true) {
302 const tok = nextToken(it);
303 switch (tok.ptr.id) {
304 .LBrace => level += 1,
305 .RBrace => {
306 if (level == 0) {
307 putBackToken(it, tok.index);
308 return;
309 }
310 level -= 1;
311 },
312 .Semicolon => {
313 if (level == 0) {
314 return;
315 }
316 },
317 .Eof => {
318 putBackToken(it, tok.index);
319 return;
320 },
321 else => {},
322 }
323 }
324}
325
203/// Eat a multiline container doc comment326/// Eat a multiline container doc comment
204fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {327fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
205 var lines = Node.DocComment.LineList.init(arena);328 var lines = Node.DocComment.LineList.init(arena);
...@@ -279,22 +402,30 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -279,22 +402,30 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
279 fn_node.*.extern_export_inline_token = extern_export_inline_token;402 fn_node.*.extern_export_inline_token = extern_export_inline_token;
280 fn_node.*.lib_name = lib_name;403 fn_node.*.lib_name = lib_name;
281 if (eatToken(it, .Semicolon)) |_| return node;404 if (eatToken(it, .Semicolon)) |_| return node;
282 if (try parseBlock(arena, it, tree)) |body_node| {405 if (parseBlock(arena, it, tree) catch |err| switch (err) {
406 error.OutOfMemory => return error.OutOfMemory,
407 // since parseBlock only return error.ParseError on
408 // a missing '}' we can assume this function was
409 // supposed to end here.
410 error.ParseError => return node,
411 }) |body_node| {
283 fn_node.body_node = body_node;412 fn_node.body_node = body_node;
284 return node;413 return node;
285 }414 }
286 try tree.errors.push(.{415 try tree.errors.push(.{
287 .ExpectedSemiOrLBrace = .{ .token = it.index },416 .ExpectedSemiOrLBrace = .{ .token = it.index },
288 });417 });
289 return null;418 return error.ParseError;
290 }419 }
291420
292 if (extern_export_inline_token) |token| {421 if (extern_export_inline_token) |token| {
293 if (tree.tokens.at(token).id == .Keyword_inline or422 if (tree.tokens.at(token).id == .Keyword_inline or
294 tree.tokens.at(token).id == .Keyword_noinline)423 tree.tokens.at(token).id == .Keyword_noinline)
295 {424 {
296 putBackToken(it, token);425 try tree.errors.push(.{
297 return null;426 .ExpectedFn = .{ .token = it.index },
427 });
428 return error.ParseError;
298 }429 }
299 }430 }
300431
...@@ -313,26 +444,19 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -313,26 +444,19 @@ fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
313 try tree.errors.push(.{444 try tree.errors.push(.{
314 .ExpectedVarDecl = .{ .token = it.index },445 .ExpectedVarDecl = .{ .token = it.index },
315 });446 });
447 // ignore this and try again;
316 return error.ParseError;448 return error.ParseError;
317 }449 }
318450
319 if (extern_export_inline_token) |token| {451 if (extern_export_inline_token) |token| {
320 if (lib_name) |string_literal_node|452 try tree.errors.push(.{
321 putBackToken(it, string_literal_node.cast(Node.StringLiteral).?.token);453 .ExpectedVarDeclOrFn = .{ .token = it.index },
322 putBackToken(it, token);454 });
323 return null;455 // ignore this and try again;
456 return error.ParseError;
324 }457 }
325458
326 const use_node = (try parseUse(arena, it, tree)) orelse return null;459 return try parseUse(arena, it, tree);
327 const expr_node = try expectNode(arena, it, tree, parseExpr, .{
328 .ExpectedExpr = .{ .token = it.index },
329 });
330 const semicolon_token = try expectToken(it, tree, .Semicolon);
331 const use_node_raw = use_node.cast(Node.Use).?;
332 use_node_raw.*.expr = expr_node;
333 use_node_raw.*.semicolon_token = semicolon_token;
334
335 return use_node;
336}460}
337461
338/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)462/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr)
...@@ -366,18 +490,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -366,18 +490,23 @@ fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
366 const exclamation_token = eatToken(it, .Bang);490 const exclamation_token = eatToken(it, .Bang);
367491
368 const return_type_expr = (try parseVarType(arena, it, tree)) orelse492 const return_type_expr = (try parseVarType(arena, it, tree)) orelse
369 try expectNode(arena, it, tree, parseTypeExpr, .{493 (try parseTypeExpr(arena, it, tree)) orelse blk: {
370 .ExpectedReturnType = .{ .token = it.index },494 try tree.errors.push(.{
371 });495 .ExpectedReturnType = .{ .token = it.index },
496 });
497 // most likely the user forgot to specify the return type.
498 // Mark return type as invalid and try to continue.
499 break :blk null;
500 };
372501
373 const return_type: Node.FnProto.ReturnType = if (exclamation_token != null)502 // TODO https://github.com/ziglang/zig/issues/3750
374 .{503 const R = Node.FnProto.ReturnType;
375 .InferErrorSet = return_type_expr,504 const return_type = if (return_type_expr == null)
376 }505 R{ .Invalid = rparen }
506 else if (exclamation_token != null)
507 R{ .InferErrorSet = return_type_expr.? }
377 else508 else
378 .{509 R{ .Explicit = return_type_expr.? };
379 .Explicit = return_type_expr,
380 };
381510
382 const var_args_token = if (params.len > 0)511 const var_args_token = if (params.len > 0)
383 params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token512 params.at(params.len - 1).*.cast(Node.ParamDecl).?.var_args_token
...@@ -578,7 +707,12 @@ fn parseStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*No...@@ -578,7 +707,12 @@ fn parseStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*No
578 if (try parseLabeledStatement(arena, it, tree)) |node| return node;707 if (try parseLabeledStatement(arena, it, tree)) |node| return node;
579 if (try parseSwitchExpr(arena, it, tree)) |node| return node;708 if (try parseSwitchExpr(arena, it, tree)) |node| return node;
580 if (try parseAssignExpr(arena, it, tree)) |node| {709 if (try parseAssignExpr(arena, it, tree)) |node| {
581 _ = try expectToken(it, tree, .Semicolon);710 _ = eatToken(it, .Semicolon) orelse {
711 try tree.errors.push(.{
712 .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
713 });
714 // pretend we saw a semicolon and continue parsing
715 };
582 return node;716 return node;
583 }717 }
584718
...@@ -687,8 +821,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod...@@ -687,8 +821,13 @@ fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Nod
687 node.cast(Node.While).?.inline_token = inline_token;821 node.cast(Node.While).?.inline_token = inline_token;
688 return node;822 return node;
689 }823 }
824 if (inline_token == null) return null;
690825
691 return null;826 // If we've seen "inline", there should have been a "for" or "while"
827 try tree.errors.push(.{
828 .ExpectedInlinable = .{ .token = it.index },
829 });
830 return error.ParseError;
692}831}
693832
694/// ForStatement833/// ForStatement
...@@ -817,7 +956,12 @@ fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No...@@ -817,7 +956,12 @@ fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No
817fn parseBlockExprStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {956fn parseBlockExprStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
818 if (try parseBlockExpr(arena, it, tree)) |node| return node;957 if (try parseBlockExpr(arena, it, tree)) |node| return node;
819 if (try parseAssignExpr(arena, it, tree)) |node| {958 if (try parseAssignExpr(arena, it, tree)) |node| {
820 _ = try expectToken(it, tree, .Semicolon);959 _ = eatToken(it, .Semicolon) orelse {
960 try tree.errors.push(.{
961 .ExpectedToken = .{ .token = it.index, .expected_id = .Semicolon },
962 });
963 // pretend we saw a semicolon and continue parsing
964 };
821 return node;965 return node;
822 }966 }
823 return null;967 return null;
...@@ -924,7 +1068,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -924,7 +1068,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
924 const node = try arena.create(Node.ControlFlowExpression);1068 const node = try arena.create(Node.ControlFlowExpression);
925 node.* = .{1069 node.* = .{
926 .ltoken = token,1070 .ltoken = token,
927 .kind = Node.ControlFlowExpression.Kind{ .Break = label },1071 .kind = .{ .Break = label },
928 .rhs = expr_node,1072 .rhs = expr_node,
929 };1073 };
930 return &node.base;1074 return &node.base;
...@@ -960,7 +1104,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -960,7 +1104,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
960 const node = try arena.create(Node.ControlFlowExpression);1104 const node = try arena.create(Node.ControlFlowExpression);
961 node.* = .{1105 node.* = .{
962 .ltoken = token,1106 .ltoken = token,
963 .kind = Node.ControlFlowExpression.Kind{ .Continue = label },1107 .kind = .{ .Continue = label },
964 .rhs = null,1108 .rhs = null,
965 };1109 };
966 return &node.base;1110 return &node.base;
...@@ -984,7 +1128,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -984,7 +1128,7 @@ fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
984 const node = try arena.create(Node.ControlFlowExpression);1128 const node = try arena.create(Node.ControlFlowExpression);
985 node.* = .{1129 node.* = .{
986 .ltoken = token,1130 .ltoken = token,
987 .kind = Node.ControlFlowExpression.Kind.Return,1131 .kind = .Return,
988 .rhs = expr_node,1132 .rhs = expr_node,
989 };1133 };
990 return &node.base;1134 return &node.base;
...@@ -1022,7 +1166,14 @@ fn parseBlock(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -1022,7 +1166,14 @@ fn parseBlock(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
10221166
1023 var statements = Node.Block.StatementList.init(arena);1167 var statements = Node.Block.StatementList.init(arena);
1024 while (true) {1168 while (true) {
1025 const statement = (try parseStatement(arena, it, tree)) orelse break;1169 const statement = (parseStatement(arena, it, tree) catch |err| switch (err) {
1170 error.OutOfMemory => return error.OutOfMemory,
1171 error.ParseError => {
1172 // try to skip to the next statement
1173 findNextStmt(it);
1174 continue;
1175 },
1176 }) orelse break;
1026 try statements.push(statement);1177 try statements.push(statement);
1027 }1178 }
10281179
...@@ -1222,7 +1373,8 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -1222,7 +1373,8 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
1222 try tree.errors.push(.{1373 try tree.errors.push(.{
1223 .ExpectedParamList = .{ .token = it.index },1374 .ExpectedParamList = .{ .token = it.index },
1224 });1375 });
1225 return null;1376 // ignore this, continue parsing
1377 return res;
1226 };1378 };
1227 const node = try arena.create(Node.SuffixOp);1379 const node = try arena.create(Node.SuffixOp);
1228 node.* = .{1380 node.* = .{
...@@ -1287,7 +1439,6 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -1287,7 +1439,6 @@ fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
1287/// / IfTypeExpr1439/// / IfTypeExpr
1288/// / INTEGER1440/// / INTEGER
1289/// / KEYWORD_comptime TypeExpr1441/// / KEYWORD_comptime TypeExpr
1290/// / KEYWORD_nosuspend TypeExpr
1291/// / KEYWORD_error DOT IDENTIFIER1442/// / KEYWORD_error DOT IDENTIFIER
1292/// / KEYWORD_false1443/// / KEYWORD_false
1293/// / KEYWORD_null1444/// / KEYWORD_null
...@@ -1326,15 +1477,6 @@ fn parsePrimaryTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*N...@@ -1326,15 +1477,6 @@ fn parsePrimaryTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*N
1326 };1477 };
1327 return &node.base;1478 return &node.base;
1328 }1479 }
1329 if (eatToken(it, .Keyword_nosuspend)) |token| {
1330 const expr = (try parseTypeExpr(arena, it, tree)) orelse return null;
1331 const node = try arena.create(Node.Nosuspend);
1332 node.* = .{
1333 .nosuspend_token = token,
1334 .expr = expr,
1335 };
1336 return &node.base;
1337 }
1338 if (eatToken(it, .Keyword_error)) |token| {1480 if (eatToken(it, .Keyword_error)) |token| {
1339 const period = try expectToken(it, tree, .Period);1481 const period = try expectToken(it, tree, .Period);
1340 const identifier = try expectNode(arena, it, tree, parseIdentifier, .{1482 const identifier = try expectNode(arena, it, tree, parseIdentifier, .{
...@@ -2271,7 +2413,7 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2271,7 +2413,7 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2271 const node = try arena.create(Node.AnyFrameType);2413 const node = try arena.create(Node.AnyFrameType);
2272 node.* = .{2414 node.* = .{
2273 .anyframe_token = token,2415 .anyframe_token = token,
2274 .result = Node.AnyFrameType.Result{2416 .result = .{
2275 .arrow_token = arrow,2417 .arrow_token = arrow,
2276 .return_type = undefined, // set by caller2418 .return_type = undefined, // set by caller
2277 },2419 },
...@@ -2312,6 +2454,13 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2312,6 +2454,13 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2312 } else null;2454 } else null;
2313 _ = try expectToken(it, tree, .RParen);2455 _ = try expectToken(it, tree, .RParen);
23142456
2457 if (ptr_info.align_info != null) {
2458 try tree.errors.push(.{
2459 .ExtraAlignQualifier = .{ .token = it.index - 1 },
2460 });
2461 continue;
2462 }
2463
2315 ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{2464 ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{
2316 .node = expr_node,2465 .node = expr_node,
2317 .bit_range = bit_range,2466 .bit_range = bit_range,
...@@ -2320,14 +2469,32 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2320,14 +2469,32 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2320 continue;2469 continue;
2321 }2470 }
2322 if (eatToken(it, .Keyword_const)) |const_token| {2471 if (eatToken(it, .Keyword_const)) |const_token| {
2472 if (ptr_info.const_token != null) {
2473 try tree.errors.push(.{
2474 .ExtraConstQualifier = .{ .token = it.index - 1 },
2475 });
2476 continue;
2477 }
2323 ptr_info.const_token = const_token;2478 ptr_info.const_token = const_token;
2324 continue;2479 continue;
2325 }2480 }
2326 if (eatToken(it, .Keyword_volatile)) |volatile_token| {2481 if (eatToken(it, .Keyword_volatile)) |volatile_token| {
2482 if (ptr_info.volatile_token != null) {
2483 try tree.errors.push(.{
2484 .ExtraVolatileQualifier = .{ .token = it.index - 1 },
2485 });
2486 continue;
2487 }
2327 ptr_info.volatile_token = volatile_token;2488 ptr_info.volatile_token = volatile_token;
2328 continue;2489 continue;
2329 }2490 }
2330 if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {2491 if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
2492 if (ptr_info.allowzero_token != null) {
2493 try tree.errors.push(.{
2494 .ExtraAllowZeroQualifier = .{ .token = it.index - 1 },
2495 });
2496 continue;
2497 }
2331 ptr_info.allowzero_token = allowzero_token;2498 ptr_info.allowzero_token = allowzero_token;
2332 continue;2499 continue;
2333 }2500 }
...@@ -2346,9 +2513,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2346,9 +2513,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2346 if (try parseByteAlign(arena, it, tree)) |align_expr| {2513 if (try parseByteAlign(arena, it, tree)) |align_expr| {
2347 if (slice_type.align_info != null) {2514 if (slice_type.align_info != null) {
2348 try tree.errors.push(.{2515 try tree.errors.push(.{
2349 .ExtraAlignQualifier = .{ .token = it.index },2516 .ExtraAlignQualifier = .{ .token = it.index - 1 },
2350 });2517 });
2351 return error.ParseError;2518 continue;
2352 }2519 }
2353 slice_type.align_info = Node.PrefixOp.PtrInfo.Align{2520 slice_type.align_info = Node.PrefixOp.PtrInfo.Align{
2354 .node = align_expr,2521 .node = align_expr,
...@@ -2359,9 +2526,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2359,9 +2526,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2359 if (eatToken(it, .Keyword_const)) |const_token| {2526 if (eatToken(it, .Keyword_const)) |const_token| {
2360 if (slice_type.const_token != null) {2527 if (slice_type.const_token != null) {
2361 try tree.errors.push(.{2528 try tree.errors.push(.{
2362 .ExtraConstQualifier = .{ .token = it.index },2529 .ExtraConstQualifier = .{ .token = it.index - 1 },
2363 });2530 });
2364 return error.ParseError;2531 continue;
2365 }2532 }
2366 slice_type.const_token = const_token;2533 slice_type.const_token = const_token;
2367 continue;2534 continue;
...@@ -2369,9 +2536,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2369,9 +2536,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2369 if (eatToken(it, .Keyword_volatile)) |volatile_token| {2536 if (eatToken(it, .Keyword_volatile)) |volatile_token| {
2370 if (slice_type.volatile_token != null) {2537 if (slice_type.volatile_token != null) {
2371 try tree.errors.push(.{2538 try tree.errors.push(.{
2372 .ExtraVolatileQualifier = .{ .token = it.index },2539 .ExtraVolatileQualifier = .{ .token = it.index - 1 },
2373 });2540 });
2374 return error.ParseError;2541 continue;
2375 }2542 }
2376 slice_type.volatile_token = volatile_token;2543 slice_type.volatile_token = volatile_token;
2377 continue;2544 continue;
...@@ -2379,9 +2546,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2379,9 +2546,9 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2379 if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {2546 if (eatToken(it, .Keyword_allowzero)) |allowzero_token| {
2380 if (slice_type.allowzero_token != null) {2547 if (slice_type.allowzero_token != null) {
2381 try tree.errors.push(.{2548 try tree.errors.push(.{
2382 .ExtraAllowZeroQualifier = .{ .token = it.index },2549 .ExtraAllowZeroQualifier = .{ .token = it.index - 1 },
2383 });2550 });
2384 return error.ParseError;2551 continue;
2385 }2552 }
2386 slice_type.allowzero_token = allowzero_token;2553 slice_type.allowzero_token = allowzero_token;
2387 continue;2554 continue;
...@@ -2730,7 +2897,19 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {...@@ -2730,7 +2897,19 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
2730 var list = L.init(arena);2897 var list = L.init(arena);
2731 while (try nodeParseFn(arena, it, tree)) |node| {2898 while (try nodeParseFn(arena, it, tree)) |node| {
2732 try list.push(node);2899 try list.push(node);
2733 if (eatToken(it, .Comma) == null) break;2900
2901 switch (it.peek().?.id) {
2902 .Comma => _ = nextToken(it),
2903 // all possible delimiters
2904 .Colon, .RParen, .RBrace, .RBracket => break,
2905 else => {
2906 // this is likely just a missing comma,
2907 // continue parsing this list and give an error
2908 try tree.errors.push(.{
2909 .ExpectedToken = .{ .token = it.index, .expected_id = .Comma },
2910 });
2911 },
2912 }
2734 }2913 }
2735 return list;2914 return list;
2736 }2915 }
...@@ -2740,7 +2919,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {...@@ -2740,7 +2919,17 @@ fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) {
2740fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {2919fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn {
2741 return struct {2920 return struct {
2742 pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node {2921 pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node {
2743 const op_token = eatToken(it, token) orelse return null;2922 const op_token = if (token == .Keyword_and) switch (it.peek().?.id) {
2923 .Keyword_and => nextToken(it).index,
2924 .Invalid_ampersands => blk: {
2925 try tree.errors.push(.{
2926 .InvalidAnd = .{ .token = it.index },
2927 });
2928 break :blk nextToken(it).index;
2929 },
2930 else => return null,
2931 } else eatToken(it, token) orelse return null;
2932
2744 const node = try arena.create(Node.InfixOp);2933 const node = try arena.create(Node.InfixOp);
2745 node.* = .{2934 node.* = .{
2746 .op_token = op_token,2935 .op_token = op_token,
...@@ -2761,7 +2950,13 @@ fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node...@@ -2761,7 +2950,13 @@ fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node
2761 try tree.errors.push(.{2950 try tree.errors.push(.{
2762 .ExpectedParamList = .{ .token = it.index },2951 .ExpectedParamList = .{ .token = it.index },
2763 });2952 });
2764 return error.ParseError;2953
2954 // lets pretend this was an identifier so we can continue parsing
2955 const node = try arena.create(Node.Identifier);
2956 node.* = .{
2957 .token = token,
2958 };
2959 return &node.base;
2765 };2960 };
2766 const node = try arena.create(Node.BuiltinCall);2961 const node = try arena.create(Node.BuiltinCall);
2767 node.* = .{2962 node.* = .{
...@@ -2877,8 +3072,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {...@@ -2877,8 +3072,10 @@ fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node {
2877 .doc_comments = null,3072 .doc_comments = null,
2878 .visib_token = null,3073 .visib_token = null,
2879 .use_token = token,3074 .use_token = token,
2880 .expr = undefined, // set by caller3075 .expr = try expectNode(arena, it, tree, parseExpr, .{
2881 .semicolon_token = undefined, // set by caller3076 .ExpectedExpr = .{ .token = it.index },
3077 }),
3078 .semicolon_token = try expectToken(it, tree, .Semicolon),
2882 };3079 };
2883 return &node.base;3080 return &node.base;
2884}3081}
...@@ -3058,6 +3255,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {...@@ -3058,6 +3255,8 @@ fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex {
3058 try tree.errors.push(.{3255 try tree.errors.push(.{
3059 .ExpectedToken = .{ .token = token.index, .expected_id = id },3256 .ExpectedToken = .{ .token = token.index, .expected_id = id },
3060 });3257 });
3258 // go back so that we can recover properly
3259 putBackToken(it, token.index);
3061 return error.ParseError;3260 return error.ParseError;
3062 }3261 }
3063 return token.index;3262 return token.index;
lib/std/zig/parser_test.zig+167-4
...@@ -1,3 +1,153 @@...@@ -1,3 +1,153 @@
1test "recovery: top level" {
2 try testError(
3 \\test "" {inline}
4 \\test "" {inline}
5 , &[_]Error{
6 .ExpectedInlinable,
7 .ExpectedInlinable,
8 });
9}
10
11test "recovery: block statements" {
12 try testError(
13 \\test "" {
14 \\ foo + +;
15 \\ inline;
16 \\}
17 , &[_]Error{
18 .InvalidToken,
19 .ExpectedInlinable,
20 });
21}
22
23test "recovery: missing comma" {
24 try testError(
25 \\test "" {
26 \\ switch (foo) {
27 \\ 2 => {}
28 \\ 3 => {}
29 \\ else => {
30 \\ foo && bar +;
31 \\ }
32 \\ }
33 \\}
34 , &[_]Error{
35 .ExpectedToken,
36 .ExpectedToken,
37 .InvalidAnd,
38 .InvalidToken,
39 });
40}
41
42test "recovery: extra qualifier" {
43 try testError(
44 \\const a: *const const u8;
45 \\test ""
46 , &[_]Error{
47 .ExtraConstQualifier,
48 .ExpectedLBrace,
49 });
50}
51
52test "recovery: missing return type" {
53 try testError(
54 \\fn foo() {
55 \\ a && b;
56 \\}
57 \\test ""
58 , &[_]Error{
59 .ExpectedReturnType,
60 .InvalidAnd,
61 .ExpectedLBrace,
62 });
63}
64
65test "recovery: continue after invalid decl" {
66 try testError(
67 \\fn foo {
68 \\ inline;
69 \\}
70 \\pub test "" {
71 \\ async a && b;
72 \\}
73 , &[_]Error{
74 .ExpectedToken,
75 .ExpectedPubItem,
76 .ExpectedParamList,
77 .InvalidAnd,
78 });
79 try testError(
80 \\threadlocal test "" {
81 \\ @a && b;
82 \\}
83 , &[_]Error{
84 .ExpectedVarDecl,
85 .ExpectedParamList,
86 .InvalidAnd,
87 });
88}
89
90test "recovery: invalid extern/inline" {
91 try testError(
92 \\inline test "" { a && b; }
93 , &[_]Error{
94 .ExpectedFn,
95 .InvalidAnd,
96 });
97 try testError(
98 \\extern "" test "" { a && b; }
99 , &[_]Error{
100 .ExpectedVarDeclOrFn,
101 .InvalidAnd,
102 });
103}
104
105test "recovery: missing semicolon" {
106 try testError(
107 \\test "" {
108 \\ comptime a && b
109 \\ c && d
110 \\ @foo
111 \\}
112 , &[_]Error{
113 .InvalidAnd,
114 .ExpectedToken,
115 .InvalidAnd,
116 .ExpectedToken,
117 .ExpectedParamList,
118 .ExpectedToken,
119 });
120}
121
122test "recovery: invalid container members" {
123 try testError(
124 \\usingnamespace;
125 \\foo+
126 \\bar@,
127 \\while (a == 2) { test "" {}}
128 \\test "" {
129 \\ a && b
130 \\}
131 , &[_]Error{
132 .ExpectedExpr,
133 .ExpectedToken,
134 .ExpectedToken,
135 .ExpectedContainerMembers,
136 .InvalidAnd,
137 .ExpectedToken,
138 });
139}
140
141test "recovery: invalid parameter" {
142 try testError(
143 \\fn main() void {
144 \\ a(comptime T: type)
145 \\}
146 , &[_]Error{
147 .ExpectedToken,
148 });
149}
150
1test "zig fmt: top-level fields" {151test "zig fmt: top-level fields" {
2 try testCanonical(152 try testCanonical(
3 \\a: did_you_know,153 \\a: did_you_know,
...@@ -19,7 +169,9 @@ test "zig fmt: decl between fields" {...@@ -19,7 +169,9 @@ test "zig fmt: decl between fields" {
19 \\ const baz1 = 2;169 \\ const baz1 = 2;
20 \\ b: usize,170 \\ b: usize,
21 \\};171 \\};
22 );172 , &[_]Error{
173 .DeclBetweenFields,
174 });
23}175}
24176
25test "zig fmt: errdefer with payload" {177test "zig fmt: errdefer with payload" {
...@@ -2828,7 +2980,10 @@ test "zig fmt: extern without container keyword returns error" {...@@ -2828,7 +2980,10 @@ test "zig fmt: extern without container keyword returns error" {
2828 try testError(2980 try testError(
2829 \\const container = extern {};2981 \\const container = extern {};
2830 \\2982 \\
2831 );2983 , &[_]Error{
2984 .ExpectedExpr,
2985 .ExpectedVarDeclOrFn,
2986 });
2832}2987}
28332988
2834test "zig fmt: integer literals with underscore separators" {2989test "zig fmt: integer literals with underscore separators" {
...@@ -3030,9 +3185,17 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {...@@ -3030,9 +3185,17 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
3030fn testCanonical(source: []const u8) !void {3185fn testCanonical(source: []const u8) !void {
3031 return testTransform(source, source);3186 return testTransform(source, source);
3032}3187}
3033fn testError(source: []const u8) !void {3188
3189const Error = @TagType(std.zig.ast.Error);
3190
3191fn testError(source: []const u8, expected_errors: []const Error) !void {
3034 const tree = try std.zig.parse(std.testing.allocator, source);3192 const tree = try std.zig.parse(std.testing.allocator, source);
3035 defer tree.deinit();3193 defer tree.deinit();
30363194
3037 std.testing.expect(tree.errors.len != 0);3195 std.testing.expect(tree.errors.len == expected_errors.len);
3196 for (expected_errors) |expected, i| {
3197 const err = tree.errors.at(i);
3198
3199 std.testing.expect(expected == err.*);
3200 }
3038}3201}
lib/std/zig/render.zig+7-2
...@@ -13,6 +13,9 @@ pub const Error = error{...@@ -13,6 +13,9 @@ pub const Error = error{
1313
14/// Returns whether anything changed14/// Returns whether anything changed
15pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Error || Error)!bool {15pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Error || Error)!bool {
16 // cannot render an invalid tree
17 std.debug.assert(tree.errors.len == 0);
18
16 // make a passthrough stream that checks whether something changed19 // make a passthrough stream that checks whether something changed
17 const MyStream = struct {20 const MyStream = struct {
18 const MyStream = @This();21 const MyStream = @This();
...@@ -1444,6 +1447,7 @@ fn renderExpression(...@@ -1444,6 +1447,7 @@ fn renderExpression(
1444 else switch (fn_proto.return_type) {1447 else switch (fn_proto.return_type) {
1445 .Explicit => |node| node.firstToken(),1448 .Explicit => |node| node.firstToken(),
1446 .InferErrorSet => |node| tree.prevToken(node.firstToken()),1449 .InferErrorSet => |node| tree.prevToken(node.firstToken()),
1450 .Invalid => unreachable,
1447 });1451 });
1448 assert(tree.tokens.at(rparen).id == .RParen);1452 assert(tree.tokens.at(rparen).id == .RParen);
14491453
...@@ -1518,13 +1522,14 @@ fn renderExpression(...@@ -1518,13 +1522,14 @@ fn renderExpression(
1518 }1522 }
15191523
1520 switch (fn_proto.return_type) {1524 switch (fn_proto.return_type) {
1521 ast.Node.FnProto.ReturnType.Explicit => |node| {1525 .Explicit => |node| {
1522 return renderExpression(allocator, stream, tree, indent, start_col, node, space);1526 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
1523 },1527 },
1524 ast.Node.FnProto.ReturnType.InferErrorSet => |node| {1528 .InferErrorSet => |node| {
1525 try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !1529 try renderToken(tree, stream, tree.prevToken(node.firstToken()), indent, start_col, Space.None); // !
1526 return renderExpression(allocator, stream, tree, indent, start_col, node, space);1530 return renderExpression(allocator, stream, tree, indent, start_col, node, space);
1527 },1531 },
1532 .Invalid => unreachable,
1528 }1533 }
1529 },1534 },
15301535
lib/std/zig/tokenizer.zig+477-477
...@@ -353,64 +353,64 @@ pub const Tokenizer = struct {...@@ -353,64 +353,64 @@ pub const Tokenizer = struct {
353 }353 }
354354
355 const State = enum {355 const State = enum {
356 Start,356 start,
357 Identifier,357 identifier,
358 Builtin,358 builtin,
359 StringLiteral,359 string_literal,
360 StringLiteralBackslash,360 string_literal_backslash,
361 MultilineStringLiteralLine,361 multiline_string_literal_line,
362 CharLiteral,362 char_literal,
363 CharLiteralBackslash,363 char_literal_backslash,
364 CharLiteralHexEscape,364 char_literal_hex_escape,
365 CharLiteralUnicodeEscapeSawU,365 char_literal_unicode_escape_saw_u,
366 CharLiteralUnicodeEscape,366 char_literal_unicode_escape,
367 CharLiteralUnicodeInvalid,367 char_literal_unicode_invalid,
368 CharLiteralUnicode,368 char_literal_unicode,
369 CharLiteralEnd,369 char_literal_end,
370 Backslash,370 backslash,
371 Equal,371 equal,
372 Bang,372 bang,
373 Pipe,373 pipe,
374 Minus,374 minus,
375 MinusPercent,375 minus_percent,
376 Asterisk,376 asterisk,
377 AsteriskPercent,377 asterisk_percent,
378 Slash,378 slash,
379 LineCommentStart,379 line_comment_start,
380 LineComment,380 line_comment,
381 DocCommentStart,381 doc_comment_start,
382 DocComment,382 doc_comment,
383 ContainerDocComment,383 container_doc_comment,
384 Zero,384 zero,
385 IntegerLiteralDec,385 int_literal_dec,
386 IntegerLiteralDecNoUnderscore,386 int_literal_dec_no_underscore,
387 IntegerLiteralBin,387 int_literal_bin,
388 IntegerLiteralBinNoUnderscore,388 int_literal_bin_no_underscore,
389 IntegerLiteralOct,389 int_literal_oct,
390 IntegerLiteralOctNoUnderscore,390 int_literal_oct_no_underscore,
391 IntegerLiteralHex,391 int_literal_hex,
392 IntegerLiteralHexNoUnderscore,392 int_literal_hex_no_underscore,
393 NumberDotDec,393 num_dot_dec,
394 NumberDotHex,394 num_dot_hex,
395 FloatFractionDec,395 float_fraction_dec,
396 FloatFractionDecNoUnderscore,396 float_fraction_dec_no_underscore,
397 FloatFractionHex,397 float_fraction_hex,
398 FloatFractionHexNoUnderscore,398 float_fraction_hex_no_underscore,
399 FloatExponentUnsigned,399 float_exponent_unsigned,
400 FloatExponentNumber,400 float_exponent_num,
401 FloatExponentNumberNoUnderscore,401 float_exponent_num_no_underscore,
402 Ampersand,402 ampersand,
403 Caret,403 caret,
404 Percent,404 percent,
405 Plus,405 plus,
406 PlusPercent,406 plus_percent,
407 AngleBracketLeft,407 angle_bracket_left,
408 AngleBracketAngleBracketLeft,408 angle_bracket_angle_bracket_left,
409 AngleBracketRight,409 angle_bracket_right,
410 AngleBracketAngleBracketRight,410 angle_bracket_angle_bracket_right,
411 Period,411 period,
412 Period2,412 period_2,
413 SawAtSign,413 saw_at_sign,
414 };414 };
415415
416 fn isIdentifierChar(char: u8) bool {416 fn isIdentifierChar(char: u8) bool {
...@@ -423,9 +423,9 @@ pub const Tokenizer = struct {...@@ -423,9 +423,9 @@ pub const Tokenizer = struct {
423 return token;423 return token;
424 }424 }
425 const start_index = self.index;425 const start_index = self.index;
426 var state = State.Start;426 var state: State = .start;
427 var result = Token{427 var result = Token{
428 .id = Token.Id.Eof,428 .id = .Eof,
429 .start = self.index,429 .start = self.index,
430 .end = undefined,430 .end = undefined,
431 };431 };
...@@ -434,40 +434,40 @@ pub const Tokenizer = struct {...@@ -434,40 +434,40 @@ pub const Tokenizer = struct {
434 while (self.index < self.buffer.len) : (self.index += 1) {434 while (self.index < self.buffer.len) : (self.index += 1) {
435 const c = self.buffer[self.index];435 const c = self.buffer[self.index];
436 switch (state) {436 switch (state) {
437 State.Start => switch (c) {437 .start => switch (c) {
438 ' ', '\n', '\t', '\r' => {438 ' ', '\n', '\t', '\r' => {
439 result.start = self.index + 1;439 result.start = self.index + 1;
440 },440 },
441 '"' => {441 '"' => {
442 state = State.StringLiteral;442 state = .string_literal;
443 result.id = Token.Id.StringLiteral;443 result.id = .StringLiteral;
444 },444 },
445 '\'' => {445 '\'' => {
446 state = State.CharLiteral;446 state = .char_literal;
447 },447 },
448 'a'...'z', 'A'...'Z', '_' => {448 'a'...'z', 'A'...'Z', '_' => {
449 state = State.Identifier;449 state = .identifier;
450 result.id = Token.Id.Identifier;450 result.id = .Identifier;
451 },451 },
452 '@' => {452 '@' => {
453 state = State.SawAtSign;453 state = .saw_at_sign;
454 },454 },
455 '=' => {455 '=' => {
456 state = State.Equal;456 state = .equal;
457 },457 },
458 '!' => {458 '!' => {
459 state = State.Bang;459 state = .bang;
460 },460 },
461 '|' => {461 '|' => {
462 state = State.Pipe;462 state = .pipe;
463 },463 },
464 '(' => {464 '(' => {
465 result.id = Token.Id.LParen;465 result.id = .LParen;
466 self.index += 1;466 self.index += 1;
467 break;467 break;
468 },468 },
469 ')' => {469 ')' => {
470 result.id = Token.Id.RParen;470 result.id = .RParen;
471 self.index += 1;471 self.index += 1;
472 break;472 break;
473 },473 },
...@@ -477,213 +477,213 @@ pub const Tokenizer = struct {...@@ -477,213 +477,213 @@ pub const Tokenizer = struct {
477 break;477 break;
478 },478 },
479 ']' => {479 ']' => {
480 result.id = Token.Id.RBracket;480 result.id = .RBracket;
481 self.index += 1;481 self.index += 1;
482 break;482 break;
483 },483 },
484 ';' => {484 ';' => {
485 result.id = Token.Id.Semicolon;485 result.id = .Semicolon;
486 self.index += 1;486 self.index += 1;
487 break;487 break;
488 },488 },
489 ',' => {489 ',' => {
490 result.id = Token.Id.Comma;490 result.id = .Comma;
491 self.index += 1;491 self.index += 1;
492 break;492 break;
493 },493 },
494 '?' => {494 '?' => {
495 result.id = Token.Id.QuestionMark;495 result.id = .QuestionMark;
496 self.index += 1;496 self.index += 1;
497 break;497 break;
498 },498 },
499 ':' => {499 ':' => {
500 result.id = Token.Id.Colon;500 result.id = .Colon;
501 self.index += 1;501 self.index += 1;
502 break;502 break;
503 },503 },
504 '%' => {504 '%' => {
505 state = State.Percent;505 state = .percent;
506 },506 },
507 '*' => {507 '*' => {
508 state = State.Asterisk;508 state = .asterisk;
509 },509 },
510 '+' => {510 '+' => {
511 state = State.Plus;511 state = .plus;
512 },512 },
513 '<' => {513 '<' => {
514 state = State.AngleBracketLeft;514 state = .angle_bracket_left;
515 },515 },
516 '>' => {516 '>' => {
517 state = State.AngleBracketRight;517 state = .angle_bracket_right;
518 },518 },
519 '^' => {519 '^' => {
520 state = State.Caret;520 state = .caret;
521 },521 },
522 '\\' => {522 '\\' => {
523 state = State.Backslash;523 state = .backslash;
524 result.id = Token.Id.MultilineStringLiteralLine;524 result.id = .MultilineStringLiteralLine;
525 },525 },
526 '{' => {526 '{' => {
527 result.id = Token.Id.LBrace;527 result.id = .LBrace;
528 self.index += 1;528 self.index += 1;
529 break;529 break;
530 },530 },
531 '}' => {531 '}' => {
532 result.id = Token.Id.RBrace;532 result.id = .RBrace;
533 self.index += 1;533 self.index += 1;
534 break;534 break;
535 },535 },
536 '~' => {536 '~' => {
537 result.id = Token.Id.Tilde;537 result.id = .Tilde;
538 self.index += 1;538 self.index += 1;
539 break;539 break;
540 },540 },
541 '.' => {541 '.' => {
542 state = State.Period;542 state = .period;
543 },543 },
544 '-' => {544 '-' => {
545 state = State.Minus;545 state = .minus;
546 },546 },
547 '/' => {547 '/' => {
548 state = State.Slash;548 state = .slash;
549 },549 },
550 '&' => {550 '&' => {
551 state = State.Ampersand;551 state = .ampersand;
552 },552 },
553 '0' => {553 '0' => {
554 state = State.Zero;554 state = .zero;
555 result.id = Token.Id.IntegerLiteral;555 result.id = .IntegerLiteral;
556 },556 },
557 '1'...'9' => {557 '1'...'9' => {
558 state = State.IntegerLiteralDec;558 state = .int_literal_dec;
559 result.id = Token.Id.IntegerLiteral;559 result.id = .IntegerLiteral;
560 },560 },
561 else => {561 else => {
562 result.id = Token.Id.Invalid;562 result.id = .Invalid;
563 self.index += 1;563 self.index += 1;
564 break;564 break;
565 },565 },
566 },566 },
567567
568 State.SawAtSign => switch (c) {568 .saw_at_sign => switch (c) {
569 '"' => {569 '"' => {
570 result.id = Token.Id.Identifier;570 result.id = .Identifier;
571 state = State.StringLiteral;571 state = .string_literal;
572 },572 },
573 else => {573 else => {
574 // reinterpret as a builtin574 // reinterpret as a builtin
575 self.index -= 1;575 self.index -= 1;
576 state = State.Builtin;576 state = .builtin;
577 result.id = Token.Id.Builtin;577 result.id = .Builtin;
578 },578 },
579 },579 },
580580
581 State.Ampersand => switch (c) {581 .ampersand => switch (c) {
582 '&' => {582 '&' => {
583 result.id = Token.Id.Invalid_ampersands;583 result.id = .Invalid_ampersands;
584 self.index += 1;584 self.index += 1;
585 break;585 break;
586 },586 },
587 '=' => {587 '=' => {
588 result.id = Token.Id.AmpersandEqual;588 result.id = .AmpersandEqual;
589 self.index += 1;589 self.index += 1;
590 break;590 break;
591 },591 },
592 else => {592 else => {
593 result.id = Token.Id.Ampersand;593 result.id = .Ampersand;
594 break;594 break;
595 },595 },
596 },596 },
597597
598 State.Asterisk => switch (c) {598 .asterisk => switch (c) {
599 '=' => {599 '=' => {
600 result.id = Token.Id.AsteriskEqual;600 result.id = .AsteriskEqual;
601 self.index += 1;601 self.index += 1;
602 break;602 break;
603 },603 },
604 '*' => {604 '*' => {
605 result.id = Token.Id.AsteriskAsterisk;605 result.id = .AsteriskAsterisk;
606 self.index += 1;606 self.index += 1;
607 break;607 break;
608 },608 },
609 '%' => {609 '%' => {
610 state = State.AsteriskPercent;610 state = .asterisk_percent;
611 },611 },
612 else => {612 else => {
613 result.id = Token.Id.Asterisk;613 result.id = .Asterisk;
614 break;614 break;
615 },615 },
616 },616 },
617617
618 State.AsteriskPercent => switch (c) {618 .asterisk_percent => switch (c) {
619 '=' => {619 '=' => {
620 result.id = Token.Id.AsteriskPercentEqual;620 result.id = .AsteriskPercentEqual;
621 self.index += 1;621 self.index += 1;
622 break;622 break;
623 },623 },
624 else => {624 else => {
625 result.id = Token.Id.AsteriskPercent;625 result.id = .AsteriskPercent;
626 break;626 break;
627 },627 },
628 },628 },
629629
630 State.Percent => switch (c) {630 .percent => switch (c) {
631 '=' => {631 '=' => {
632 result.id = Token.Id.PercentEqual;632 result.id = .PercentEqual;
633 self.index += 1;633 self.index += 1;
634 break;634 break;
635 },635 },
636 else => {636 else => {
637 result.id = Token.Id.Percent;637 result.id = .Percent;
638 break;638 break;
639 },639 },
640 },640 },
641641
642 State.Plus => switch (c) {642 .plus => switch (c) {
643 '=' => {643 '=' => {
644 result.id = Token.Id.PlusEqual;644 result.id = .PlusEqual;
645 self.index += 1;645 self.index += 1;
646 break;646 break;
647 },647 },
648 '+' => {648 '+' => {
649 result.id = Token.Id.PlusPlus;649 result.id = .PlusPlus;
650 self.index += 1;650 self.index += 1;
651 break;651 break;
652 },652 },
653 '%' => {653 '%' => {
654 state = State.PlusPercent;654 state = .plus_percent;
655 },655 },
656 else => {656 else => {
657 result.id = Token.Id.Plus;657 result.id = .Plus;
658 break;658 break;
659 },659 },
660 },660 },
661661
662 State.PlusPercent => switch (c) {662 .plus_percent => switch (c) {
663 '=' => {663 '=' => {
664 result.id = Token.Id.PlusPercentEqual;664 result.id = .PlusPercentEqual;
665 self.index += 1;665 self.index += 1;
666 break;666 break;
667 },667 },
668 else => {668 else => {
669 result.id = Token.Id.PlusPercent;669 result.id = .PlusPercent;
670 break;670 break;
671 },671 },
672 },672 },
673673
674 State.Caret => switch (c) {674 .caret => switch (c) {
675 '=' => {675 '=' => {
676 result.id = Token.Id.CaretEqual;676 result.id = .CaretEqual;
677 self.index += 1;677 self.index += 1;
678 break;678 break;
679 },679 },
680 else => {680 else => {
681 result.id = Token.Id.Caret;681 result.id = .Caret;
682 break;682 break;
683 },683 },
684 },684 },
685685
686 State.Identifier => switch (c) {686 .identifier => switch (c) {
687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},687 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
688 else => {688 else => {
689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {689 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
...@@ -692,19 +692,19 @@ pub const Tokenizer = struct {...@@ -692,19 +692,19 @@ pub const Tokenizer = struct {
692 break;692 break;
693 },693 },
694 },694 },
695 State.Builtin => switch (c) {695 .builtin => switch (c) {
696 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},696 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
697 else => break,697 else => break,
698 },698 },
699 State.Backslash => switch (c) {699 .backslash => switch (c) {
700 '\\' => {700 '\\' => {
701 state = State.MultilineStringLiteralLine;701 state = .multiline_string_literal_line;
702 },702 },
703 else => break,703 else => break,
704 },704 },
705 State.StringLiteral => switch (c) {705 .string_literal => switch (c) {
706 '\\' => {706 '\\' => {
707 state = State.StringLiteralBackslash;707 state = .string_literal_backslash;
708 },708 },
709 '"' => {709 '"' => {
710 self.index += 1;710 self.index += 1;
...@@ -714,98 +714,98 @@ pub const Tokenizer = struct {...@@ -714,98 +714,98 @@ pub const Tokenizer = struct {
714 else => self.checkLiteralCharacter(),714 else => self.checkLiteralCharacter(),
715 },715 },
716716
717 State.StringLiteralBackslash => switch (c) {717 .string_literal_backslash => switch (c) {
718 '\n', '\r' => break, // Look for this error later.718 '\n', '\r' => break, // Look for this error later.
719 else => {719 else => {
720 state = State.StringLiteral;720 state = .string_literal;
721 },721 },
722 },722 },
723723
724 State.CharLiteral => switch (c) {724 .char_literal => switch (c) {
725 '\\' => {725 '\\' => {
726 state = State.CharLiteralBackslash;726 state = .char_literal_backslash;
727 },727 },
728 '\'', 0x80...0xbf, 0xf8...0xff => {728 '\'', 0x80...0xbf, 0xf8...0xff => {
729 result.id = Token.Id.Invalid;729 result.id = .Invalid;
730 break;730 break;
731 },731 },
732 0xc0...0xdf => { // 110xxxxx732 0xc0...0xdf => { // 110xxxxx
733 remaining_code_units = 1;733 remaining_code_units = 1;
734 state = State.CharLiteralUnicode;734 state = .char_literal_unicode;
735 },735 },
736 0xe0...0xef => { // 1110xxxx736 0xe0...0xef => { // 1110xxxx
737 remaining_code_units = 2;737 remaining_code_units = 2;
738 state = State.CharLiteralUnicode;738 state = .char_literal_unicode;
739 },739 },
740 0xf0...0xf7 => { // 11110xxx740 0xf0...0xf7 => { // 11110xxx
741 remaining_code_units = 3;741 remaining_code_units = 3;
742 state = State.CharLiteralUnicode;742 state = .char_literal_unicode;
743 },743 },
744 else => {744 else => {
745 state = State.CharLiteralEnd;745 state = .char_literal_end;
746 },746 },
747 },747 },
748748
749 State.CharLiteralBackslash => switch (c) {749 .char_literal_backslash => switch (c) {
750 '\n' => {750 '\n' => {
751 result.id = Token.Id.Invalid;751 result.id = .Invalid;
752 break;752 break;
753 },753 },
754 'x' => {754 'x' => {
755 state = State.CharLiteralHexEscape;755 state = .char_literal_hex_escape;
756 seen_escape_digits = 0;756 seen_escape_digits = 0;
757 },757 },
758 'u' => {758 'u' => {
759 state = State.CharLiteralUnicodeEscapeSawU;759 state = .char_literal_unicode_escape_saw_u;
760 },760 },
761 else => {761 else => {
762 state = State.CharLiteralEnd;762 state = .char_literal_end;
763 },763 },
764 },764 },
765765
766 State.CharLiteralHexEscape => switch (c) {766 .char_literal_hex_escape => switch (c) {
767 '0'...'9', 'a'...'f', 'A'...'F' => {767 '0'...'9', 'a'...'f', 'A'...'F' => {
768 seen_escape_digits += 1;768 seen_escape_digits += 1;
769 if (seen_escape_digits == 2) {769 if (seen_escape_digits == 2) {
770 state = State.CharLiteralEnd;770 state = .char_literal_end;
771 }771 }
772 },772 },
773 else => {773 else => {
774 result.id = Token.Id.Invalid;774 result.id = .Invalid;
775 break;775 break;
776 },776 },
777 },777 },
778778
779 State.CharLiteralUnicodeEscapeSawU => switch (c) {779 .char_literal_unicode_escape_saw_u => switch (c) {
780 '{' => {780 '{' => {
781 state = State.CharLiteralUnicodeEscape;781 state = .char_literal_unicode_escape;
782 seen_escape_digits = 0;782 seen_escape_digits = 0;
783 },783 },
784 else => {784 else => {
785 result.id = Token.Id.Invalid;785 result.id = .Invalid;
786 state = State.CharLiteralUnicodeInvalid;786 state = .char_literal_unicode_invalid;
787 },787 },
788 },788 },
789789
790 State.CharLiteralUnicodeEscape => switch (c) {790 .char_literal_unicode_escape => switch (c) {
791 '0'...'9', 'a'...'f', 'A'...'F' => {791 '0'...'9', 'a'...'f', 'A'...'F' => {
792 seen_escape_digits += 1;792 seen_escape_digits += 1;
793 },793 },
794 '}' => {794 '}' => {
795 if (seen_escape_digits == 0) {795 if (seen_escape_digits == 0) {
796 result.id = Token.Id.Invalid;796 result.id = .Invalid;
797 state = State.CharLiteralUnicodeInvalid;797 state = .char_literal_unicode_invalid;
798 } else {798 } else {
799 state = State.CharLiteralEnd;799 state = .char_literal_end;
800 }800 }
801 },801 },
802 else => {802 else => {
803 result.id = Token.Id.Invalid;803 result.id = .Invalid;
804 state = State.CharLiteralUnicodeInvalid;804 state = .char_literal_unicode_invalid;
805 },805 },
806 },806 },
807807
808 State.CharLiteralUnicodeInvalid => switch (c) {808 .char_literal_unicode_invalid => switch (c) {
809 // Keep consuming characters until an obvious stopping point.809 // Keep consuming characters until an obvious stopping point.
810 // This consolidates e.g. `u{0ab1Q}` into a single invalid token810 // This consolidates e.g. `u{0ab1Q}` into a single invalid token
811 // instead of creating the tokens `u{0ab1`, `Q`, `}`811 // instead of creating the tokens `u{0ab1`, `Q`, `}`
...@@ -813,32 +813,32 @@ pub const Tokenizer = struct {...@@ -813,32 +813,32 @@ pub const Tokenizer = struct {
813 else => break,813 else => break,
814 },814 },
815815
816 State.CharLiteralEnd => switch (c) {816 .char_literal_end => switch (c) {
817 '\'' => {817 '\'' => {
818 result.id = Token.Id.CharLiteral;818 result.id = .CharLiteral;
819 self.index += 1;819 self.index += 1;
820 break;820 break;
821 },821 },
822 else => {822 else => {
823 result.id = Token.Id.Invalid;823 result.id = .Invalid;
824 break;824 break;
825 },825 },
826 },826 },
827827
828 State.CharLiteralUnicode => switch (c) {828 .char_literal_unicode => switch (c) {
829 0x80...0xbf => {829 0x80...0xbf => {
830 remaining_code_units -= 1;830 remaining_code_units -= 1;
831 if (remaining_code_units == 0) {831 if (remaining_code_units == 0) {
832 state = State.CharLiteralEnd;832 state = .char_literal_end;
833 }833 }
834 },834 },
835 else => {835 else => {
836 result.id = Token.Id.Invalid;836 result.id = .Invalid;
837 break;837 break;
838 },838 },
839 },839 },
840840
841 State.MultilineStringLiteralLine => switch (c) {841 .multiline_string_literal_line => switch (c) {
842 '\n' => {842 '\n' => {
843 self.index += 1;843 self.index += 1;
844 break;844 break;
...@@ -847,449 +847,449 @@ pub const Tokenizer = struct {...@@ -847,449 +847,449 @@ pub const Tokenizer = struct {
847 else => self.checkLiteralCharacter(),847 else => self.checkLiteralCharacter(),
848 },848 },
849849
850 State.Bang => switch (c) {850 .bang => switch (c) {
851 '=' => {851 '=' => {
852 result.id = Token.Id.BangEqual;852 result.id = .BangEqual;
853 self.index += 1;853 self.index += 1;
854 break;854 break;
855 },855 },
856 else => {856 else => {
857 result.id = Token.Id.Bang;857 result.id = .Bang;
858 break;858 break;
859 },859 },
860 },860 },
861861
862 State.Pipe => switch (c) {862 .pipe => switch (c) {
863 '=' => {863 '=' => {
864 result.id = Token.Id.PipeEqual;864 result.id = .PipeEqual;
865 self.index += 1;865 self.index += 1;
866 break;866 break;
867 },867 },
868 '|' => {868 '|' => {
869 result.id = Token.Id.PipePipe;869 result.id = .PipePipe;
870 self.index += 1;870 self.index += 1;
871 break;871 break;
872 },872 },
873 else => {873 else => {
874 result.id = Token.Id.Pipe;874 result.id = .Pipe;
875 break;875 break;
876 },876 },
877 },877 },
878878
879 State.Equal => switch (c) {879 .equal => switch (c) {
880 '=' => {880 '=' => {
881 result.id = Token.Id.EqualEqual;881 result.id = .EqualEqual;
882 self.index += 1;882 self.index += 1;
883 break;883 break;
884 },884 },
885 '>' => {885 '>' => {
886 result.id = Token.Id.EqualAngleBracketRight;886 result.id = .EqualAngleBracketRight;
887 self.index += 1;887 self.index += 1;
888 break;888 break;
889 },889 },
890 else => {890 else => {
891 result.id = Token.Id.Equal;891 result.id = .Equal;
892 break;892 break;
893 },893 },
894 },894 },
895895
896 State.Minus => switch (c) {896 .minus => switch (c) {
897 '>' => {897 '>' => {
898 result.id = Token.Id.Arrow;898 result.id = .Arrow;
899 self.index += 1;899 self.index += 1;
900 break;900 break;
901 },901 },
902 '=' => {902 '=' => {
903 result.id = Token.Id.MinusEqual;903 result.id = .MinusEqual;
904 self.index += 1;904 self.index += 1;
905 break;905 break;
906 },906 },
907 '%' => {907 '%' => {
908 state = State.MinusPercent;908 state = .minus_percent;
909 },909 },
910 else => {910 else => {
911 result.id = Token.Id.Minus;911 result.id = .Minus;
912 break;912 break;
913 },913 },
914 },914 },
915915
916 State.MinusPercent => switch (c) {916 .minus_percent => switch (c) {
917 '=' => {917 '=' => {
918 result.id = Token.Id.MinusPercentEqual;918 result.id = .MinusPercentEqual;
919 self.index += 1;919 self.index += 1;
920 break;920 break;
921 },921 },
922 else => {922 else => {
923 result.id = Token.Id.MinusPercent;923 result.id = .MinusPercent;
924 break;924 break;
925 },925 },
926 },926 },
927927
928 State.AngleBracketLeft => switch (c) {928 .angle_bracket_left => switch (c) {
929 '<' => {929 '<' => {
930 state = State.AngleBracketAngleBracketLeft;930 state = .angle_bracket_angle_bracket_left;
931 },931 },
932 '=' => {932 '=' => {
933 result.id = Token.Id.AngleBracketLeftEqual;933 result.id = .AngleBracketLeftEqual;
934 self.index += 1;934 self.index += 1;
935 break;935 break;
936 },936 },
937 else => {937 else => {
938 result.id = Token.Id.AngleBracketLeft;938 result.id = .AngleBracketLeft;
939 break;939 break;
940 },940 },
941 },941 },
942942
943 State.AngleBracketAngleBracketLeft => switch (c) {943 .angle_bracket_angle_bracket_left => switch (c) {
944 '=' => {944 '=' => {
945 result.id = Token.Id.AngleBracketAngleBracketLeftEqual;945 result.id = .AngleBracketAngleBracketLeftEqual;
946 self.index += 1;946 self.index += 1;
947 break;947 break;
948 },948 },
949 else => {949 else => {
950 result.id = Token.Id.AngleBracketAngleBracketLeft;950 result.id = .AngleBracketAngleBracketLeft;
951 break;951 break;
952 },952 },
953 },953 },
954954
955 State.AngleBracketRight => switch (c) {955 .angle_bracket_right => switch (c) {
956 '>' => {956 '>' => {
957 state = State.AngleBracketAngleBracketRight;957 state = .angle_bracket_angle_bracket_right;
958 },958 },
959 '=' => {959 '=' => {
960 result.id = Token.Id.AngleBracketRightEqual;960 result.id = .AngleBracketRightEqual;
961 self.index += 1;961 self.index += 1;
962 break;962 break;
963 },963 },
964 else => {964 else => {
965 result.id = Token.Id.AngleBracketRight;965 result.id = .AngleBracketRight;
966 break;966 break;
967 },967 },
968 },968 },
969969
970 State.AngleBracketAngleBracketRight => switch (c) {970 .angle_bracket_angle_bracket_right => switch (c) {
971 '=' => {971 '=' => {
972 result.id = Token.Id.AngleBracketAngleBracketRightEqual;972 result.id = .AngleBracketAngleBracketRightEqual;
973 self.index += 1;973 self.index += 1;
974 break;974 break;
975 },975 },
976 else => {976 else => {
977 result.id = Token.Id.AngleBracketAngleBracketRight;977 result.id = .AngleBracketAngleBracketRight;
978 break;978 break;
979 },979 },
980 },980 },
981981
982 State.Period => switch (c) {982 .period => switch (c) {
983 '.' => {983 '.' => {
984 state = State.Period2;984 state = .period_2;
985 },985 },
986 '*' => {986 '*' => {
987 result.id = Token.Id.PeriodAsterisk;987 result.id = .PeriodAsterisk;
988 self.index += 1;988 self.index += 1;
989 break;989 break;
990 },990 },
991 else => {991 else => {
992 result.id = Token.Id.Period;992 result.id = .Period;
993 break;993 break;
994 },994 },
995 },995 },
996996
997 State.Period2 => switch (c) {997 .period_2 => switch (c) {
998 '.' => {998 '.' => {
999 result.id = Token.Id.Ellipsis3;999 result.id = .Ellipsis3;
1000 self.index += 1;1000 self.index += 1;
1001 break;1001 break;
1002 },1002 },
1003 else => {1003 else => {
1004 result.id = Token.Id.Ellipsis2;1004 result.id = .Ellipsis2;
1005 break;1005 break;
1006 },1006 },
1007 },1007 },
10081008
1009 State.Slash => switch (c) {1009 .slash => switch (c) {
1010 '/' => {1010 '/' => {
1011 state = State.LineCommentStart;1011 state = .line_comment_start;
1012 result.id = Token.Id.LineComment;1012 result.id = .LineComment;
1013 },1013 },
1014 '=' => {1014 '=' => {
1015 result.id = Token.Id.SlashEqual;1015 result.id = .SlashEqual;
1016 self.index += 1;1016 self.index += 1;
1017 break;1017 break;
1018 },1018 },
1019 else => {1019 else => {
1020 result.id = Token.Id.Slash;1020 result.id = .Slash;
1021 break;1021 break;
1022 },1022 },
1023 },1023 },
1024 State.LineCommentStart => switch (c) {1024 .line_comment_start => switch (c) {
1025 '/' => {1025 '/' => {
1026 state = State.DocCommentStart;1026 state = .doc_comment_start;
1027 },1027 },
1028 '!' => {1028 '!' => {
1029 result.id = Token.Id.ContainerDocComment;1029 result.id = .ContainerDocComment;
1030 state = State.ContainerDocComment;1030 state = .container_doc_comment;
1031 },1031 },
1032 '\n' => break,1032 '\n' => break,
1033 else => {1033 else => {
1034 state = State.LineComment;1034 state = .line_comment;
1035 self.checkLiteralCharacter();1035 self.checkLiteralCharacter();
1036 },1036 },
1037 },1037 },
1038 State.DocCommentStart => switch (c) {1038 .doc_comment_start => switch (c) {
1039 '/' => {1039 '/' => {
1040 state = State.LineComment;1040 state = .line_comment;
1041 },1041 },
1042 '\n' => {1042 '\n' => {
1043 result.id = Token.Id.DocComment;1043 result.id = .DocComment;
1044 break;1044 break;
1045 },1045 },
1046 else => {1046 else => {
1047 state = State.DocComment;1047 state = .doc_comment;
1048 result.id = Token.Id.DocComment;1048 result.id = .DocComment;
1049 self.checkLiteralCharacter();1049 self.checkLiteralCharacter();
1050 },1050 },
1051 },1051 },
1052 State.LineComment, State.DocComment, State.ContainerDocComment => switch (c) {1052 .line_comment, .doc_comment, .container_doc_comment => switch (c) {
1053 '\n' => break,1053 '\n' => break,
1054 else => self.checkLiteralCharacter(),1054 else => self.checkLiteralCharacter(),
1055 },1055 },
1056 State.Zero => switch (c) {1056 .zero => switch (c) {
1057 'b' => {1057 'b' => {
1058 state = State.IntegerLiteralBinNoUnderscore;1058 state = .int_literal_bin_no_underscore;
1059 },1059 },
1060 'o' => {1060 'o' => {
1061 state = State.IntegerLiteralOctNoUnderscore;1061 state = .int_literal_oct_no_underscore;
1062 },1062 },
1063 'x' => {1063 'x' => {
1064 state = State.IntegerLiteralHexNoUnderscore;1064 state = .int_literal_hex_no_underscore;
1065 },1065 },
1066 '0'...'9', '_', '.', 'e', 'E' => {1066 '0'...'9', '_', '.', 'e', 'E' => {
1067 // reinterpret as a decimal number1067 // reinterpret as a decimal number
1068 self.index -= 1;1068 self.index -= 1;
1069 state = State.IntegerLiteralDec;1069 state = .int_literal_dec;
1070 },1070 },
1071 else => {1071 else => {
1072 if (isIdentifierChar(c)) {1072 if (isIdentifierChar(c)) {
1073 result.id = Token.Id.Invalid;1073 result.id = .Invalid;
1074 }1074 }
1075 break;1075 break;
1076 },1076 },
1077 },1077 },
1078 State.IntegerLiteralBinNoUnderscore => switch (c) {1078 .int_literal_bin_no_underscore => switch (c) {
1079 '0'...'1' => {1079 '0'...'1' => {
1080 state = State.IntegerLiteralBin;1080 state = .int_literal_bin;
1081 },1081 },
1082 else => {1082 else => {
1083 result.id = Token.Id.Invalid;1083 result.id = .Invalid;
1084 break;1084 break;
1085 },1085 },
1086 },1086 },
1087 State.IntegerLiteralBin => switch (c) {1087 .int_literal_bin => switch (c) {
1088 '_' => {1088 '_' => {
1089 state = State.IntegerLiteralBinNoUnderscore;1089 state = .int_literal_bin_no_underscore;
1090 },1090 },
1091 '0'...'1' => {},1091 '0'...'1' => {},
1092 else => {1092 else => {
1093 if (isIdentifierChar(c)) {1093 if (isIdentifierChar(c)) {
1094 result.id = Token.Id.Invalid;1094 result.id = .Invalid;
1095 }1095 }
1096 break;1096 break;
1097 },1097 },
1098 },1098 },
1099 State.IntegerLiteralOctNoUnderscore => switch (c) {1099 .int_literal_oct_no_underscore => switch (c) {
1100 '0'...'7' => {1100 '0'...'7' => {
1101 state = State.IntegerLiteralOct;1101 state = .int_literal_oct;
1102 },1102 },
1103 else => {1103 else => {
1104 result.id = Token.Id.Invalid;1104 result.id = .Invalid;
1105 break;1105 break;
1106 },1106 },
1107 },1107 },
1108 State.IntegerLiteralOct => switch (c) {1108 .int_literal_oct => switch (c) {
1109 '_' => {1109 '_' => {
1110 state = State.IntegerLiteralOctNoUnderscore;1110 state = .int_literal_oct_no_underscore;
1111 },1111 },
1112 '0'...'7' => {},1112 '0'...'7' => {},
1113 else => {1113 else => {
1114 if (isIdentifierChar(c)) {1114 if (isIdentifierChar(c)) {
1115 result.id = Token.Id.Invalid;1115 result.id = .Invalid;
1116 }1116 }
1117 break;1117 break;
1118 },1118 },
1119 },1119 },
1120 State.IntegerLiteralDecNoUnderscore => switch (c) {1120 .int_literal_dec_no_underscore => switch (c) {
1121 '0'...'9' => {1121 '0'...'9' => {
1122 state = State.IntegerLiteralDec;1122 state = .int_literal_dec;
1123 },1123 },
1124 else => {1124 else => {
1125 result.id = Token.Id.Invalid;1125 result.id = .Invalid;
1126 break;1126 break;
1127 },1127 },
1128 },1128 },
1129 State.IntegerLiteralDec => switch (c) {1129 .int_literal_dec => switch (c) {
1130 '_' => {1130 '_' => {
1131 state = State.IntegerLiteralDecNoUnderscore;1131 state = .int_literal_dec_no_underscore;
1132 },1132 },
1133 '.' => {1133 '.' => {
1134 state = State.NumberDotDec;1134 state = .num_dot_dec;
1135 result.id = Token.Id.FloatLiteral;1135 result.id = .FloatLiteral;
1136 },1136 },
1137 'e', 'E' => {1137 'e', 'E' => {
1138 state = State.FloatExponentUnsigned;1138 state = .float_exponent_unsigned;
1139 result.id = Token.Id.FloatLiteral;1139 result.id = .FloatLiteral;
1140 },1140 },
1141 '0'...'9' => {},1141 '0'...'9' => {},
1142 else => {1142 else => {
1143 if (isIdentifierChar(c)) {1143 if (isIdentifierChar(c)) {
1144 result.id = Token.Id.Invalid;1144 result.id = .Invalid;
1145 }1145 }
1146 break;1146 break;
1147 },1147 },
1148 },1148 },
1149 State.IntegerLiteralHexNoUnderscore => switch (c) {1149 .int_literal_hex_no_underscore => switch (c) {
1150 '0'...'9', 'a'...'f', 'A'...'F' => {1150 '0'...'9', 'a'...'f', 'A'...'F' => {
1151 state = State.IntegerLiteralHex;1151 state = .int_literal_hex;
1152 },1152 },
1153 else => {1153 else => {
1154 result.id = Token.Id.Invalid;1154 result.id = .Invalid;
1155 break;1155 break;
1156 },1156 },
1157 },1157 },
1158 State.IntegerLiteralHex => switch (c) {1158 .int_literal_hex => switch (c) {
1159 '_' => {1159 '_' => {
1160 state = State.IntegerLiteralHexNoUnderscore;1160 state = .int_literal_hex_no_underscore;
1161 },1161 },
1162 '.' => {1162 '.' => {
1163 state = State.NumberDotHex;1163 state = .num_dot_hex;
1164 result.id = Token.Id.FloatLiteral;1164 result.id = .FloatLiteral;
1165 },1165 },
1166 'p', 'P' => {1166 'p', 'P' => {
1167 state = State.FloatExponentUnsigned;1167 state = .float_exponent_unsigned;
1168 result.id = Token.Id.FloatLiteral;1168 result.id = .FloatLiteral;
1169 },1169 },
1170 '0'...'9', 'a'...'f', 'A'...'F' => {},1170 '0'...'9', 'a'...'f', 'A'...'F' => {},
1171 else => {1171 else => {
1172 if (isIdentifierChar(c)) {1172 if (isIdentifierChar(c)) {
1173 result.id = Token.Id.Invalid;1173 result.id = .Invalid;
1174 }1174 }
1175 break;1175 break;
1176 },1176 },
1177 },1177 },
1178 State.NumberDotDec => switch (c) {1178 .num_dot_dec => switch (c) {
1179 '.' => {1179 '.' => {
1180 self.index -= 1;1180 self.index -= 1;
1181 state = State.Start;1181 state = .start;
1182 break;1182 break;
1183 },1183 },
1184 'e', 'E' => {1184 'e', 'E' => {
1185 state = State.FloatExponentUnsigned;1185 state = .float_exponent_unsigned;
1186 },1186 },
1187 '0'...'9' => {1187 '0'...'9' => {
1188 result.id = Token.Id.FloatLiteral;1188 result.id = .FloatLiteral;
1189 state = State.FloatFractionDec;1189 state = .float_fraction_dec;
1190 },1190 },
1191 else => {1191 else => {
1192 if (isIdentifierChar(c)) {1192 if (isIdentifierChar(c)) {
1193 result.id = Token.Id.Invalid;1193 result.id = .Invalid;
1194 }1194 }
1195 break;1195 break;
1196 },1196 },
1197 },1197 },
1198 State.NumberDotHex => switch (c) {1198 .num_dot_hex => switch (c) {
1199 '.' => {1199 '.' => {
1200 self.index -= 1;1200 self.index -= 1;
1201 state = State.Start;1201 state = .start;
1202 break;1202 break;
1203 },1203 },
1204 'p', 'P' => {1204 'p', 'P' => {
1205 state = State.FloatExponentUnsigned;1205 state = .float_exponent_unsigned;
1206 },1206 },
1207 '0'...'9', 'a'...'f', 'A'...'F' => {1207 '0'...'9', 'a'...'f', 'A'...'F' => {
1208 result.id = Token.Id.FloatLiteral;1208 result.id = .FloatLiteral;
1209 state = State.FloatFractionHex;1209 state = .float_fraction_hex;
1210 },1210 },
1211 else => {1211 else => {
1212 if (isIdentifierChar(c)) {1212 if (isIdentifierChar(c)) {
1213 result.id = Token.Id.Invalid;1213 result.id = .Invalid;
1214 }1214 }
1215 break;1215 break;
1216 },1216 },
1217 },1217 },
1218 State.FloatFractionDecNoUnderscore => switch (c) {1218 .float_fraction_dec_no_underscore => switch (c) {
1219 '0'...'9' => {1219 '0'...'9' => {
1220 state = State.FloatFractionDec;1220 state = .float_fraction_dec;
1221 },1221 },
1222 else => {1222 else => {
1223 result.id = Token.Id.Invalid;1223 result.id = .Invalid;
1224 break;1224 break;
1225 },1225 },
1226 },1226 },
1227 State.FloatFractionDec => switch (c) {1227 .float_fraction_dec => switch (c) {
1228 '_' => {1228 '_' => {
1229 state = State.FloatFractionDecNoUnderscore;1229 state = .float_fraction_dec_no_underscore;
1230 },1230 },
1231 'e', 'E' => {1231 'e', 'E' => {
1232 state = State.FloatExponentUnsigned;1232 state = .float_exponent_unsigned;
1233 },1233 },
1234 '0'...'9' => {},1234 '0'...'9' => {},
1235 else => {1235 else => {
1236 if (isIdentifierChar(c)) {1236 if (isIdentifierChar(c)) {
1237 result.id = Token.Id.Invalid;1237 result.id = .Invalid;
1238 }1238 }
1239 break;1239 break;
1240 },1240 },
1241 },1241 },
1242 State.FloatFractionHexNoUnderscore => switch (c) {1242 .float_fraction_hex_no_underscore => switch (c) {
1243 '0'...'9', 'a'...'f', 'A'...'F' => {1243 '0'...'9', 'a'...'f', 'A'...'F' => {
1244 state = State.FloatFractionHex;1244 state = .float_fraction_hex;
1245 },1245 },
1246 else => {1246 else => {
1247 result.id = Token.Id.Invalid;1247 result.id = .Invalid;
1248 break;1248 break;
1249 },1249 },
1250 },1250 },
1251 State.FloatFractionHex => switch (c) {1251 .float_fraction_hex => switch (c) {
1252 '_' => {1252 '_' => {
1253 state = State.FloatFractionHexNoUnderscore;1253 state = .float_fraction_hex_no_underscore;
1254 },1254 },
1255 'p', 'P' => {1255 'p', 'P' => {
1256 state = State.FloatExponentUnsigned;1256 state = .float_exponent_unsigned;
1257 },1257 },
1258 '0'...'9', 'a'...'f', 'A'...'F' => {},1258 '0'...'9', 'a'...'f', 'A'...'F' => {},
1259 else => {1259 else => {
1260 if (isIdentifierChar(c)) {1260 if (isIdentifierChar(c)) {
1261 result.id = Token.Id.Invalid;1261 result.id = .Invalid;
1262 }1262 }
1263 break;1263 break;
1264 },1264 },
1265 },1265 },
1266 State.FloatExponentUnsigned => switch (c) {1266 .float_exponent_unsigned => switch (c) {
1267 '+', '-' => {1267 '+', '-' => {
1268 state = State.FloatExponentNumberNoUnderscore;1268 state = .float_exponent_num_no_underscore;
1269 },1269 },
1270 else => {1270 else => {
1271 // reinterpret as a normal exponent number1271 // reinterpret as a normal exponent number
1272 self.index -= 1;1272 self.index -= 1;
1273 state = State.FloatExponentNumberNoUnderscore;1273 state = .float_exponent_num_no_underscore;
1274 },1274 },
1275 },1275 },
1276 State.FloatExponentNumberNoUnderscore => switch (c) {1276 .float_exponent_num_no_underscore => switch (c) {
1277 '0'...'9' => {1277 '0'...'9' => {
1278 state = State.FloatExponentNumber;1278 state = .float_exponent_num;
1279 },1279 },
1280 else => {1280 else => {
1281 result.id = Token.Id.Invalid;1281 result.id = .Invalid;
1282 break;1282 break;
1283 },1283 },
1284 },1284 },
1285 State.FloatExponentNumber => switch (c) {1285 .float_exponent_num => switch (c) {
1286 '_' => {1286 '_' => {
1287 state = State.FloatExponentNumberNoUnderscore;1287 state = .float_exponent_num_no_underscore;
1288 },1288 },
1289 '0'...'9' => {},1289 '0'...'9' => {},
1290 else => {1290 else => {
1291 if (isIdentifierChar(c)) {1291 if (isIdentifierChar(c)) {
1292 result.id = Token.Id.Invalid;1292 result.id = .Invalid;
1293 }1293 }
1294 break;1294 break;
1295 },1295 },
...@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {...@@ -1297,123 +1297,123 @@ pub const Tokenizer = struct {
1297 }1297 }
1298 } else if (self.index == self.buffer.len) {1298 } else if (self.index == self.buffer.len) {
1299 switch (state) {1299 switch (state) {
1300 State.Start,1300 .start,
1301 State.IntegerLiteralDec,1301 .int_literal_dec,
1302 State.IntegerLiteralBin,1302 .int_literal_bin,
1303 State.IntegerLiteralOct,1303 .int_literal_oct,
1304 State.IntegerLiteralHex,1304 .int_literal_hex,
1305 State.NumberDotDec,1305 .num_dot_dec,
1306 State.NumberDotHex,1306 .num_dot_hex,
1307 State.FloatFractionDec,1307 .float_fraction_dec,
1308 State.FloatFractionHex,1308 .float_fraction_hex,
1309 State.FloatExponentNumber,1309 .float_exponent_num,
1310 State.StringLiteral, // find this error later1310 .string_literal, // find this error later
1311 State.MultilineStringLiteralLine,1311 .multiline_string_literal_line,
1312 State.Builtin,1312 .builtin,
1313 => {},1313 => {},
13141314
1315 State.Identifier => {1315 .identifier => {
1316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {1316 if (Token.getKeyword(self.buffer[result.start..self.index])) |id| {
1317 result.id = id;1317 result.id = id;
1318 }1318 }
1319 },1319 },
1320 State.LineCommentStart, State.LineComment => {1320 .line_comment, .line_comment_start => {
1321 result.id = Token.Id.LineComment;1321 result.id = .LineComment;
1322 },1322 },
1323 State.DocComment, State.DocCommentStart => {1323 .doc_comment, .doc_comment_start => {
1324 result.id = Token.Id.DocComment;1324 result.id = .DocComment;
1325 },1325 },
1326 State.ContainerDocComment => {1326 .container_doc_comment => {
1327 result.id = Token.Id.ContainerDocComment;1327 result.id = .ContainerDocComment;
1328 },1328 },
13291329
1330 State.IntegerLiteralDecNoUnderscore,1330 .int_literal_dec_no_underscore,
1331 State.IntegerLiteralBinNoUnderscore,1331 .int_literal_bin_no_underscore,
1332 State.IntegerLiteralOctNoUnderscore,1332 .int_literal_oct_no_underscore,
1333 State.IntegerLiteralHexNoUnderscore,1333 .int_literal_hex_no_underscore,
1334 State.FloatFractionDecNoUnderscore,1334 .float_fraction_dec_no_underscore,
1335 State.FloatFractionHexNoUnderscore,1335 .float_fraction_hex_no_underscore,
1336 State.FloatExponentNumberNoUnderscore,1336 .float_exponent_num_no_underscore,
1337 State.FloatExponentUnsigned,1337 .float_exponent_unsigned,
1338 State.SawAtSign,1338 .saw_at_sign,
1339 State.Backslash,1339 .backslash,
1340 State.CharLiteral,1340 .char_literal,
1341 State.CharLiteralBackslash,1341 .char_literal_backslash,
1342 State.CharLiteralHexEscape,1342 .char_literal_hex_escape,
1343 State.CharLiteralUnicodeEscapeSawU,1343 .char_literal_unicode_escape_saw_u,
1344 State.CharLiteralUnicodeEscape,1344 .char_literal_unicode_escape,
1345 State.CharLiteralUnicodeInvalid,1345 .char_literal_unicode_invalid,
1346 State.CharLiteralEnd,1346 .char_literal_end,
1347 State.CharLiteralUnicode,1347 .char_literal_unicode,
1348 State.StringLiteralBackslash,1348 .string_literal_backslash,
1349 => {1349 => {
1350 result.id = Token.Id.Invalid;1350 result.id = .Invalid;
1351 },1351 },
13521352
1353 State.Equal => {1353 .equal => {
1354 result.id = Token.Id.Equal;1354 result.id = .Equal;
1355 },1355 },
1356 State.Bang => {1356 .bang => {
1357 result.id = Token.Id.Bang;1357 result.id = .Bang;
1358 },1358 },
1359 State.Minus => {1359 .minus => {
1360 result.id = Token.Id.Minus;1360 result.id = .Minus;
1361 },1361 },
1362 State.Slash => {1362 .slash => {
1363 result.id = Token.Id.Slash;1363 result.id = .Slash;
1364 },1364 },
1365 State.Zero => {1365 .zero => {
1366 result.id = Token.Id.IntegerLiteral;1366 result.id = .IntegerLiteral;
1367 },1367 },
1368 State.Ampersand => {1368 .ampersand => {
1369 result.id = Token.Id.Ampersand;1369 result.id = .Ampersand;
1370 },1370 },
1371 State.Period => {1371 .period => {
1372 result.id = Token.Id.Period;1372 result.id = .Period;
1373 },1373 },
1374 State.Period2 => {1374 .period_2 => {
1375 result.id = Token.Id.Ellipsis2;1375 result.id = .Ellipsis2;
1376 },1376 },
1377 State.Pipe => {1377 .pipe => {
1378 result.id = Token.Id.Pipe;1378 result.id = .Pipe;
1379 },1379 },
1380 State.AngleBracketAngleBracketRight => {1380 .angle_bracket_angle_bracket_right => {
1381 result.id = Token.Id.AngleBracketAngleBracketRight;1381 result.id = .AngleBracketAngleBracketRight;
1382 },1382 },
1383 State.AngleBracketRight => {1383 .angle_bracket_right => {
1384 result.id = Token.Id.AngleBracketRight;1384 result.id = .AngleBracketRight;
1385 },1385 },
1386 State.AngleBracketAngleBracketLeft => {1386 .angle_bracket_angle_bracket_left => {
1387 result.id = Token.Id.AngleBracketAngleBracketLeft;1387 result.id = .AngleBracketAngleBracketLeft;
1388 },1388 },
1389 State.AngleBracketLeft => {1389 .angle_bracket_left => {
1390 result.id = Token.Id.AngleBracketLeft;1390 result.id = .AngleBracketLeft;
1391 },1391 },
1392 State.PlusPercent => {1392 .plus_percent => {
1393 result.id = Token.Id.PlusPercent;1393 result.id = .PlusPercent;
1394 },1394 },
1395 State.Plus => {1395 .plus => {
1396 result.id = Token.Id.Plus;1396 result.id = .Plus;
1397 },1397 },
1398 State.Percent => {1398 .percent => {
1399 result.id = Token.Id.Percent;1399 result.id = .Percent;
1400 },1400 },
1401 State.Caret => {1401 .caret => {
1402 result.id = Token.Id.Caret;1402 result.id = .Caret;
1403 },1403 },
1404 State.AsteriskPercent => {1404 .asterisk_percent => {
1405 result.id = Token.Id.AsteriskPercent;1405 result.id = .AsteriskPercent;
1406 },1406 },
1407 State.Asterisk => {1407 .asterisk => {
1408 result.id = Token.Id.Asterisk;1408 result.id = .Asterisk;
1409 },1409 },
1410 State.MinusPercent => {1410 .minus_percent => {
1411 result.id = Token.Id.MinusPercent;1411 result.id = .MinusPercent;
1412 },1412 },
1413 }1413 }
1414 }1414 }
14151415
1416 if (result.id == Token.Id.Eof) {1416 if (result.id == .Eof) {
1417 if (self.pending_invalid_token) |token| {1417 if (self.pending_invalid_token) |token| {
1418 self.pending_invalid_token = null;1418 self.pending_invalid_token = null;
1419 return token;1419 return token;
...@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {...@@ -1428,8 +1428,8 @@ pub const Tokenizer = struct {
1428 if (self.pending_invalid_token != null) return;1428 if (self.pending_invalid_token != null) return;
1429 const invalid_length = self.getInvalidCharacterLength();1429 const invalid_length = self.getInvalidCharacterLength();
1430 if (invalid_length == 0) return;1430 if (invalid_length == 0) return;
1431 self.pending_invalid_token = Token{1431 self.pending_invalid_token = .{
1432 .id = Token.Id.Invalid,1432 .id = .Invalid,
1433 .start = self.index,1433 .start = self.index,
1434 .end = self.index + invalid_length,1434 .end = self.index + invalid_length,
1435 };1435 };
...@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {...@@ -1474,7 +1474,7 @@ pub const Tokenizer = struct {
1474};1474};
14751475
1476test "tokenizer" {1476test "tokenizer" {
1477 testTokenize("test", &[_]Token.Id{Token.Id.Keyword_test});1477 testTokenize("test", &[_]Token.Id{.Keyword_test});
1478}1478}
14791479
1480test "tokenizer - unknown length pointer and then c pointer" {1480test "tokenizer - unknown length pointer and then c pointer" {
...@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {...@@ -1482,15 +1482,15 @@ test "tokenizer - unknown length pointer and then c pointer" {
1482 \\[*]u81482 \\[*]u8
1483 \\[*c]u81483 \\[*c]u8
1484 , &[_]Token.Id{1484 , &[_]Token.Id{
1485 Token.Id.LBracket,1485 .LBracket,
1486 Token.Id.Asterisk,1486 .Asterisk,
1487 Token.Id.RBracket,1487 .RBracket,
1488 Token.Id.Identifier,1488 .Identifier,
1489 Token.Id.LBracket,1489 .LBracket,
1490 Token.Id.Asterisk,1490 .Asterisk,
1491 Token.Id.Identifier,1491 .Identifier,
1492 Token.Id.RBracket,1492 .RBracket,
1493 Token.Id.Identifier,1493 .Identifier,
1494 });1494 });
1495}1495}
14961496
...@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {...@@ -1561,125 +1561,125 @@ test "tokenizer - char literal with unicode code point" {
15611561
1562test "tokenizer - float literal e exponent" {1562test "tokenizer - float literal e exponent" {
1563 testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{1563 testTokenize("a = 4.94065645841246544177e-324;\n", &[_]Token.Id{
1564 Token.Id.Identifier,1564 .Identifier,
1565 Token.Id.Equal,1565 .Equal,
1566 Token.Id.FloatLiteral,1566 .FloatLiteral,
1567 Token.Id.Semicolon,1567 .Semicolon,
1568 });1568 });
1569}1569}
15701570
1571test "tokenizer - float literal p exponent" {1571test "tokenizer - float literal p exponent" {
1572 testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{1572 testTokenize("a = 0x1.a827999fcef32p+1022;\n", &[_]Token.Id{
1573 Token.Id.Identifier,1573 .Identifier,
1574 Token.Id.Equal,1574 .Equal,
1575 Token.Id.FloatLiteral,1575 .FloatLiteral,
1576 Token.Id.Semicolon,1576 .Semicolon,
1577 });1577 });
1578}1578}
15791579
1580test "tokenizer - chars" {1580test "tokenizer - chars" {
1581 testTokenize("'c'", &[_]Token.Id{Token.Id.CharLiteral});1581 testTokenize("'c'", &[_]Token.Id{.CharLiteral});
1582}1582}
15831583
1584test "tokenizer - invalid token characters" {1584test "tokenizer - invalid token characters" {
1585 testTokenize("#", &[_]Token.Id{Token.Id.Invalid});1585 testTokenize("#", &[_]Token.Id{.Invalid});
1586 testTokenize("`", &[_]Token.Id{Token.Id.Invalid});1586 testTokenize("`", &[_]Token.Id{.Invalid});
1587 testTokenize("'c", &[_]Token.Id{Token.Id.Invalid});1587 testTokenize("'c", &[_]Token.Id{.Invalid});
1588 testTokenize("'", &[_]Token.Id{Token.Id.Invalid});1588 testTokenize("'", &[_]Token.Id{.Invalid});
1589 testTokenize("''", &[_]Token.Id{ Token.Id.Invalid, Token.Id.Invalid });1589 testTokenize("''", &[_]Token.Id{ .Invalid, .Invalid });
1590}1590}
15911591
1592test "tokenizer - invalid literal/comment characters" {1592test "tokenizer - invalid literal/comment characters" {
1593 testTokenize("\"\x00\"", &[_]Token.Id{1593 testTokenize("\"\x00\"", &[_]Token.Id{
1594 Token.Id.StringLiteral,1594 .StringLiteral,
1595 Token.Id.Invalid,1595 .Invalid,
1596 });1596 });
1597 testTokenize("//\x00", &[_]Token.Id{1597 testTokenize("//\x00", &[_]Token.Id{
1598 Token.Id.LineComment,1598 .LineComment,
1599 Token.Id.Invalid,1599 .Invalid,
1600 });1600 });
1601 testTokenize("//\x1f", &[_]Token.Id{1601 testTokenize("//\x1f", &[_]Token.Id{
1602 Token.Id.LineComment,1602 .LineComment,
1603 Token.Id.Invalid,1603 .Invalid,
1604 });1604 });
1605 testTokenize("//\x7f", &[_]Token.Id{1605 testTokenize("//\x7f", &[_]Token.Id{
1606 Token.Id.LineComment,1606 .LineComment,
1607 Token.Id.Invalid,1607 .Invalid,
1608 });1608 });
1609}1609}
16101610
1611test "tokenizer - utf8" {1611test "tokenizer - utf8" {
1612 testTokenize("//\xc2\x80", &[_]Token.Id{Token.Id.LineComment});1612 testTokenize("//\xc2\x80", &[_]Token.Id{.LineComment});
1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{Token.Id.LineComment});1613 testTokenize("//\xf4\x8f\xbf\xbf", &[_]Token.Id{.LineComment});
1614}1614}
16151615
1616test "tokenizer - invalid utf8" {1616test "tokenizer - invalid utf8" {
1617 testTokenize("//\x80", &[_]Token.Id{1617 testTokenize("//\x80", &[_]Token.Id{
1618 Token.Id.LineComment,1618 .LineComment,
1619 Token.Id.Invalid,1619 .Invalid,
1620 });1620 });
1621 testTokenize("//\xbf", &[_]Token.Id{1621 testTokenize("//\xbf", &[_]Token.Id{
1622 Token.Id.LineComment,1622 .LineComment,
1623 Token.Id.Invalid,1623 .Invalid,
1624 });1624 });
1625 testTokenize("//\xf8", &[_]Token.Id{1625 testTokenize("//\xf8", &[_]Token.Id{
1626 Token.Id.LineComment,1626 .LineComment,
1627 Token.Id.Invalid,1627 .Invalid,
1628 });1628 });
1629 testTokenize("//\xff", &[_]Token.Id{1629 testTokenize("//\xff", &[_]Token.Id{
1630 Token.Id.LineComment,1630 .LineComment,
1631 Token.Id.Invalid,1631 .Invalid,
1632 });1632 });
1633 testTokenize("//\xc2\xc0", &[_]Token.Id{1633 testTokenize("//\xc2\xc0", &[_]Token.Id{
1634 Token.Id.LineComment,1634 .LineComment,
1635 Token.Id.Invalid,1635 .Invalid,
1636 });1636 });
1637 testTokenize("//\xe0", &[_]Token.Id{1637 testTokenize("//\xe0", &[_]Token.Id{
1638 Token.Id.LineComment,1638 .LineComment,
1639 Token.Id.Invalid,1639 .Invalid,
1640 });1640 });
1641 testTokenize("//\xf0", &[_]Token.Id{1641 testTokenize("//\xf0", &[_]Token.Id{
1642 Token.Id.LineComment,1642 .LineComment,
1643 Token.Id.Invalid,1643 .Invalid,
1644 });1644 });
1645 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{1645 testTokenize("//\xf0\x90\x80\xc0", &[_]Token.Id{
1646 Token.Id.LineComment,1646 .LineComment,
1647 Token.Id.Invalid,1647 .Invalid,
1648 });1648 });
1649}1649}
16501650
1651test "tokenizer - illegal unicode codepoints" {1651test "tokenizer - illegal unicode codepoints" {
1652 // unicode newline characters.U+0085, U+2028, U+20291652 // unicode newline characters.U+0085, U+2028, U+2029
1653 testTokenize("//\xc2\x84", &[_]Token.Id{Token.Id.LineComment});1653 testTokenize("//\xc2\x84", &[_]Token.Id{.LineComment});
1654 testTokenize("//\xc2\x85", &[_]Token.Id{1654 testTokenize("//\xc2\x85", &[_]Token.Id{
1655 Token.Id.LineComment,1655 .LineComment,
1656 Token.Id.Invalid,1656 .Invalid,
1657 });1657 });
1658 testTokenize("//\xc2\x86", &[_]Token.Id{Token.Id.LineComment});1658 testTokenize("//\xc2\x86", &[_]Token.Id{.LineComment});
1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{Token.Id.LineComment});1659 testTokenize("//\xe2\x80\xa7", &[_]Token.Id{.LineComment});
1660 testTokenize("//\xe2\x80\xa8", &[_]Token.Id{1660 testTokenize("//\xe2\x80\xa8", &[_]Token.Id{
1661 Token.Id.LineComment,1661 .LineComment,
1662 Token.Id.Invalid,1662 .Invalid,
1663 });1663 });
1664 testTokenize("//\xe2\x80\xa9", &[_]Token.Id{1664 testTokenize("//\xe2\x80\xa9", &[_]Token.Id{
1665 Token.Id.LineComment,1665 .LineComment,
1666 Token.Id.Invalid,1666 .Invalid,
1667 });1667 });
1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{Token.Id.LineComment});1668 testTokenize("//\xe2\x80\xaa", &[_]Token.Id{.LineComment});
1669}1669}
16701670
1671test "tokenizer - string identifier and builtin fns" {1671test "tokenizer - string identifier and builtin fns" {
1672 testTokenize(1672 testTokenize(
1673 \\const @"if" = @import("std");1673 \\const @"if" = @import("std");
1674 , &[_]Token.Id{1674 , &[_]Token.Id{
1675 Token.Id.Keyword_const,1675 .Keyword_const,
1676 Token.Id.Identifier,1676 .Identifier,
1677 Token.Id.Equal,1677 .Equal,
1678 Token.Id.Builtin,1678 .Builtin,
1679 Token.Id.LParen,1679 .LParen,
1680 Token.Id.StringLiteral,1680 .StringLiteral,
1681 Token.Id.RParen,1681 .RParen,
1682 Token.Id.Semicolon,1682 .Semicolon,
1683 });1683 });
1684}1684}
16851685
...@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {...@@ -1687,26 +1687,26 @@ test "tokenizer - multiline string literal with literal tab" {
1687 testTokenize(1687 testTokenize(
1688 \\\\foo bar1688 \\\\foo bar
1689 , &[_]Token.Id{1689 , &[_]Token.Id{
1690 Token.Id.MultilineStringLiteralLine,1690 .MultilineStringLiteralLine,
1691 });1691 });
1692}1692}
16931693
1694test "tokenizer - pipe and then invalid" {1694test "tokenizer - pipe and then invalid" {
1695 testTokenize("||=", &[_]Token.Id{1695 testTokenize("||=", &[_]Token.Id{
1696 Token.Id.PipePipe,1696 .PipePipe,
1697 Token.Id.Equal,1697 .Equal,
1698 });1698 });
1699}1699}
17001700
1701test "tokenizer - line comment and doc comment" {1701test "tokenizer - line comment and doc comment" {
1702 testTokenize("//", &[_]Token.Id{Token.Id.LineComment});1702 testTokenize("//", &[_]Token.Id{.LineComment});
1703 testTokenize("// a / b", &[_]Token.Id{Token.Id.LineComment});1703 testTokenize("// a / b", &[_]Token.Id{.LineComment});
1704 testTokenize("// /", &[_]Token.Id{Token.Id.LineComment});1704 testTokenize("// /", &[_]Token.Id{.LineComment});
1705 testTokenize("/// a", &[_]Token.Id{Token.Id.DocComment});1705 testTokenize("/// a", &[_]Token.Id{.DocComment});
1706 testTokenize("///", &[_]Token.Id{Token.Id.DocComment});1706 testTokenize("///", &[_]Token.Id{.DocComment});
1707 testTokenize("////", &[_]Token.Id{Token.Id.LineComment});1707 testTokenize("////", &[_]Token.Id{.LineComment});
1708 testTokenize("//!", &[_]Token.Id{Token.Id.ContainerDocComment});1708 testTokenize("//!", &[_]Token.Id{.ContainerDocComment});
1709 testTokenize("//!!", &[_]Token.Id{Token.Id.ContainerDocComment});1709 testTokenize("//!!", &[_]Token.Id{.ContainerDocComment});
1710}1710}
17111711
1712test "tokenizer - line comment followed by identifier" {1712test "tokenizer - line comment followed by identifier" {
...@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {...@@ -1715,28 +1715,28 @@ test "tokenizer - line comment followed by identifier" {
1715 \\ // another1715 \\ // another
1716 \\ Another,1716 \\ Another,
1717 , &[_]Token.Id{1717 , &[_]Token.Id{
1718 Token.Id.Identifier,1718 .Identifier,
1719 Token.Id.Comma,1719 .Comma,
1720 Token.Id.LineComment,1720 .LineComment,
1721 Token.Id.Identifier,1721 .Identifier,
1722 Token.Id.Comma,1722 .Comma,
1723 });1723 });
1724}1724}
17251725
1726test "tokenizer - UTF-8 BOM is recognized and skipped" {1726test "tokenizer - UTF-8 BOM is recognized and skipped" {
1727 testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{1727 testTokenize("\xEF\xBB\xBFa;\n", &[_]Token.Id{
1728 Token.Id.Identifier,1728 .Identifier,
1729 Token.Id.Semicolon,1729 .Semicolon,
1730 });1730 });
1731}1731}
17321732
1733test "correctly parse pointer assignment" {1733test "correctly parse pointer assignment" {
1734 testTokenize("b.*=3;\n", &[_]Token.Id{1734 testTokenize("b.*=3;\n", &[_]Token.Id{
1735 Token.Id.Identifier,1735 .Identifier,
1736 Token.Id.PeriodAsterisk,1736 .PeriodAsterisk,
1737 Token.Id.Equal,1737 .Equal,
1738 Token.Id.IntegerLiteral,1738 .IntegerLiteral,
1739 Token.Id.Semicolon,1739 .Semicolon,
1740 });1740 });
1741}1741}
17421742
...@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {...@@ -1979,5 +1979,5 @@ fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
1979 }1979 }
1980 }1980 }
1981 const last_token = tokenizer.next();1981 const last_token = tokenizer.next();
1982 std.testing.expect(last_token.id == Token.Id.Eof);1982 std.testing.expect(last_token.id == .Eof);
1983}1983}
src-self-hosted/translate_c.zig+3-3
...@@ -668,7 +668,7 @@ fn transTypeDefAsBuiltin(c: *Context, typedef_decl: *const ZigClangTypedefNameDe...@@ -668,7 +668,7 @@ fn transTypeDefAsBuiltin(c: *Context, typedef_decl: *const ZigClangTypedefNameDe
668 return transCreateNodeIdentifier(c, builtin_name);668 return transCreateNodeIdentifier(c, builtin_name);
669}669}
670670
671fn checkForBuiltinTypedef(checked_name: []const u8) !?[]const u8 {671fn checkForBuiltinTypedef(checked_name: []const u8) ?[]const u8 {
672 const table = [_][2][]const u8{672 const table = [_][2][]const u8{
673 .{ "uint8_t", "u8" },673 .{ "uint8_t", "u8" },
674 .{ "int8_t", "i8" },674 .{ "int8_t", "i8" },
...@@ -703,7 +703,7 @@ fn transTypeDef(c: *Context, typedef_decl: *const ZigClangTypedefNameDecl, top_l...@@ -703,7 +703,7 @@ fn transTypeDef(c: *Context, typedef_decl: *const ZigClangTypedefNameDecl, top_l
703 // TODO https://github.com/ziglang/zig/issues/3756703 // TODO https://github.com/ziglang/zig/issues/3756
704 // TODO https://github.com/ziglang/zig/issues/1802704 // TODO https://github.com/ziglang/zig/issues/1802
705 const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.a(), "{}_{}", .{ typedef_name, c.getMangle() }) else typedef_name;705 const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.a(), "{}_{}", .{ typedef_name, c.getMangle() }) else typedef_name;
706 if (try checkForBuiltinTypedef(checked_name)) |builtin| {706 if (checkForBuiltinTypedef(checked_name)) |builtin| {
707 return transTypeDefAsBuiltin(c, typedef_decl, builtin);707 return transTypeDefAsBuiltin(c, typedef_decl, builtin);
708 }708 }
709709
...@@ -1411,7 +1411,7 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt)...@@ -1411,7 +1411,7 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt)
1411 const underlying_type = ZigClangQualType_getTypePtr(underlying_qual);1411 const underlying_type = ZigClangQualType_getTypePtr(underlying_qual);
14121412
1413 const mangled_name = try block_scope.makeMangledName(c, name);1413 const mangled_name = try block_scope.makeMangledName(c, name);
1414 if (try checkForBuiltinTypedef(name)) |builtin| {1414 if (checkForBuiltinTypedef(name)) |builtin| {
1415 try block_scope.variables.push(.{1415 try block_scope.variables.push(.{
1416 .alias = builtin,1416 .alias = builtin,
1417 .name = mangled_name,1417 .name = mangled_name,
src/parser.cpp-9
...@@ -1609,7 +1609,6 @@ static AstNode *ast_parse_suffix_expr(ParseContext *pc) {...@@ -1609,7 +1609,6 @@ static AstNode *ast_parse_suffix_expr(ParseContext *pc) {
1609// / IfTypeExpr1609// / IfTypeExpr
1610// / INTEGER1610// / INTEGER
1611// / KEYWORD_comptime TypeExpr1611// / KEYWORD_comptime TypeExpr
1612// / KEYWORD_nosuspend TypeExpr
1613// / KEYWORD_error DOT IDENTIFIER1612// / KEYWORD_error DOT IDENTIFIER
1614// / KEYWORD_false1613// / KEYWORD_false
1615// / KEYWORD_null1614// / KEYWORD_null
...@@ -1711,14 +1710,6 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {...@@ -1711,14 +1710,6 @@ static AstNode *ast_parse_primary_type_expr(ParseContext *pc) {
1711 return res;1710 return res;
1712 }1711 }
17131712
1714 Token *nosuspend = eat_token_if(pc, TokenIdKeywordNoSuspend);
1715 if (nosuspend != nullptr) {
1716 AstNode *expr = ast_expect(pc, ast_parse_type_expr);
1717 AstNode *res = ast_create_node(pc, NodeTypeNoSuspend, nosuspend);
1718 res->data.nosuspend_expr.expr = expr;
1719 return res;
1720 }
1721
1722 Token *error = eat_token_if(pc, TokenIdKeywordError);1713 Token *error = eat_token_if(pc, TokenIdKeywordError);
1723 if (error != nullptr) {1714 if (error != nullptr) {
1724 Token *dot = expect_token(pc, TokenIdDot);1715 Token *dot = expect_token(pc, TokenIdDot);