authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-01-29 12:13:53-05:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-01-29 12:13:53-05:00
log0cc2489d22a27b2dc82ee9ef72e945b9fa97c8fe
treea6424c9aee30ed8439c06bd36e472be96ed43999
parent534014f84e2e9605022ba6d6c2d2b7be1e575468
parentabd1a7c91c611b35754e5d22a8755cfbebc65861
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #4317 from Vexu/std-c

Add (unfinished) C parser to std lib

4 files changed, 3701 insertions(+), 0 deletions(-)

lib/std/c.zig+6
...@@ -2,6 +2,12 @@ const builtin = @import("builtin");...@@ -2,6 +2,12 @@ const builtin = @import("builtin");
2const std = @import("std");2const std = @import("std");
3const page_size = std.mem.page_size;3const page_size = std.mem.page_size;
44
5pub const tokenizer = @import("c/tokenizer.zig");
6pub const Token = tokenizer.Token;
7pub const Tokenizer = tokenizer.Tokenizer;
8pub const parse = @import("c/parse.zig").parse;
9pub const ast = @import("c/ast.zig");
10
5pub usingnamespace @import("os/bits.zig");11pub usingnamespace @import("os/bits.zig");
612
7pub usingnamespace switch (builtin.os) {13pub usingnamespace switch (builtin.os) {
lib/std/c/ast.zig created+681
...@@ -0,0 +1,681 @@
1const std = @import("std");
2const SegmentedList = std.SegmentedList;
3const Token = std.c.Token;
4const Source = std.c.tokenizer.Source;
5
6pub const TokenIndex = usize;
7
8pub const Tree = struct {
9 tokens: TokenList,
10 sources: SourceList,
11 root_node: *Node.Root,
12 arena_allocator: std.heap.ArenaAllocator,
13 msgs: MsgList,
14
15 pub const SourceList = SegmentedList(Source, 4);
16 pub const TokenList = Source.TokenList;
17 pub const MsgList = SegmentedList(Msg, 0);
18
19 pub fn deinit(self: *Tree) void {
20 // Here we copy the arena allocator into stack memory, because
21 // otherwise it would destroy itself while it was still working.
22 var arena_allocator = self.arena_allocator;
23 arena_allocator.deinit();
24 // self is destroyed
25 }
26
27 pub fn tokenSlice(tree: *Tree, token: TokenIndex) []const u8 {
28 return tree.tokens.at(token).slice();
29 }
30
31 pub fn tokenEql(tree: *Tree, a: TokenIndex, b: TokenIndex) bool {
32 const atok = tree.tokens.at(a);
33 const btok = tree.tokens.at(b);
34 return atok.eql(btok.*);
35 }
36};
37
38pub const Msg = struct {
39 kind: enum {
40 Error,
41 Warning,
42 Note,
43 },
44 inner: Error,
45};
46
47pub const Error = union(enum) {
48 InvalidToken: SingleTokenError("invalid token '{}'"),
49 ExpectedToken: ExpectedToken,
50 ExpectedExpr: SingleTokenError("expected expression, found '{}'"),
51 ExpectedTypeName: SingleTokenError("expected type name, found '{}'"),
52 ExpectedFnBody: SingleTokenError("expected function body, found '{}'"),
53 ExpectedDeclarator: SingleTokenError("expected declarator, found '{}'"),
54 ExpectedInitializer: SingleTokenError("expected initializer, found '{}'"),
55 ExpectedEnumField: SingleTokenError("expected enum field, found '{}'"),
56 ExpectedType: SingleTokenError("expected enum field, found '{}'"),
57 InvalidTypeSpecifier: InvalidTypeSpecifier,
58 InvalidStorageClass: SingleTokenError("invalid storage class, found '{}'"),
59 InvalidDeclarator: SimpleError("invalid declarator"),
60 DuplicateQualifier: SingleTokenError("duplicate type qualifier '{}'"),
61 DuplicateSpecifier: SingleTokenError("duplicate declaration specifier '{}'"),
62 MustUseKwToRefer: MustUseKwToRefer,
63 FnSpecOnNonFn: SingleTokenError("function specifier '{}' on non function"),
64 NothingDeclared: SimpleError("declaration doesn't declare anything"),
65 QualifierIgnored: SingleTokenError("qualifier '{}' ignored"),
66
67 pub fn render(self: *const Error, tree: *Tree, stream: var) !void {
68 switch (self.*) {
69 .InvalidToken => |*x| return x.render(tree, stream),
70 .ExpectedToken => |*x| return x.render(tree, stream),
71 .ExpectedExpr => |*x| return x.render(tree, stream),
72 .ExpectedTypeName => |*x| return x.render(tree, stream),
73 .ExpectedDeclarator => |*x| return x.render(tree, stream),
74 .ExpectedFnBody => |*x| return x.render(tree, stream),
75 .ExpectedInitializer => |*x| return x.render(tree, stream),
76 .ExpectedEnumField => |*x| return x.render(tree, stream),
77 .ExpectedType => |*x| return x.render(tree, stream),
78 .InvalidTypeSpecifier => |*x| return x.render(tree, stream),
79 .InvalidStorageClass => |*x| return x.render(tree, stream),
80 .InvalidDeclarator => |*x| return x.render(tree, stream),
81 .DuplicateQualifier => |*x| return x.render(tree, stream),
82 .DuplicateSpecifier => |*x| return x.render(tree, stream),
83 .MustUseKwToRefer => |*x| return x.render(tree, stream),
84 .FnSpecOnNonFn => |*x| return x.render(tree, stream),
85 .NothingDeclared => |*x| return x.render(tree, stream),
86 .QualifierIgnored => |*x| return x.render(tree, stream),
87 }
88 }
89
90 pub fn loc(self: *const Error) TokenIndex {
91 switch (self.*) {
92 .InvalidToken => |x| return x.token,
93 .ExpectedToken => |x| return x.token,
94 .ExpectedExpr => |x| return x.token,
95 .ExpectedTypeName => |x| return x.token,
96 .ExpectedDeclarator => |x| return x.token,
97 .ExpectedFnBody => |x| return x.token,
98 .ExpectedInitializer => |x| return x.token,
99 .ExpectedEnumField => |x| return x.token,
100 .ExpectedType => |*x| return x.token,
101 .InvalidTypeSpecifier => |x| return x.token,
102 .InvalidStorageClass => |x| return x.token,
103 .InvalidDeclarator => |x| return x.token,
104 .DuplicateQualifier => |x| return x.token,
105 .DuplicateSpecifier => |x| return x.token,
106 .MustUseKwToRefer => |*x| return x.name,
107 .FnSpecOnNonFn => |*x| return x.name,
108 .NothingDeclared => |*x| return x.name,
109 .QualifierIgnored => |*x| return x.name,
110 }
111 }
112
113 pub const ExpectedToken = struct {
114 token: TokenIndex,
115 expected_id: @TagType(Token.Id),
116
117 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
118 const found_token = tree.tokens.at(self.token);
119 if (found_token.id == .Invalid) {
120 return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()});
121 } else {
122 const token_name = found_token.id.symbol();
123 return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name });
124 }
125 }
126 };
127
128 pub const InvalidTypeSpecifier = struct {
129 token: TokenIndex,
130 type_spec: *Node.TypeSpec,
131
132 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
133 try stream.write("invalid type specifier '");
134 try type_spec.spec.print(tree, stream);
135 const token_name = tree.tokens.at(self.token).id.symbol();
136 return stream.print("{}'", .{token_name});
137 }
138 };
139
140 pub const MustUseKwToRefer = struct {
141 kw: TokenIndex,
142 name: TokenIndex,
143
144 pub fn render(self: *const ExpectedToken, tree: *Tree, stream: var) !void {
145 return stream.print("must use '{}' tag to refer to type '{}'", .{ tree.slice(kw), tree.slice(name) });
146 }
147 };
148
149 fn SingleTokenError(comptime msg: []const u8) type {
150 return struct {
151 token: TokenIndex,
152
153 pub fn render(self: *const @This(), tree: *Tree, stream: var) !void {
154 const actual_token = tree.tokens.at(self.token);
155 return stream.print(msg, .{actual_token.id.symbol()});
156 }
157 };
158 }
159
160 fn SimpleError(comptime msg: []const u8) type {
161 return struct {
162 const ThisError = @This();
163
164 token: TokenIndex,
165
166 pub fn render(self: *const ThisError, tokens: *Tree.TokenList, stream: var) !void {
167 return stream.write(msg);
168 }
169 };
170 }
171};
172
173pub const Type = struct {
174 pub const TypeList = std.SegmentedList(*Type, 4);
175 @"const": bool = false,
176 atomic: bool = false,
177 @"volatile": bool = false,
178 restrict: bool = false,
179
180 id: union(enum) {
181 Int: struct {
182 id: Id,
183 is_signed: bool,
184
185 pub const Id = enum {
186 Char,
187 Short,
188 Int,
189 Long,
190 LongLong,
191 };
192 },
193 Float: struct {
194 id: Id,
195
196 pub const Id = enum {
197 Float,
198 Double,
199 LongDouble,
200 };
201 },
202 Pointer: *Type,
203 Function: struct {
204 return_type: *Type,
205 param_types: TypeList,
206 },
207 Typedef: *Type,
208 Record: *Node.RecordType,
209 Enum: *Node.EnumType,
210
211 /// Special case for macro parameters that can be any type.
212 /// Only present if `retain_macros == true`.
213 Macro,
214 },
215};
216
217pub const Node = struct {
218 id: Id,
219
220 pub const Id = enum {
221 Root,
222 EnumField,
223 RecordField,
224 RecordDeclarator,
225 JumpStmt,
226 ExprStmt,
227 LabeledStmt,
228 CompoundStmt,
229 IfStmt,
230 SwitchStmt,
231 WhileStmt,
232 DoStmt,
233 ForStmt,
234 StaticAssert,
235 Declarator,
236 Pointer,
237 FnDecl,
238 Typedef,
239 VarDecl,
240 };
241
242 pub const Root = struct {
243 base: Node = Node{ .id = .Root },
244 decls: DeclList,
245 eof: TokenIndex,
246
247 pub const DeclList = SegmentedList(*Node, 4);
248 };
249
250 pub const DeclSpec = struct {
251 storage_class: union(enum) {
252 Auto: TokenIndex,
253 Extern: TokenIndex,
254 Register: TokenIndex,
255 Static: TokenIndex,
256 Typedef: TokenIndex,
257 None,
258 } = .None,
259 thread_local: ?TokenIndex = null,
260 type_spec: TypeSpec = TypeSpec{},
261 fn_spec: union(enum) {
262 Inline: TokenIndex,
263 Noreturn: TokenIndex,
264 None,
265 } = .None,
266 align_spec: ?struct {
267 alignas: TokenIndex,
268 expr: *Node,
269 rparen: TokenIndex,
270 } = null,
271 };
272
273 pub const TypeSpec = struct {
274 qual: TypeQual = TypeQual{},
275 spec: union(enum) {
276 /// error or default to int
277 None,
278 Void: TokenIndex,
279 Char: struct {
280 sign: ?TokenIndex = null,
281 char: TokenIndex,
282 },
283 Short: struct {
284 sign: ?TokenIndex = null,
285 short: TokenIndex = null,
286 int: ?TokenIndex = null,
287 },
288 Int: struct {
289 sign: ?TokenIndex = null,
290 int: ?TokenIndex = null,
291 },
292 Long: struct {
293 sign: ?TokenIndex = null,
294 long: TokenIndex,
295 longlong: ?TokenIndex = null,
296 int: ?TokenIndex = null,
297 },
298 Float: struct {
299 float: TokenIndex,
300 complex: ?TokenIndex = null,
301 },
302 Double: struct {
303 long: ?TokenIndex = null,
304 double: ?TokenIndex,
305 complex: ?TokenIndex = null,
306 },
307 Bool: TokenIndex,
308 Atomic: struct {
309 atomic: TokenIndex,
310 typename: *Node,
311 rparen: TokenIndex,
312 },
313 Enum: *EnumType,
314 Record: *RecordType,
315 Typedef: struct {
316 sym: TokenIndex,
317 sym_type: *Type,
318 },
319
320 pub fn print(self: *@This(), self: *const @This(), tree: *Tree, stream: var) !void {
321 switch (self.spec) {
322 .None => unreachable,
323 .Void => |index| try stream.write(tree.slice(index)),
324 .Char => |char| {
325 if (char.sign) |s| {
326 try stream.write(tree.slice(s));
327 try stream.writeByte(' ');
328 }
329 try stream.write(tree.slice(char.char));
330 },
331 .Short => |short| {
332 if (short.sign) |s| {
333 try stream.write(tree.slice(s));
334 try stream.writeByte(' ');
335 }
336 try stream.write(tree.slice(short.short));
337 if (short.int) |i| {
338 try stream.writeByte(' ');
339 try stream.write(tree.slice(i));
340 }
341 },
342 .Int => |int| {
343 if (int.sign) |s| {
344 try stream.write(tree.slice(s));
345 try stream.writeByte(' ');
346 }
347 if (int.int) |i| {
348 try stream.writeByte(' ');
349 try stream.write(tree.slice(i));
350 }
351 },
352 .Long => |long| {
353 if (long.sign) |s| {
354 try stream.write(tree.slice(s));
355 try stream.writeByte(' ');
356 }
357 try stream.write(tree.slice(long.long));
358 if (long.longlong) |l| {
359 try stream.writeByte(' ');
360 try stream.write(tree.slice(l));
361 }
362 if (long.int) |i| {
363 try stream.writeByte(' ');
364 try stream.write(tree.slice(i));
365 }
366 },
367 .Float => |float| {
368 try stream.write(tree.slice(float.float));
369 if (float.complex) |c| {
370 try stream.writeByte(' ');
371 try stream.write(tree.slice(c));
372 }
373 },
374 .Double => |double| {
375 if (double.long) |l| {
376 try stream.write(tree.slice(l));
377 try stream.writeByte(' ');
378 }
379 try stream.write(tree.slice(double.double));
380 if (double.complex) |c| {
381 try stream.writeByte(' ');
382 try stream.write(tree.slice(c));
383 }
384 },
385 .Bool => |index| try stream.write(tree.slice(index)),
386 .Typedef => |typedef| try stream.write(tree.slice(typedef.sym)),
387 else => try stream.print("TODO print {}", self.spec),
388 }
389 }
390 } = .None,
391 };
392
393 pub const EnumType = struct {
394 tok: TokenIndex,
395 name: ?TokenIndex,
396 body: ?struct {
397 lbrace: TokenIndex,
398
399 /// always EnumField
400 fields: FieldList,
401 rbrace: TokenIndex,
402 },
403
404 pub const FieldList = Root.DeclList;
405 };
406
407 pub const EnumField = struct {
408 base: Node = Node{ .id = .EnumField },
409 name: TokenIndex,
410 value: ?*Node,
411 };
412
413 pub const RecordType = struct {
414 tok: TokenIndex,
415 kind: enum {
416 Struct,
417 Union,
418 },
419 name: ?TokenIndex,
420 body: ?struct {
421 lbrace: TokenIndex,
422
423 /// RecordField or StaticAssert
424 fields: FieldList,
425 rbrace: TokenIndex,
426 },
427
428 pub const FieldList = Root.DeclList;
429 };
430
431 pub const RecordField = struct {
432 base: Node = Node{ .id = .RecordField },
433 type_spec: TypeSpec,
434 declarators: DeclaratorList,
435 semicolon: TokenIndex,
436
437 pub const DeclaratorList = Root.DeclList;
438 };
439
440 pub const RecordDeclarator = struct {
441 base: Node = Node{ .id = .RecordDeclarator },
442 declarator: ?*Declarator,
443 bit_field_expr: ?*Expr,
444 };
445
446 pub const TypeQual = struct {
447 @"const": ?TokenIndex = null,
448 atomic: ?TokenIndex = null,
449 @"volatile": ?TokenIndex = null,
450 restrict: ?TokenIndex = null,
451 };
452
453 pub const JumpStmt = struct {
454 base: Node = Node{ .id = .JumpStmt },
455 ltoken: TokenIndex,
456 kind: union(enum) {
457 Break,
458 Continue,
459 Return: ?*Node,
460 Goto: TokenIndex,
461 },
462 semicolon: TokenIndex,
463 };
464
465 pub const ExprStmt = struct {
466 base: Node = Node{ .id = .ExprStmt },
467 expr: ?*Expr,
468 semicolon: TokenIndex,
469 };
470
471 pub const LabeledStmt = struct {
472 base: Node = Node{ .id = .LabeledStmt },
473 kind: union(enum) {
474 Label: TokenIndex,
475 Case: TokenIndex,
476 Default: TokenIndex,
477 },
478 stmt: *Node,
479 };
480
481 pub const CompoundStmt = struct {
482 base: Node = Node{ .id = .CompoundStmt },
483 lbrace: TokenIndex,
484 statements: StmtList,
485 rbrace: TokenIndex,
486
487 pub const StmtList = Root.DeclList;
488 };
489
490 pub const IfStmt = struct {
491 base: Node = Node{ .id = .IfStmt },
492 @"if": TokenIndex,
493 cond: *Node,
494 body: *Node,
495 @"else": ?struct {
496 tok: TokenIndex,
497 body: *Node,
498 },
499 };
500
501 pub const SwitchStmt = struct {
502 base: Node = Node{ .id = .SwitchStmt },
503 @"switch": TokenIndex,
504 expr: *Expr,
505 rparen: TokenIndex,
506 stmt: *Node,
507 };
508
509 pub const WhileStmt = struct {
510 base: Node = Node{ .id = .WhileStmt },
511 @"while": TokenIndex,
512 cond: *Expr,
513 rparen: TokenIndex,
514 body: *Node,
515 };
516
517 pub const DoStmt = struct {
518 base: Node = Node{ .id = .DoStmt },
519 do: TokenIndex,
520 body: *Node,
521 @"while": TokenIndex,
522 cond: *Expr,
523 semicolon: TokenIndex,
524 };
525
526 pub const ForStmt = struct {
527 base: Node = Node{ .id = .ForStmt },
528 @"for": TokenIndex,
529 init: ?*Node,
530 cond: ?*Expr,
531 semicolon: TokenIndex,
532 incr: ?*Expr,
533 rparen: TokenIndex,
534 body: *Node,
535 };
536
537 pub const StaticAssert = struct {
538 base: Node = Node{ .id = .StaticAssert },
539 assert: TokenIndex,
540 expr: *Node,
541 semicolon: TokenIndex,
542 };
543
544 pub const Declarator = struct {
545 base: Node = Node{ .id = .Declarator },
546 pointer: ?*Pointer,
547 prefix: union(enum) {
548 None,
549 Identifer: TokenIndex,
550 Complex: struct {
551 lparen: TokenIndex,
552 inner: *Node,
553 rparen: TokenIndex,
554 },
555 },
556 suffix: union(enum) {
557 None,
558 Fn: struct {
559 lparen: TokenIndex,
560 params: Params,
561 rparen: TokenIndex,
562 },
563 Array: Arrays,
564 },
565
566 pub const Arrays = std.SegmentedList(*Array, 2);
567 pub const Params = std.SegmentedList(*Param, 4);
568 };
569
570 pub const Array = struct {
571 lbracket: TokenIndex,
572 inner: union(enum) {
573 Inferred,
574 Unspecified: TokenIndex,
575 Variable: struct {
576 asterisk: ?TokenIndex,
577 static: ?TokenIndex,
578 qual: TypeQual,
579 expr: *Expr,
580 },
581 },
582 rbracket: TokenIndex,
583 };
584
585 pub const Pointer = struct {
586 base: Node = Node{ .id = .Pointer },
587 asterisk: TokenIndex,
588 qual: TypeQual,
589 pointer: ?*Pointer,
590 };
591
592 pub const Param = struct {
593 kind: union(enum) {
594 Variable,
595 Old: TokenIndex,
596 Normal: struct {
597 decl_spec: *DeclSpec,
598 declarator: *Node,
599 },
600 },
601 };
602
603 pub const FnDecl = struct {
604 base: Node = Node{ .id = .FnDecl },
605 decl_spec: DeclSpec,
606 declarator: *Declarator,
607 old_decls: OldDeclList,
608 body: ?*CompoundStmt,
609
610 pub const OldDeclList = SegmentedList(*Node, 0);
611 };
612
613 pub const Typedef = struct {
614 base: Node = Node{ .id = .Typedef },
615 decl_spec: DeclSpec,
616 declarators: DeclaratorList,
617 semicolon: TokenIndex,
618
619 pub const DeclaratorList = Root.DeclList;
620 };
621
622 pub const VarDecl = struct {
623 base: Node = Node{ .id = .VarDecl },
624 decl_spec: DeclSpec,
625 initializers: Initializers,
626 semicolon: TokenIndex,
627
628 pub const Initializers = Root.DeclList;
629 };
630
631 pub const Initialized = struct {
632 base: Node = Node{ .id = Initialized },
633 declarator: *Declarator,
634 eq: TokenIndex,
635 init: Initializer,
636 };
637
638 pub const Initializer = union(enum) {
639 list: struct {
640 initializers: InitializerList,
641 rbrace: TokenIndex,
642 },
643 expr: *Expr,
644 pub const InitializerList = std.SegmentedList(*Initializer, 4);
645 };
646
647 pub const Macro = struct {
648 base: Node = Node{ .id = Macro },
649 kind: union(enum) {
650 Undef: []const u8,
651 Fn: struct {
652 params: []const []const u8,
653 expr: *Expr,
654 },
655 Expr: *Expr,
656 },
657 };
658};
659
660pub const Expr = struct {
661 id: Id,
662 ty: *Type,
663 value: union(enum) {
664 None,
665 },
666
667 pub const Id = enum {
668 Infix,
669 Literal,
670 };
671
672 pub const Infix = struct {
673 base: Expr = Expr{ .id = .Infix },
674 lhs: *Expr,
675 op_token: TokenIndex,
676 op: Op,
677 rhs: *Expr,
678
679 pub const Op = enum {};
680 };
681};
lib/std/c/parse.zig created+1431
...@@ -0,0 +1,1431 @@
1const std = @import("std");
2const mem = std.mem;
3const assert = std.debug.assert;
4const Allocator = std.mem.Allocator;
5const ast = std.c.ast;
6const Node = ast.Node;
7const Type = ast.Type;
8const Tree = ast.Tree;
9const TokenIndex = ast.TokenIndex;
10const Token = std.c.Token;
11const TokenIterator = ast.Tree.TokenList.Iterator;
12
13pub const Error = error{ParseError} || Allocator.Error;
14
15pub const Options = struct {
16 // /// Keep simple macros unexpanded and add the definitions to the ast
17 // retain_macros: bool = false,
18 /// Warning or error
19 warn_as_err: union(enum) {
20 /// All warnings are warnings
21 None,
22
23 /// Some warnings are errors
24 Some: []@TagType(ast.Error),
25
26 /// All warnings are errors
27 All,
28 } = .All,
29};
30
31/// Result should be freed with tree.deinit() when there are
32/// no more references to any of the tokens or nodes.
33pub fn parse(allocator: *Allocator, source: []const u8, options: Options) !*Tree {
34 const tree = blk: {
35 // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists
36 // from being initialized with a pointer to this `arena`, which is created on
37 // the stack. Following code should instead refer to `&tree.arena_allocator`, a
38 // pointer to data which lives safely on the heap and will outlive `parse`.
39 var arena = std.heap.ArenaAllocator.init(allocator);
40 errdefer arena.deinit();
41 const tree = try arena.allocator.create(ast.Tree);
42 tree.* = .{
43 .root_node = undefined,
44 .arena_allocator = arena,
45 .tokens = undefined,
46 .sources = undefined,
47 };
48 break :blk tree;
49 };
50 errdefer tree.deinit();
51 const arena = &tree.arena_allocator.allocator;
52
53 tree.tokens = ast.Tree.TokenList.init(arena);
54 tree.sources = ast.Tree.SourceList.init(arena);
55
56 var tokenizer = std.zig.Tokenizer.init(source);
57 while (true) {
58 const tree_token = try tree.tokens.addOne();
59 tree_token.* = tokenizer.next();
60 if (tree_token.id == .Eof) break;
61 }
62 // TODO preprocess here
63 var it = tree.tokens.iterator(0);
64
65 while (true) {
66 const tok = it.peek().?.id;
67 switch (id) {
68 .LineComment,
69 .MultiLineComment,
70 => {
71 _ = it.next();
72 },
73 else => break,
74 }
75 }
76
77 var parse_arena = std.heap.ArenaAllocator.init(allocator);
78 defer parse_arena.deinit();
79
80 var parser = Parser{
81 .scopes = Parser.SymbolList.init(allocator),
82 .arena = &parse_arena.allocator,
83 .it = &it,
84 .tree = tree,
85 .options = options,
86 };
87 defer parser.symbols.deinit();
88
89 tree.root_node = try parser.root();
90 return tree;
91}
92
93const Parser = struct {
94 arena: *Allocator,
95 it: *TokenIterator,
96 tree: *Tree,
97
98 arena: *Allocator,
99 scopes: ScopeList,
100 options: Options,
101
102 const ScopeList = std.SegmentedLists(Scope);
103 const SymbolList = std.SegmentedLists(Symbol);
104
105 const Scope = struct {
106 kind: ScopeKind,
107 syms: SymbolList,
108 };
109
110 const Symbol = struct {
111 name: []const u8,
112 ty: *Type,
113 };
114
115 const ScopeKind = enum {
116 Block,
117 Loop,
118 Root,
119 Switch,
120 };
121
122 fn pushScope(parser: *Parser, kind: ScopeKind) !void {
123 const new = try parser.scopes.addOne();
124 new.* = .{
125 .kind = kind,
126 .syms = SymbolList.init(parser.arena),
127 };
128 }
129
130 fn popScope(parser: *Parser, len: usize) void {
131 _ = parser.scopes.pop();
132 }
133
134 fn getSymbol(parser: *Parser, tok: TokenIndex) ?*Symbol {
135 const name = parser.tree.tokenSlice(tok);
136 var scope_it = parser.scopes.iterator(parser.scopes.len);
137 while (scope_it.prev()) |scope| {
138 var sym_it = scope.syms.iterator(scope.syms.len);
139 while (sym_it.prev()) |sym| {
140 if (mem.eql(u8, sym.name, name)) {
141 return sym;
142 }
143 }
144 }
145 return null;
146 }
147
148 fn declareSymbol(parser: *Parser, type_spec: Node.TypeSpec, dr: *Node.Declarator) Error!void {
149 return; // TODO
150 }
151
152 /// Root <- ExternalDeclaration* eof
153 fn root(parser: *Parser) Allocator.Error!*Node.Root {
154 try parser.pushScope(.Root);
155 defer parser.popScope();
156 const node = try parser.arena.create(Node.Root);
157 node.* = .{
158 .decls = Node.Root.DeclList.init(parser.arena),
159 .eof = undefined,
160 };
161 while (parser.externalDeclarations() catch |e| switch (e) {
162 error.OutOfMemory => return error.OutOfMemory,
163 error.ParseError => return node,
164 }) |decl| {
165 try node.decls.push(decl);
166 }
167 node.eof = parser.eatToken(.Eof) orelse return node;
168 return node;
169 }
170
171 /// ExternalDeclaration
172 /// <- DeclSpec Declarator OldStyleDecl* CompoundStmt
173 /// / Declaration
174 /// OldStyleDecl <- DeclSpec Declarator (COMMA Declarator)* SEMICOLON
175 fn externalDeclarations(parser: *Parser) !?*Node {
176 return parser.declarationExtra(false);
177 }
178
179 /// Declaration
180 /// <- DeclSpec DeclInit SEMICOLON
181 /// / StaticAssert
182 /// DeclInit <- Declarator (EQUAL Initializer)? (COMMA Declarator (EQUAL Initializer)?)*
183 fn declaration(parser: *Parser) !?*Node {
184 return parser.declarationExtra(true);
185 }
186
187 fn declarationExtra(parser: *Parser, local: bool) !?*Node {
188 if (try parser.staticAssert()) |decl| return decl;
189 const begin = parser.it.index + 1;
190 var ds = Node.DeclSpec{};
191 const got_ds = try parser.declSpec(&ds);
192 if (local and !got_ds) {
193 // not a declaration
194 return null;
195 }
196 switch (ds.storage_class) {
197 .Auto, .Register => |tok| return parser.err(.{
198 .InvalidStorageClass = .{ .token = tok },
199 }),
200 .Typedef => {
201 const node = try parser.arena.create(Node.Typedef);
202 node.* = .{
203 .decl_spec = ds,
204 .declarators = Node.Typedef.DeclaratorList.init(parser.arena),
205 .semicolon = undefined,
206 };
207 while (true) {
208 const dr = @fieldParentPtr(Node.Declarator, "base", (try parser.declarator(.Must)) orelse return parser.err(.{
209 .ExpectedDeclarator = .{ .token = parser.it.index },
210 }));
211 try parser.declareSymbol(ds.type_spec, dr);
212 try node.declarators.push(&dr.base);
213 if (parser.eatToken(.Comma)) |_| {} else break;
214 }
215 return &node.base;
216 },
217 else => {},
218 }
219 var first_dr = try parser.declarator(.Must);
220 if (first_dr != null and declaratorIsFunction(first_dr.?)) {
221 // TODO typedeffed fn proto-only
222 const dr = @fieldParentPtr(Node.Declarator, "base", first_dr.?);
223 try parser.declareSymbol(ds.type_spec, dr);
224 var old_decls = Node.FnDecl.OldDeclList.init(parser.arena);
225 const body = if (parser.eatToken(.Semicolon)) |_|
226 null
227 else blk: {
228 if (local) {
229 // TODO nested function warning
230 }
231 // TODO first_dr.is_old
232 // while (true) {
233 // var old_ds = Node.DeclSpec{};
234 // if (!(try parser.declSpec(&old_ds))) {
235 // // not old decl
236 // break;
237 // }
238 // var old_dr = (try parser.declarator(.Must));
239 // // if (old_dr == null)
240 // // try parser.err(.{
241 // // .NoParamName = .{ .token = parser.it.index },
242 // // });
243 // // try old_decls.push(decl);
244 // }
245 const body_node = (try parser.compoundStmt()) orelse return parser.err(.{
246 .ExpectedFnBody = .{ .token = parser.it.index },
247 });
248 break :blk @fieldParentPtr(Node.CompoundStmt, "base", body_node);
249 };
250
251 const node = try parser.arena.create(Node.FnDecl);
252 node.* = .{
253 .decl_spec = ds,
254 .declarator = dr,
255 .old_decls = old_decls,
256 .body = body,
257 };
258 return &node.base;
259 } else {
260 switch (ds.fn_spec) {
261 .Inline, .Noreturn => |tok| return parser.err(.{
262 .FnSpecOnNonFn = .{ .token = tok },
263 }),
264 else => {},
265 }
266 // TODO threadlocal without static or extern on local variable
267 const node = try parser.arena.create(Node.VarDecl);
268 node.* = .{
269 .decl_spec = ds,
270 .initializers = Node.VarDecl.Initializers.init(parser.arena),
271 .semicolon = undefined,
272 };
273 if (first_dr == null) {
274 node.semicolon = try parser.expectToken(.Semicolon);
275 const ok = switch (ds.type_spec.spec) {
276 .Enum => |e| e.name != null,
277 .Record => |r| r.name != null,
278 else => false,
279 };
280 const q = ds.type_spec.qual;
281 if (!ok)
282 try parser.warn(.{
283 .NothingDeclared = .{ .token = begin },
284 })
285 else if (q.@"const" orelse q.atomic orelse q.@"volatile" orelse q.restrict) |tok|
286 try parser.warn(.{
287 .QualifierIgnored = .{ .token = tok },
288 });
289 return &node.base;
290 }
291 var dr = @fieldParentPtr(Node.Declarator, "base", first_dr.?);
292 while (true) {
293 try parser.declareSymbol(ds.type_spec, dr);
294 if (parser.eatToken(.Equal)) |tok| {
295 try node.initializers.push((try parser.initializer(dr)) orelse return parser.err(.{
296 .ExpectedInitializer = .{ .token = parser.it.index },
297 }));
298 } else
299 try node.initializers.push(&dr.base);
300 if (parser.eatToken(.Comma) != null) break;
301 dr = @fieldParentPtr(Node.Declarator, "base", (try parser.declarator(.Must)) orelse return parser.err(.{
302 .ExpectedDeclarator = .{ .token = parser.it.index },
303 }));
304 }
305 node.semicolon = try parser.expectToken(.Semicolon);
306 return &node.base;
307 }
308 }
309
310 fn declaratorIsFunction(node: *Node) bool {
311 if (node.id != .Declarator) return false;
312 assert(node.id == .Declarator);
313 const dr = @fieldParentPtr(Node.Declarator, "base", node);
314 if (dr.suffix != .Fn) return false;
315 switch (dr.prefix) {
316 .None, .Identifer => return true,
317 .Complex => |inner| {
318 var inner_node = inner.inner;
319 while (true) {
320 if (inner_node.id != .Declarator) return false;
321 assert(inner_node.id == .Declarator);
322 const inner_dr = @fieldParentPtr(Node.Declarator, "base", inner_node);
323 if (inner_dr.pointer != null) return false;
324 switch (inner_dr.prefix) {
325 .None, .Identifer => return true,
326 .Complex => |c| inner_node = c.inner,
327 }
328 }
329 },
330 }
331 }
332
333 /// StaticAssert <- Keyword_static_assert LPAREN ConstExpr COMMA STRINGLITERAL RPAREN SEMICOLON
334 fn staticAssert(parser: *Parser) !?*Node {
335 const tok = parser.eatToken(.Keyword_static_assert) orelse return null;
336 _ = try parser.expectToken(.LParen);
337 const const_expr = (try parser.constExpr()) orelse parser.err(.{
338 .ExpectedExpr = .{ .token = parser.it.index },
339 });
340 _ = try parser.expectToken(.Comma);
341 const str = try parser.expectToken(.StringLiteral);
342 _ = try parser.expectToken(.RParen);
343 const node = try parser.arena.create(Node.StaticAssert);
344 node.* = .{
345 .assert = tok,
346 .expr = const_expr,
347 .semicolon = try parser.expectToken(.Semicolon),
348 };
349 return &node.base;
350 }
351
352 /// DeclSpec <- (StorageClassSpec / TypeSpec / FnSpec / AlignSpec)*
353 /// returns true if any tokens were consumed
354 fn declSpec(parser: *Parser, ds: *Node.DeclSpec) !bool {
355 var got = false;
356 while ((try parser.storageClassSpec(ds)) or (try parser.typeSpec(&ds.type_spec)) or (try parser.fnSpec(ds)) or (try parser.alignSpec(ds))) {
357 got = true;
358 }
359 return got;
360 }
361
362 /// StorageClassSpec
363 /// <- Keyword_typedef / Keyword_extern / Keyword_static / Keyword_thread_local / Keyword_auto / Keyword_register
364 fn storageClassSpec(parser: *Parser, ds: *Node.DeclSpec) !bool {
365 blk: {
366 if (parser.eatToken(.Keyword_typedef)) |tok| {
367 if (ds.storage_class != .None or ds.thread_local != null)
368 break :blk;
369 ds.storage_class = .{ .Typedef = tok };
370 } else if (parser.eatToken(.Keyword_extern)) |tok| {
371 if (ds.storage_class != .None)
372 break :blk;
373 ds.storage_class = .{ .Extern = tok };
374 } else if (parser.eatToken(.Keyword_static)) |tok| {
375 if (ds.storage_class != .None)
376 break :blk;
377 ds.storage_class = .{ .Static = tok };
378 } else if (parser.eatToken(.Keyword_thread_local)) |tok| {
379 switch (ds.storage_class) {
380 .None, .Extern, .Static => {},
381 else => break :blk,
382 }
383 ds.thread_local = tok;
384 } else if (parser.eatToken(.Keyword_auto)) |tok| {
385 if (ds.storage_class != .None or ds.thread_local != null)
386 break :blk;
387 ds.storage_class = .{ .Auto = tok };
388 } else if (parser.eatToken(.Keyword_register)) |tok| {
389 if (ds.storage_class != .None or ds.thread_local != null)
390 break :blk;
391 ds.storage_class = .{ .Register = tok };
392 } else return false;
393 return true;
394 }
395 try parser.warn(.{
396 .DuplicateSpecifier = .{ .token = parser.it.index },
397 });
398 return true;
399 }
400
401 /// TypeSpec
402 /// <- Keyword_void / Keyword_char / Keyword_short / Keyword_int / Keyword_long / Keyword_float / Keyword_double
403 /// / Keyword_signed / Keyword_unsigned / Keyword_bool / Keyword_complex / Keyword_imaginary /
404 /// / Keyword_atomic LPAREN TypeName RPAREN
405 /// / EnumSpec
406 /// / RecordSpec
407 /// / IDENTIFIER // typedef name
408 /// / TypeQual
409 fn typeSpec(parser: *Parser, type_spec: *Node.TypeSpec) !bool {
410 blk: {
411 if (parser.eatToken(.Keyword_void)) |tok| {
412 if (type_spec.spec != .None)
413 break :blk;
414 type_spec.spec = .{ .Void = tok };
415 } else if (parser.eatToken(.Keyword_char)) |tok| {
416 switch (type_spec.spec) {
417 .None => {
418 type_spec.spec = .{
419 .Char = .{
420 .char = tok,
421 },
422 };
423 },
424 .Int => |int| {
425 if (int.int != null)
426 break :blk;
427 type_spec.spec = .{
428 .Char = .{
429 .char = tok,
430 .sign = int.sign,
431 },
432 };
433 },
434 else => break :blk,
435 }
436 } else if (parser.eatToken(.Keyword_short)) |tok| {
437 switch (type_spec.spec) {
438 .None => {
439 type_spec.spec = .{
440 .Short = .{
441 .short = tok,
442 },
443 };
444 },
445 .Int => |int| {
446 if (int.int != null)
447 break :blk;
448 type_spec.spec = .{
449 .Short = .{
450 .short = tok,
451 .sign = int.sign,
452 },
453 };
454 },
455 else => break :blk,
456 }
457 } else if (parser.eatToken(.Keyword_long)) |tok| {
458 switch (type_spec.spec) {
459 .None => {
460 type_spec.spec = .{
461 .Long = .{
462 .long = tok,
463 },
464 };
465 },
466 .Int => |int| {
467 type_spec.spec = .{
468 .Long = .{
469 .long = tok,
470 .sign = int.sign,
471 .int = int.int,
472 },
473 };
474 },
475 .Long => |*long| {
476 if (long.longlong != null)
477 break :blk;
478 long.longlong = tok;
479 },
480 .Double => |*double| {
481 if (double.long != null)
482 break :blk;
483 double.long = tok;
484 },
485 else => break :blk,
486 }
487 } else if (parser.eatToken(.Keyword_int)) |tok| {
488 switch (type_spec.spec) {
489 .None => {
490 type_spec.spec = .{
491 .Int = .{
492 .int = tok,
493 },
494 };
495 },
496 .Short => |*short| {
497 if (short.int != null)
498 break :blk;
499 short.int = tok;
500 },
501 .Int => |*int| {
502 if (int.int != null)
503 break :blk;
504 int.int = tok;
505 },
506 .Long => |*long| {
507 if (long.int != null)
508 break :blk;
509 long.int = tok;
510 },
511 else => break :blk,
512 }
513 } else if (parser.eatToken(.Keyword_signed) orelse parser.eatToken(.Keyword_unsigned)) |tok| {
514 switch (type_spec.spec) {
515 .None => {
516 type_spec.spec = .{
517 .Int = .{
518 .sign = tok,
519 },
520 };
521 },
522 .Char => |*char| {
523 if (char.sign != null)
524 break :blk;
525 char.sign = tok;
526 },
527 .Short => |*short| {
528 if (short.sign != null)
529 break :blk;
530 short.sign = tok;
531 },
532 .Int => |*int| {
533 if (int.sign != null)
534 break :blk;
535 int.sign = tok;
536 },
537 .Long => |*long| {
538 if (long.sign != null)
539 break :blk;
540 long.sign = tok;
541 },
542 else => break :blk,
543 }
544 } else if (parser.eatToken(.Keyword_float)) |tok| {
545 if (type_spec.spec != .None)
546 break :blk;
547 type_spec.spec = .{
548 .Float = .{
549 .float = tok,
550 },
551 };
552 } else if (parser.eatToken(.Keyword_double)) |tok| {
553 if (type_spec.spec != .None)
554 break :blk;
555 type_spec.spec = .{
556 .Double = .{
557 .double = tok,
558 },
559 };
560 } else if (parser.eatToken(.Keyword_complex)) |tok| {
561 switch (type_spec.spec) {
562 .None => {
563 type_spec.spec = .{
564 .Double = .{
565 .complex = tok,
566 .double = null,
567 },
568 };
569 },
570 .Float => |*float| {
571 if (float.complex != null)
572 break :blk;
573 float.complex = tok;
574 },
575 .Double => |*double| {
576 if (double.complex != null)
577 break :blk;
578 double.complex = tok;
579 },
580 else => break :blk,
581 }
582 } else if (parser.eatToken(.Keyword_bool)) |tok| {
583 if (type_spec.spec != .None)
584 break :blk;
585 type_spec.spec = .{ .Bool = tok };
586 } else if (parser.eatToken(.Keyword_atomic)) |tok| {
587 // might be _Atomic qualifier
588 if (parser.eatToken(.LParen)) |_| {
589 if (type_spec.spec != .None)
590 break :blk;
591 const name = (try parser.typeName()) orelse return parser.err(.{
592 .ExpectedTypeName = .{ .token = parser.it.index },
593 });
594 type_spec.spec.Atomic = .{
595 .atomic = tok,
596 .typename = name,
597 .rparen = try parser.expectToken(.RParen),
598 };
599 } else {
600 parser.putBackToken(tok);
601 }
602 } else if (parser.eatToken(.Keyword_enum)) |tok| {
603 if (type_spec.spec != .None)
604 break :blk;
605 type_spec.spec.Enum = try parser.enumSpec(tok);
606 } else if (parser.eatToken(.Keyword_union) orelse parser.eatToken(.Keyword_struct)) |tok| {
607 if (type_spec.spec != .None)
608 break :blk;
609 type_spec.spec.Record = try parser.recordSpec(tok);
610 } else if (parser.eatToken(.Identifier)) |tok| {
611 const ty = parser.getSymbol(tok) orelse {
612 parser.putBackToken(tok);
613 return false;
614 };
615 switch (ty.id) {
616 .Enum => |e| blk: {
617 if (e.name) |some|
618 if (!parser.tree.tokenEql(some, tok))
619 break :blk;
620 return parser.err(.{
621 .MustUseKwToRefer = .{ .kw = e.tok, .name = tok },
622 });
623 },
624 .Record => |r| blk: {
625 if (r.name) |some|
626 if (!parser.tree.tokenEql(some, tok))
627 break :blk;
628 return parser.err(.{
629 .MustUseKwToRefer = .{
630 .kw = r.tok,
631 .name = tok,
632 },
633 });
634 },
635 .Typedef => {
636 type_spec.spec = .{
637 .Typedef = .{
638 .sym = tok,
639 .sym_type = ty,
640 },
641 };
642 return true;
643 },
644 else => {},
645 }
646 parser.putBackToken(tok);
647 return false;
648 }
649 return parser.typeQual(&type_spec.qual);
650 }
651 return parser.err(.{
652 .InvalidTypeSpecifier = .{
653 .token = parser.it.index,
654 .type_spec = type_spec,
655 },
656 });
657 }
658
659 /// TypeQual <- Keyword_const / Keyword_restrict / Keyword_volatile / Keyword_atomic
660 fn typeQual(parser: *Parser, qual: *Node.TypeQual) !bool {
661 blk: {
662 if (parser.eatToken(.Keyword_const)) |tok| {
663 if (qual.@"const" != null)
664 break :blk;
665 qual.@"const" = tok;
666 } else if (parser.eatToken(.Keyword_restrict)) |tok| {
667 if (qual.atomic != null)
668 break :blk;
669 qual.atomic = tok;
670 } else if (parser.eatToken(.Keyword_volatile)) |tok| {
671 if (qual.@"volatile" != null)
672 break :blk;
673 qual.@"volatile" = tok;
674 } else if (parser.eatToken(.Keyword_atomic)) |tok| {
675 if (qual.atomic != null)
676 break :blk;
677 qual.atomic = tok;
678 } else return false;
679 return true;
680 }
681 try parser.warn(.{
682 .DuplicateQualifier = .{ .token = parser.it.index },
683 });
684 return true;
685 }
686
687 /// FnSpec <- Keyword_inline / Keyword_noreturn
688 fn fnSpec(parser: *Parser, ds: *Node.DeclSpec) !bool {
689 blk: {
690 if (parser.eatToken(.Keyword_inline)) |tok| {
691 if (ds.fn_spec != .None)
692 break :blk;
693 ds.fn_spec = .{ .Inline = tok };
694 } else if (parser.eatToken(.Keyword_noreturn)) |tok| {
695 if (ds.fn_spec != .None)
696 break :blk;
697 ds.fn_spec = .{ .Noreturn = tok };
698 } else return false;
699 return true;
700 }
701 try parser.warn(.{
702 .DuplicateSpecifier = .{ .token = parser.it.index },
703 });
704 return true;
705 }
706
707 /// AlignSpec <- Keyword_alignas LPAREN (TypeName / ConstExpr) RPAREN
708 fn alignSpec(parser: *Parser, ds: *Node.DeclSpec) !bool {
709 if (parser.eatToken(.Keyword_alignas)) |tok| {
710 _ = try parser.expectToken(.LParen);
711 const node = (try parser.typeName()) orelse (try parser.constExpr()) orelse parser.err(.{
712 .ExpectedExpr = .{ .token = parser.it.index },
713 });
714 if (ds.align_spec != null) {
715 try parser.warn(.{
716 .DuplicateSpecifier = .{ .token = parser.it.index },
717 });
718 }
719 ds.align_spec = .{
720 .alignas = tok,
721 .expr = node,
722 .rparen = try parser.expectToken(.RParen),
723 };
724 return true;
725 }
726 return false;
727 }
728
729 /// EnumSpec <- Keyword_enum IDENTIFIER? (LBRACE EnumField RBRACE)?
730 fn enumSpec(parser: *Parser, tok: TokenIndex) !*Node.EnumType {
731 const node = try parser.arena.create(Node.EnumType);
732 const name = parser.eatToken(.Identifier);
733 node.* = .{
734 .tok = tok,
735 .name = name,
736 .body = null,
737 };
738 const ty = try parser.arena.create(Type);
739 ty.* = .{
740 .id = .{
741 .Enum = node,
742 },
743 };
744 if (name) |some|
745 try parser.symbols.append(.{
746 .name = parser.tree.tokenSlice(some),
747 .ty = ty,
748 });
749 if (parser.eatToken(.LBrace)) |lbrace| {
750 var fields = Node.EnumType.FieldList.init(parser.arena);
751 try fields.push((try parser.enumField()) orelse return parser.err(.{
752 .ExpectedEnumField = .{ .token = parser.it.index },
753 }));
754 while (parser.eatToken(.Comma)) |_| {
755 try fields.push((try parser.enumField()) orelse break);
756 }
757 node.body = .{
758 .lbrace = lbrace,
759 .fields = fields,
760 .rbrace = try parser.expectToken(.RBrace),
761 };
762 }
763 return node;
764 }
765
766 /// EnumField <- IDENTIFIER (EQUAL ConstExpr)? (COMMA EnumField) COMMA?
767 fn enumField(parser: *Parser) !?*Node {
768 const name = parser.eatToken(.Identifier) orelse return null;
769 const node = try parser.arena.create(Node.EnumField);
770 node.* = .{
771 .name = name,
772 .value = null,
773 };
774 if (parser.eatToken(.Equal)) |eq| {
775 node.value = (try parser.constExpr()) orelse parser.err(.{
776 .ExpectedExpr = .{ .token = parser.it.index },
777 });
778 }
779 return &node.base;
780 }
781
782 /// RecordSpec <- (Keyword_struct / Keyword_union) IDENTIFIER? (LBRACE RecordField+ RBRACE)?
783 fn recordSpec(parser: *Parser, tok: TokenIndex) !*Node.RecordType {
784 const node = try parser.arena.create(Node.RecordType);
785 const name = parser.eatToken(.Identifier);
786 const is_struct = parser.tree.tokenSlice(tok)[0] == 's';
787 node.* = .{
788 .tok = tok,
789 .kind = if (is_struct) .Struct else .Union,
790 .name = name,
791 .body = null,
792 };
793 const ty = try parser.arena.create(Type);
794 ty.* = .{
795 .id = .{
796 .Record = node,
797 },
798 };
799 if (name) |some|
800 try parser.symbols.append(.{
801 .name = parser.tree.tokenSlice(some),
802 .ty = ty,
803 });
804 if (parser.eatToken(.LBrace)) |lbrace| {
805 try parser.pushScope(.Block);
806 defer parser.popScope();
807 var fields = Node.RecordType.FieldList.init(parser.arena);
808 while (true) {
809 if (parser.eatToken(.RBrace)) |rbrace| {
810 node.body = .{
811 .lbrace = lbrace,
812 .fields = fields,
813 .rbrace = rbrace,
814 };
815 break;
816 }
817 try fields.push(try parser.recordField());
818 }
819 }
820 return node;
821 }
822
823 /// RecordField
824 /// <- TypeSpec* (RecordDeclarator (COMMA RecordDeclarator))? SEMICOLON
825 /// \ StaticAssert
826 fn recordField(parser: *Parser) Error!*Node {
827 if (try parser.staticAssert()) |decl| return decl;
828 var got = false;
829 var type_spec = Node.TypeSpec{};
830 while (try parser.typeSpec(&type_spec)) got = true;
831 if (!got)
832 return parser.err(.{
833 .ExpectedType = .{ .token = parser.it.index },
834 });
835 const node = try parser.arena.create(Node.RecordField);
836 node.* = .{
837 .type_spec = type_spec,
838 .declarators = Node.RecordField.DeclaratorList.init(parser.arena),
839 .semicolon = undefined,
840 };
841 while (true) {
842 const rdr = try parser.recordDeclarator();
843 try parser.declareSymbol(type_spec, rdr.declarator);
844 try node.declarators.push(&rdr.base);
845 if (parser.eatToken(.Comma)) |_| {} else break;
846 }
847
848 node.semicolon = try parser.expectToken(.Semicolon);
849 return &node.base;
850 }
851
852 /// TypeName <- TypeSpec* AbstractDeclarator?
853 fn typeName(parser: *Parser) Error!?*Node {
854 @panic("TODO");
855 }
856
857 /// RecordDeclarator <- Declarator? (COLON ConstExpr)?
858 fn recordDeclarator(parser: *Parser) Error!*Node.RecordDeclarator {
859 @panic("TODO");
860 }
861
862 /// Pointer <- ASTERISK TypeQual* Pointer?
863 fn pointer(parser: *Parser) Error!?*Node.Pointer {
864 const asterisk = parser.eatToken(.Asterisk) orelse return null;
865 const node = try parser.arena.create(Node.Pointer);
866 node.* = .{
867 .asterisk = asterisk,
868 .qual = .{},
869 .pointer = null,
870 };
871 while (try parser.typeQual(&node.qual)) {}
872 node.pointer = try parser.pointer();
873 return node;
874 }
875
876 const Named = enum {
877 Must,
878 Allowed,
879 Forbidden,
880 };
881
882 /// Declarator <- Pointer? DeclaratorSuffix
883 /// DeclaratorPrefix
884 /// <- IDENTIFIER // if named != .Forbidden
885 /// / LPAREN Declarator RPAREN
886 /// / (none) // if named != .Must
887 /// DeclaratorSuffix
888 /// <- DeclaratorPrefix (LBRACKET ArrayDeclarator? RBRACKET)*
889 /// / DeclaratorPrefix LPAREN (ParamDecl (COMMA ParamDecl)* (COMMA ELLIPSIS)?)? RPAREN
890 fn declarator(parser: *Parser, named: Named) Error!?*Node {
891 const ptr = try parser.pointer();
892 var node: *Node.Declarator = undefined;
893 var inner_fn = false;
894
895 // TODO sizof(int (int))
896 // prefix
897 if (parser.eatToken(.LParen)) |lparen| {
898 const inner = (try parser.declarator(named)) orelse return parser.err(.{
899 .ExpectedDeclarator = .{ .token = lparen + 1 },
900 });
901 inner_fn = declaratorIsFunction(inner);
902 node = try parser.arena.create(Node.Declarator);
903 node.* = .{
904 .pointer = ptr,
905 .prefix = .{
906 .Complex = .{
907 .lparen = lparen,
908 .inner = inner,
909 .rparen = try parser.expectToken(.RParen),
910 },
911 },
912 .suffix = .None,
913 };
914 } else if (named != .Forbidden) {
915 if (parser.eatToken(.Identifier)) |tok| {
916 node = try parser.arena.create(Node.Declarator);
917 node.* = .{
918 .pointer = ptr,
919 .prefix = .{ .Identifer = tok },
920 .suffix = .None,
921 };
922 } else if (named == .Must) {
923 return parser.err(.{
924 .ExpectedToken = .{ .token = parser.it.index, .expected_id = .Identifier },
925 });
926 } else {
927 if (ptr) |some|
928 return &some.base;
929 return null;
930 }
931 } else {
932 node = try parser.arena.create(Node.Declarator);
933 node.* = .{
934 .pointer = ptr,
935 .prefix = .None,
936 .suffix = .None,
937 };
938 }
939 // suffix
940 if (parser.eatToken(.LParen)) |lparen| {
941 if (inner_fn)
942 return parser.err(.{
943 .InvalidDeclarator = .{ .token = lparen },
944 });
945 node.suffix = .{
946 .Fn = .{
947 .lparen = lparen,
948 .params = Node.Declarator.Params.init(parser.arena),
949 .rparen = undefined,
950 },
951 };
952 try parser.paramDecl(node);
953 node.suffix.Fn.rparen = try parser.expectToken(.RParen);
954 } else if (parser.eatToken(.LBracket)) |tok| {
955 if (inner_fn)
956 return parser.err(.{
957 .InvalidDeclarator = .{ .token = tok },
958 });
959 node.suffix = .{ .Array = Node.Declarator.Arrays.init(parser.arena) };
960 var lbrace = tok;
961 while (true) {
962 try node.suffix.Array.push(try parser.arrayDeclarator(lbrace));
963 if (parser.eatToken(.LBracket)) |t| lbrace = t else break;
964 }
965 }
966 if (parser.eatToken(.LParen) orelse parser.eatToken(.LBracket)) |tok|
967 return parser.err(.{
968 .InvalidDeclarator = .{ .token = tok },
969 });
970 return &node.base;
971 }
972
973 /// ArrayDeclarator
974 /// <- ASTERISK
975 /// / Keyword_static TypeQual* AssignmentExpr
976 /// / TypeQual+ (ASTERISK / Keyword_static AssignmentExpr)
977 /// / TypeQual+ AssignmentExpr?
978 /// / AssignmentExpr
979 fn arrayDeclarator(parser: *Parser, lbracket: TokenIndex) !*Node.Array {
980 const arr = try parser.arena.create(Node.Array);
981 arr.* = .{
982 .lbracket = lbracket,
983 .inner = .Inferred,
984 .rbracket = undefined,
985 };
986 if (parser.eatToken(.Asterisk)) |tok| {
987 arr.inner = .{ .Unspecified = tok };
988 } else {
989 // TODO
990 }
991 arr.rbracket = try parser.expectToken(.RBracket);
992 return arr;
993 }
994
995 /// Params <- ParamDecl (COMMA ParamDecl)* (COMMA ELLIPSIS)?
996 /// ParamDecl <- DeclSpec (Declarator / AbstractDeclarator)
997 fn paramDecl(parser: *Parser, dr: *Node.Declarator) !void {
998 var old_style = false;
999 while (true) {
1000 var ds = Node.DeclSpec{};
1001 if (try parser.declSpec(&ds)) {
1002 //TODO
1003 // TODO try parser.declareSymbol(ds.type_spec, dr);
1004 } else if (parser.eatToken(.Identifier)) |tok| {
1005 old_style = true;
1006 } else if (parser.eatToken(.Ellipsis)) |tok| {
1007 // TODO
1008 }
1009 }
1010 }
1011
1012 /// Expr <- AssignmentExpr (COMMA Expr)*
1013 fn expr(parser: *Parser) Error!?*Expr {
1014 @panic("TODO");
1015 }
1016
1017 /// AssignmentExpr
1018 /// <- ConditionalExpr // TODO recursive?
1019 /// / UnaryExpr (EQUAL / ASTERISKEQUAL / SLASHEQUAL / PERCENTEQUAL / PLUSEQUAL / MINUSEQUA /
1020 /// / ANGLEBRACKETANGLEBRACKETLEFTEQUAL / ANGLEBRACKETANGLEBRACKETRIGHTEQUAL /
1021 /// / AMPERSANDEQUAL / CARETEQUAL / PIPEEQUAL) AssignmentExpr
1022 fn assignmentExpr(parser: *Parser) !?*Expr {
1023 @panic("TODO");
1024 }
1025
1026 /// ConstExpr <- ConditionalExpr
1027 fn constExpr(parser: *Parser) Error!?*Expr {
1028 const start = parser.it.index;
1029 const expression = try parser.conditionalExpr();
1030 if (expression != null and expression.?.value == .None)
1031 return parser.err(.{
1032 .ConsExpr = start,
1033 });
1034 return expression;
1035 }
1036
1037 /// ConditionalExpr <- LogicalOrExpr (QUESTIONMARK Expr COLON ConditionalExpr)?
1038 fn conditionalExpr(parser: *Parser) Error!?*Expr {
1039 @panic("TODO");
1040 }
1041
1042 /// LogicalOrExpr <- LogicalAndExpr (PIPEPIPE LogicalOrExpr)*
1043 fn logicalOrExpr(parser: *Parser) !*Node {
1044 const lhs = (try parser.logicalAndExpr()) orelse return null;
1045 }
1046
1047 /// LogicalAndExpr <- BinOrExpr (AMPERSANDAMPERSAND LogicalAndExpr)*
1048 fn logicalAndExpr(parser: *Parser) !*Node {
1049 @panic("TODO");
1050 }
1051
1052 /// BinOrExpr <- BinXorExpr (PIPE BinOrExpr)*
1053 fn binOrExpr(parser: *Parser) !*Node {
1054 @panic("TODO");
1055 }
1056
1057 /// BinXorExpr <- BinAndExpr (CARET BinXorExpr)*
1058 fn binXorExpr(parser: *Parser) !*Node {
1059 @panic("TODO");
1060 }
1061
1062 /// BinAndExpr <- EqualityExpr (AMPERSAND BinAndExpr)*
1063 fn binAndExpr(parser: *Parser) !*Node {
1064 @panic("TODO");
1065 }
1066
1067 /// EqualityExpr <- ComparisionExpr ((EQUALEQUAL / BANGEQUAL) EqualityExpr)*
1068 fn equalityExpr(parser: *Parser) !*Node {
1069 @panic("TODO");
1070 }
1071
1072 /// ComparisionExpr <- ShiftExpr (ANGLEBRACKETLEFT / ANGLEBRACKETLEFTEQUAL /ANGLEBRACKETRIGHT / ANGLEBRACKETRIGHTEQUAL) ComparisionExpr)*
1073 fn comparisionExpr(parser: *Parser) !*Node {
1074 @panic("TODO");
1075 }
1076
1077 /// ShiftExpr <- AdditiveExpr (ANGLEBRACKETANGLEBRACKETLEFT / ANGLEBRACKETANGLEBRACKETRIGHT) ShiftExpr)*
1078 fn shiftExpr(parser: *Parser) !*Node {
1079 @panic("TODO");
1080 }
1081
1082 /// AdditiveExpr <- MultiplicativeExpr (PLUS / MINUS) AdditiveExpr)*
1083 fn additiveExpr(parser: *Parser) !*Node {
1084 @panic("TODO");
1085 }
1086
1087 /// MultiplicativeExpr <- UnaryExpr (ASTERISK / SLASH / PERCENT) MultiplicativeExpr)*
1088 fn multiplicativeExpr(parser: *Parser) !*Node {
1089 @panic("TODO");
1090 }
1091
1092 /// UnaryExpr
1093 /// <- LPAREN TypeName RPAREN UnaryExpr
1094 /// / Keyword_sizeof LAPERN TypeName RPAREN
1095 /// / Keyword_sizeof UnaryExpr
1096 /// / Keyword_alignof LAPERN TypeName RPAREN
1097 /// / (AMPERSAND / ASTERISK / PLUS / PLUSPLUS / MINUS / MINUSMINUS / TILDE / BANG) UnaryExpr
1098 /// / PrimaryExpr PostFixExpr*
1099 fn unaryExpr(parser: *Parser) !*Node {
1100 @panic("TODO");
1101 }
1102
1103 /// PrimaryExpr
1104 /// <- IDENTIFIER
1105 /// / INTEGERLITERAL / FLOATLITERAL / STRINGLITERAL / CHARLITERAL
1106 /// / LPAREN Expr RPAREN
1107 /// / Keyword_generic LPAREN AssignmentExpr (COMMA Generic)+ RPAREN
1108 fn primaryExpr(parser: *Parser) !*Node {
1109 @panic("TODO");
1110 }
1111
1112 /// Generic
1113 /// <- TypeName COLON AssignmentExpr
1114 /// / Keyword_default COLON AssignmentExpr
1115 fn generic(parser: *Parser) !*Node {
1116 @panic("TODO");
1117 }
1118
1119 /// PostFixExpr
1120 /// <- LPAREN TypeName RPAREN LBRACE Initializers RBRACE
1121 /// / LBRACKET Expr RBRACKET
1122 /// / LPAREN (AssignmentExpr (COMMA AssignmentExpr)*)? RPAREN
1123 /// / (PERIOD / ARROW) IDENTIFIER
1124 /// / (PLUSPLUS / MINUSMINUS)
1125 fn postFixExpr(parser: *Parser) !*Node {
1126 @panic("TODO");
1127 }
1128
1129 /// Initializers <- ((Designator+ EQUAL)? Initializer COMMA)* (Designator+ EQUAL)? Initializer COMMA?
1130 fn initializers(parser: *Parser) !*Node {
1131 @panic("TODO");
1132 }
1133
1134 /// Initializer
1135 /// <- LBRACE Initializers RBRACE
1136 /// / AssignmentExpr
1137 fn initializer(parser: *Parser, dr: *Node.Declarator) Error!?*Node {
1138 @panic("TODO");
1139 }
1140
1141 /// Designator
1142 /// <- LBRACKET ConstExpr RBRACKET
1143 /// / PERIOD IDENTIFIER
1144 fn designator(parser: *Parser) !*Node {
1145 @panic("TODO");
1146 }
1147
1148 /// CompoundStmt <- LBRACE (Declaration / Stmt)* RBRACE
1149 fn compoundStmt(parser: *Parser) Error!?*Node {
1150 const lbrace = parser.eatToken(.LBrace) orelse return null;
1151 try parser.pushScope(.Block);
1152 defer parser.popScope();
1153 const body_node = try parser.arena.create(Node.CompoundStmt);
1154 body_node.* = .{
1155 .lbrace = lbrace,
1156 .statements = Node.CompoundStmt.StmtList.init(parser.arena),
1157 .rbrace = undefined,
1158 };
1159 while (true) {
1160 if (parser.eatToken(.RBRACE)) |rbrace| {
1161 body_node.rbrace = rbrace;
1162 break;
1163 }
1164 try body_node.statements.push((try parser.declaration()) orelse (try parser.stmt()));
1165 }
1166 return &body_node.base;
1167 }
1168
1169 /// Stmt
1170 /// <- CompoundStmt
1171 /// / Keyword_if LPAREN Expr RPAREN Stmt (Keyword_ELSE Stmt)?
1172 /// / Keyword_switch LPAREN Expr RPAREN Stmt
1173 /// / Keyword_while LPAREN Expr RPAREN Stmt
1174 /// / Keyword_do statement Keyword_while LPAREN Expr RPAREN SEMICOLON
1175 /// / Keyword_for LPAREN (Declaration / ExprStmt) ExprStmt Expr? RPAREN Stmt
1176 /// / Keyword_default COLON Stmt
1177 /// / Keyword_case ConstExpr COLON Stmt
1178 /// / Keyword_goto IDENTIFIER SEMICOLON
1179 /// / Keyword_continue SEMICOLON
1180 /// / Keyword_break SEMICOLON
1181 /// / Keyword_return Expr? SEMICOLON
1182 /// / IDENTIFIER COLON Stmt
1183 /// / ExprStmt
1184 fn stmt(parser: *Parser) Error!*Node {
1185 if (try parser.compoundStmt()) |node| return node;
1186 if (parser.eatToken(.Keyword_if)) |tok| {
1187 const node = try parser.arena.create(Node.IfStmt);
1188 _ = try parser.expectToken(.LParen);
1189 node.* = .{
1190 .@"if" = tok,
1191 .cond = (try parser.expr()) orelse return parser.err(.{
1192 .ExpectedExpr = .{ .token = parser.it.index },
1193 }),
1194 .body = undefined,
1195 .@"else" = null,
1196 };
1197 _ = try parser.expectToken(.RParen);
1198 node.body = try parser.stmt();
1199 if (parser.eatToken(.Keyword_else)) |else_tok| {
1200 node.@"else" = .{
1201 .tok = else_tok,
1202 .body = try parser.stmt(),
1203 };
1204 }
1205 return &node.base;
1206 }
1207 if (parser.eatToken(.Keyword_while)) |tok| {
1208 try parser.pushScope(.Loop);
1209 defer parser.popScope();
1210 _ = try parser.expectToken(.LParen);
1211 const cond = (try parser.expr()) orelse return parser.err(.{
1212 .ExpectedExpr = .{ .token = parser.it.index },
1213 });
1214 const rparen = try parser.expectToken(.RParen);
1215 const node = try parser.arena.create(Node.WhileStmt);
1216 node.* = .{
1217 .@"while" = tok,
1218 .cond = cond,
1219 .rparen = rparen,
1220 .body = try parser.stmt(),
1221 .semicolon = try parser.expectToken(.Semicolon),
1222 };
1223 return &node.base;
1224 }
1225 if (parser.eatToken(.Keyword_do)) |tok| {
1226 try parser.pushScope(.Loop);
1227 defer parser.popScope();
1228 const body = try parser.stmt();
1229 _ = try parser.expectToken(.LParen);
1230 const cond = (try parser.expr()) orelse return parser.err(.{
1231 .ExpectedExpr = .{ .token = parser.it.index },
1232 });
1233 _ = try parser.expectToken(.RParen);
1234 const node = try parser.arena.create(Node.DoStmt);
1235 node.* = .{
1236 .do = tok,
1237 .body = body,
1238 .cond = cond,
1239 .@"while" = @"while",
1240 .semicolon = try parser.expectToken(.Semicolon),
1241 };
1242 return &node.base;
1243 }
1244 if (parser.eatToken(.Keyword_for)) |tok| {
1245 try parser.pushScope(.Loop);
1246 defer parser.popScope();
1247 _ = try parser.expectToken(.LParen);
1248 const init = if (try parser.declaration()) |decl| blk: {
1249 // TODO disallow storage class other than auto and register
1250 break :blk decl;
1251 } else try parser.exprStmt();
1252 const cond = try parser.expr();
1253 const semicolon = try parser.expectToken(.Semicolon);
1254 const incr = try parser.expr();
1255 const rparen = try parser.expectToken(.RParen);
1256 const node = try parser.arena.create(Node.ForStmt);
1257 node.* = .{
1258 .@"for" = tok,
1259 .init = init,
1260 .cond = cond,
1261 .semicolon = semicolon,
1262 .incr = incr,
1263 .rparen = rparen,
1264 .body = try parser.stmt(),
1265 };
1266 return &node.base;
1267 }
1268 if (parser.eatToken(.Keyword_switch)) |tok| {
1269 try parser.pushScope(.Switch);
1270 defer parser.popScope();
1271 _ = try parser.expectToken(.LParen);
1272 const switch_expr = try parser.exprStmt();
1273 const rparen = try parser.expectToken(.RParen);
1274 const node = try parser.arena.create(Node.SwitchStmt);
1275 node.* = .{
1276 .@"switch" = tok,
1277 .expr = switch_expr,
1278 .rparen = rparen,
1279 .body = try parser.stmt(),
1280 };
1281 return &node.base;
1282 }
1283 if (parser.eatToken(.Keyword_default)) |tok| {
1284 _ = try parser.expectToken(.Colon);
1285 const node = try parser.arena.create(Node.LabeledStmt);
1286 node.* = .{
1287 .kind = .{ .Default = tok },
1288 .stmt = try parser.stmt(),
1289 };
1290 return &node.base;
1291 }
1292 if (parser.eatToken(.Keyword_case)) |tok| {
1293 _ = try parser.expectToken(.Colon);
1294 const node = try parser.arena.create(Node.LabeledStmt);
1295 node.* = .{
1296 .kind = .{ .Case = tok },
1297 .stmt = try parser.stmt(),
1298 };
1299 return &node.base;
1300 }
1301 if (parser.eatToken(.Keyword_goto)) |tok| {
1302 const node = try parser.arena.create(Node.JumpStmt);
1303 node.* = .{
1304 .ltoken = tok,
1305 .kind = .{ .Goto = tok },
1306 .semicolon = try parser.expectToken(.Semicolon),
1307 };
1308 return &node.base;
1309 }
1310 if (parser.eatToken(.Keyword_continue)) |tok| {
1311 const node = try parser.arena.create(Node.JumpStmt);
1312 node.* = .{
1313 .ltoken = tok,
1314 .kind = .Continue,
1315 .semicolon = try parser.expectToken(.Semicolon),
1316 };
1317 return &node.base;
1318 }
1319 if (parser.eatToken(.Keyword_break)) |tok| {
1320 const node = try parser.arena.create(Node.JumpStmt);
1321 node.* = .{
1322 .ltoken = tok,
1323 .kind = .Break,
1324 .semicolon = try parser.expectToken(.Semicolon),
1325 };
1326 return &node.base;
1327 }
1328 if (parser.eatToken(.Keyword_return)) |tok| {
1329 const node = try parser.arena.create(Node.JumpStmt);
1330 node.* = .{
1331 .ltoken = tok,
1332 .kind = .{ .Return = try parser.expr() },
1333 .semicolon = try parser.expectToken(.Semicolon),
1334 };
1335 return &node.base;
1336 }
1337 if (parser.eatToken(.Identifier)) |tok| {
1338 if (parser.eatToken(.Colon)) |_| {
1339 const node = try parser.arena.create(Node.LabeledStmt);
1340 node.* = .{
1341 .kind = .{ .Label = tok },
1342 .stmt = try parser.stmt(),
1343 };
1344 return &node.base;
1345 }
1346 parser.putBackToken(tok);
1347 }
1348 return parser.exprStmt();
1349 }
1350
1351 /// ExprStmt <- Expr? SEMICOLON
1352 fn exprStmt(parser: *Parser) !*Node {
1353 const node = try parser.arena.create(Node.ExprStmt);
1354 node.* = .{
1355 .expr = try parser.expr(),
1356 .semicolon = try parser.expectToken(.Semicolon),
1357 };
1358 return &node.base;
1359 }
1360
1361 fn eatToken(parser: *Parser, id: @TagType(Token.Id)) ?TokenIndex {
1362 while (true) {
1363 switch ((parser.it.next() orelse return null).id) {
1364 .LineComment, .MultiLineComment, .Nl => continue,
1365 else => |next_id| if (next_id == id) {
1366 return parser.it.index;
1367 } else {
1368 _ = parser.it.prev();
1369 return null;
1370 },
1371 }
1372 }
1373 }
1374
1375 fn expectToken(parser: *Parser, id: @TagType(Token.Id)) Error!TokenIndex {
1376 while (true) {
1377 switch ((parser.it.next() orelse return error.ParseError).id) {
1378 .LineComment, .MultiLineComment, .Nl => continue,
1379 else => |next_id| if (next_id != id) {
1380 return parser.err(.{
1381 .ExpectedToken = .{ .token = parser.it.index, .expected_id = id },
1382 });
1383 } else {
1384 return parser.it.index;
1385 },
1386 }
1387 }
1388 }
1389
1390 fn putBackToken(parser: *Parser, putting_back: TokenIndex) void {
1391 while (true) {
1392 const prev_tok = parser.it.next() orelse return;
1393 switch (prev_tok.id) {
1394 .LineComment, .MultiLineComment, .Nl => continue,
1395 else => {
1396 assert(parser.it.list.at(putting_back) == prev_tok);
1397 return;
1398 },
1399 }
1400 }
1401 }
1402
1403 fn err(parser: *Parser, msg: ast.Error) Error {
1404 try parser.tree.msgs.push(.{
1405 .kind = .Error,
1406 .inner = msg,
1407 });
1408 return error.ParseError;
1409 }
1410
1411 fn warn(parser: *Parser, msg: ast.Error) Error!void {
1412 const is_warning = switch (parser.options.warn_as_err) {
1413 .None => true,
1414 .Some => |list| for (list) |item| (if (item == msg) break false) else true,
1415 .All => false,
1416 };
1417 try parser.tree.msgs.push(.{
1418 .kind = if (is_warning) .Warning else .Error,
1419 .inner = msg,
1420 });
1421 if (!is_warning) return error.ParseError;
1422 }
1423
1424 fn note(parser: *Parser, msg: ast.Error) Error!void {
1425 try parser.tree.msgs.push(.{
1426 .kind = .Note,
1427 .inner = msg,
1428 });
1429 }
1430};
1431
lib/std/c/tokenizer.zig created+1583
...@@ -0,0 +1,1583 @@
1const std = @import("std");
2const mem = std.mem;
3
4pub const Source = struct {
5 buffer: []const u8,
6 file_name: []const u8,
7 tokens: TokenList,
8
9 pub const TokenList = std.SegmentedList(Token, 64);
10};
11
12pub const Token = struct {
13 id: Id,
14 start: usize,
15 end: usize,
16 source: *Source,
17
18 pub const Id = union(enum) {
19 Invalid,
20 Eof,
21 Nl,
22 Identifier,
23
24 /// special case for #include <...>
25 MacroString,
26 StringLiteral: StrKind,
27 CharLiteral: StrKind,
28 IntegerLiteral: NumSuffix,
29 FloatLiteral: NumSuffix,
30 Bang,
31 BangEqual,
32 Pipe,
33 PipePipe,
34 PipeEqual,
35 Equal,
36 EqualEqual,
37 LParen,
38 RParen,
39 LBrace,
40 RBrace,
41 LBracket,
42 RBracket,
43 Period,
44 Ellipsis,
45 Caret,
46 CaretEqual,
47 Plus,
48 PlusPlus,
49 PlusEqual,
50 Minus,
51 MinusMinus,
52 MinusEqual,
53 Asterisk,
54 AsteriskEqual,
55 Percent,
56 PercentEqual,
57 Arrow,
58 Colon,
59 Semicolon,
60 Slash,
61 SlashEqual,
62 Comma,
63 Ampersand,
64 AmpersandAmpersand,
65 AmpersandEqual,
66 QuestionMark,
67 AngleBracketLeft,
68 AngleBracketLeftEqual,
69 AngleBracketAngleBracketLeft,
70 AngleBracketAngleBracketLeftEqual,
71 AngleBracketRight,
72 AngleBracketRightEqual,
73 AngleBracketAngleBracketRight,
74 AngleBracketAngleBracketRightEqual,
75 Tilde,
76 LineComment,
77 MultiLineComment,
78 Hash,
79 HashHash,
80
81 Keyword_auto,
82 Keyword_break,
83 Keyword_case,
84 Keyword_char,
85 Keyword_const,
86 Keyword_continue,
87 Keyword_default,
88 Keyword_do,
89 Keyword_double,
90 Keyword_else,
91 Keyword_enum,
92 Keyword_extern,
93 Keyword_float,
94 Keyword_for,
95 Keyword_goto,
96 Keyword_if,
97 Keyword_int,
98 Keyword_long,
99 Keyword_register,
100 Keyword_return,
101 Keyword_short,
102 Keyword_signed,
103 Keyword_sizeof,
104 Keyword_static,
105 Keyword_struct,
106 Keyword_switch,
107 Keyword_typedef,
108 Keyword_union,
109 Keyword_unsigned,
110 Keyword_void,
111 Keyword_volatile,
112 Keyword_while,
113
114 // ISO C99
115 Keyword_bool,
116 Keyword_complex,
117 Keyword_imaginary,
118 Keyword_inline,
119 Keyword_restrict,
120
121 // ISO C11
122 Keyword_alignas,
123 Keyword_alignof,
124 Keyword_atomic,
125 Keyword_generic,
126 Keyword_noreturn,
127 Keyword_static_assert,
128 Keyword_thread_local,
129
130 // Preprocessor directives
131 Keyword_include,
132 Keyword_define,
133 Keyword_ifdef,
134 Keyword_ifndef,
135 Keyword_error,
136 Keyword_pragma,
137
138 pub fn symbol(id: @TagType(Id)) []const u8 {
139 return switch (id) {
140 .Invalid => "Invalid",
141 .Eof => "Eof",
142 .Nl => "NewLine",
143 .Identifier => "Identifier",
144 .MacroString => "MacroString",
145 .StringLiteral => "StringLiteral",
146 .CharLiteral => "CharLiteral",
147 .IntegerLiteral => "IntegerLiteral",
148 .FloatLiteral => "FloatLiteral",
149 .LineComment => "LineComment",
150 .MultiLineComment => "MultiLineComment",
151
152 .Bang => "!",
153 .BangEqual => "!=",
154 .Pipe => "|",
155 .PipePipe => "||",
156 .PipeEqual => "|=",
157 .Equal => "=",
158 .EqualEqual => "==",
159 .LParen => "(",
160 .RParen => ")",
161 .LBrace => "{",
162 .RBrace => "}",
163 .LBracket => "[",
164 .RBracket => "]",
165 .Period => ".",
166 .Ellipsis => "...",
167 .Caret => "^",
168 .CaretEqual => "^=",
169 .Plus => "+",
170 .PlusPlus => "++",
171 .PlusEqual => "+=",
172 .Minus => "-",
173 .MinusMinus => "--",
174 .MinusEqual => "-=",
175 .Asterisk => "*",
176 .AsteriskEqual => "*=",
177 .Percent => "%",
178 .PercentEqual => "%=",
179 .Arrow => "->",
180 .Colon => ":",
181 .Semicolon => ";",
182 .Slash => "/",
183 .SlashEqual => "/=",
184 .Comma => ",",
185 .Ampersand => "&",
186 .AmpersandAmpersand => "&&",
187 .AmpersandEqual => "&=",
188 .QuestionMark => "?",
189 .AngleBracketLeft => "<",
190 .AngleBracketLeftEqual => "<=",
191 .AngleBracketAngleBracketLeft => "<<",
192 .AngleBracketAngleBracketLeftEqual => "<<=",
193 .AngleBracketRight => ">",
194 .AngleBracketRightEqual => ">=",
195 .AngleBracketAngleBracketRight => ">>",
196 .AngleBracketAngleBracketRightEqual => ">>=",
197 .Tilde => "~",
198 .Hash => "#",
199 .HashHash => "##",
200 .Keyword_auto => "auto",
201 .Keyword_break => "break",
202 .Keyword_case => "case",
203 .Keyword_char => "char",
204 .Keyword_const => "const",
205 .Keyword_continue => "continue",
206 .Keyword_default => "default",
207 .Keyword_do => "do",
208 .Keyword_double => "double",
209 .Keyword_else => "else",
210 .Keyword_enum => "enum",
211 .Keyword_extern => "extern",
212 .Keyword_float => "float",
213 .Keyword_for => "for",
214 .Keyword_goto => "goto",
215 .Keyword_if => "if",
216 .Keyword_int => "int",
217 .Keyword_long => "long",
218 .Keyword_register => "register",
219 .Keyword_return => "return",
220 .Keyword_short => "short",
221 .Keyword_signed => "signed",
222 .Keyword_sizeof => "sizeof",
223 .Keyword_static => "static",
224 .Keyword_struct => "struct",
225 .Keyword_switch => "switch",
226 .Keyword_typedef => "typedef",
227 .Keyword_union => "union",
228 .Keyword_unsigned => "unsigned",
229 .Keyword_void => "void",
230 .Keyword_volatile => "volatile",
231 .Keyword_while => "while",
232 .Keyword_bool => "_Bool",
233 .Keyword_complex => "_Complex",
234 .Keyword_imaginary => "_Imaginary",
235 .Keyword_inline => "inline",
236 .Keyword_restrict => "restrict",
237 .Keyword_alignas => "_Alignas",
238 .Keyword_alignof => "_Alignof",
239 .Keyword_atomic => "_Atomic",
240 .Keyword_generic => "_Generic",
241 .Keyword_noreturn => "_Noreturn",
242 .Keyword_static_assert => "_Static_assert",
243 .Keyword_thread_local => "_Thread_local",
244 .Keyword_include => "include",
245 .Keyword_define => "define",
246 .Keyword_ifdef => "ifdef",
247 .Keyword_ifndef => "ifndef",
248 .Keyword_error => "error",
249 .Keyword_pragma => "pragma",
250 };
251 }
252 };
253
254 pub fn eql(a: Token, b: Token) bool {
255 // do we really need this cast here
256 if (@as(@TagType(Id), a.id) != b.id) return false;
257 return mem.eql(u8, a.slice(), b.slice());
258 }
259
260 pub fn slice(tok: Token) []const u8 {
261 return tok.source.buffer[tok.start..tok.end];
262 }
263
264 pub const Keyword = struct {
265 bytes: []const u8,
266 id: Id,
267 hash: u32,
268
269 fn init(bytes: []const u8, id: Id) Keyword {
270 @setEvalBranchQuota(2000);
271 return .{
272 .bytes = bytes,
273 .id = id,
274 .hash = std.hash_map.hashString(bytes),
275 };
276 }
277 };
278
279 // TODO extensions
280 pub const keywords = [_]Keyword{
281 Keyword.init("auto", .Keyword_auto),
282 Keyword.init("break", .Keyword_break),
283 Keyword.init("case", .Keyword_case),
284 Keyword.init("char", .Keyword_char),
285 Keyword.init("const", .Keyword_const),
286 Keyword.init("continue", .Keyword_continue),
287 Keyword.init("default", .Keyword_default),
288 Keyword.init("do", .Keyword_do),
289 Keyword.init("double", .Keyword_double),
290 Keyword.init("else", .Keyword_else),
291 Keyword.init("enum", .Keyword_enum),
292 Keyword.init("extern", .Keyword_extern),
293 Keyword.init("float", .Keyword_float),
294 Keyword.init("for", .Keyword_for),
295 Keyword.init("goto", .Keyword_goto),
296 Keyword.init("if", .Keyword_if),
297 Keyword.init("int", .Keyword_int),
298 Keyword.init("long", .Keyword_long),
299 Keyword.init("register", .Keyword_register),
300 Keyword.init("return", .Keyword_return),
301 Keyword.init("short", .Keyword_short),
302 Keyword.init("signed", .Keyword_signed),
303 Keyword.init("sizeof", .Keyword_sizeof),
304 Keyword.init("static", .Keyword_static),
305 Keyword.init("struct", .Keyword_struct),
306 Keyword.init("switch", .Keyword_switch),
307 Keyword.init("typedef", .Keyword_typedef),
308 Keyword.init("union", .Keyword_union),
309 Keyword.init("unsigned", .Keyword_unsigned),
310 Keyword.init("void", .Keyword_void),
311 Keyword.init("volatile", .Keyword_volatile),
312 Keyword.init("while", .Keyword_while),
313
314 // ISO C99
315 Keyword.init("_Bool", .Keyword_bool),
316 Keyword.init("_Complex", .Keyword_complex),
317 Keyword.init("_Imaginary", .Keyword_imaginary),
318 Keyword.init("inline", .Keyword_inline),
319 Keyword.init("restrict", .Keyword_restrict),
320
321 // ISO C11
322 Keyword.init("_Alignas", .Keyword_alignas),
323 Keyword.init("_Alignof", .Keyword_alignof),
324 Keyword.init("_Atomic", .Keyword_atomic),
325 Keyword.init("_Generic", .Keyword_generic),
326 Keyword.init("_Noreturn", .Keyword_noreturn),
327 Keyword.init("_Static_assert", .Keyword_static_assert),
328 Keyword.init("_Thread_local", .Keyword_thread_local),
329
330 // Preprocessor directives
331 Keyword.init("include", .Keyword_include),
332 Keyword.init("define", .Keyword_define),
333 Keyword.init("ifdef", .Keyword_ifdef),
334 Keyword.init("ifndef", .Keyword_ifndef),
335 Keyword.init("error", .Keyword_error),
336 Keyword.init("pragma", .Keyword_pragma),
337 };
338
339 // TODO perfect hash at comptime
340 // TODO do this in the preprocessor
341 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {
342 var hash = std.hash_map.hashString(bytes);
343 for (keywords) |kw| {
344 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {
345 switch (kw.id) {
346 .Keyword_include,
347 .Keyword_define,
348 .Keyword_ifdef,
349 .Keyword_ifndef,
350 .Keyword_error,
351 .Keyword_pragma,
352 => if (!pp_directive) return null,
353 else => {},
354 }
355 return kw.id;
356 }
357 }
358 return null;
359 }
360
361 pub const NumSuffix = enum {
362 None,
363 F,
364 L,
365 U,
366 LU,
367 LL,
368 LLU,
369 };
370
371 pub const StrKind = enum {
372 None,
373 Wide,
374 Utf8,
375 Utf16,
376 Utf32,
377 };
378};
379
380pub const Tokenizer = struct {
381 source: *Source,
382 index: usize = 0,
383 prev_tok_id: @TagType(Token.Id) = .Invalid,
384 pp_directive: bool = false,
385
386 pub fn next(self: *Tokenizer) Token {
387 const start_index = self.index;
388 var result = Token{
389 .id = .Eof,
390 .start = self.index,
391 .end = undefined,
392 .source = self.source,
393 };
394 var state: enum {
395 Start,
396 Cr,
397 BackSlash,
398 BackSlashCr,
399 u,
400 u8,
401 U,
402 L,
403 StringLiteral,
404 CharLiteralStart,
405 CharLiteral,
406 EscapeSequence,
407 CrEscape,
408 OctalEscape,
409 HexEscape,
410 UnicodeEscape,
411 Identifier,
412 Equal,
413 Bang,
414 Pipe,
415 Percent,
416 Asterisk,
417 Plus,
418
419 /// special case for #include <...>
420 MacroString,
421 AngleBracketLeft,
422 AngleBracketAngleBracketLeft,
423 AngleBracketRight,
424 AngleBracketAngleBracketRight,
425 Caret,
426 Period,
427 Period2,
428 Minus,
429 Slash,
430 Ampersand,
431 Hash,
432 LineComment,
433 MultiLineComment,
434 MultiLineCommentAsterisk,
435 Zero,
436 IntegerLiteralOct,
437 IntegerLiteralBinary,
438 IntegerLiteralHex,
439 IntegerLiteral,
440 IntegerSuffix,
441 IntegerSuffixU,
442 IntegerSuffixL,
443 IntegerSuffixLL,
444 IntegerSuffixUL,
445 FloatFraction,
446 FloatFractionHex,
447 FloatExponent,
448 FloatExponentDigits,
449 FloatSuffix,
450 } = .Start;
451 var string = false;
452 var counter: u32 = 0;
453 while (self.index < self.source.buffer.len) : (self.index += 1) {
454 const c = self.source.buffer[self.index];
455 switch (state) {
456 .Start => switch (c) {
457 '\n' => {
458 self.pp_directive = false;
459 result.id = .Nl;
460 self.index += 1;
461 break;
462 },
463 '\r' => {
464 state = .Cr;
465 },
466 '"' => {
467 result.id = .{ .StringLiteral = .None };
468 state = .StringLiteral;
469 },
470 '\'' => {
471 result.id = .{ .CharLiteral = .None };
472 state = .CharLiteralStart;
473 },
474 'u' => {
475 state = .u;
476 },
477 'U' => {
478 state = .U;
479 },
480 'L' => {
481 state = .L;
482 },
483 'a'...'t', 'v'...'z', 'A'...'K', 'M'...'T', 'V'...'Z', '_' => {
484 state = .Identifier;
485 },
486 '=' => {
487 state = .Equal;
488 },
489 '!' => {
490 state = .Bang;
491 },
492 '|' => {
493 state = .Pipe;
494 },
495 '(' => {
496 result.id = .LParen;
497 self.index += 1;
498 break;
499 },
500 ')' => {
501 result.id = .RParen;
502 self.index += 1;
503 break;
504 },
505 '[' => {
506 result.id = .LBracket;
507 self.index += 1;
508 break;
509 },
510 ']' => {
511 result.id = .RBracket;
512 self.index += 1;
513 break;
514 },
515 ';' => {
516 result.id = .Semicolon;
517 self.index += 1;
518 break;
519 },
520 ',' => {
521 result.id = .Comma;
522 self.index += 1;
523 break;
524 },
525 '?' => {
526 result.id = .QuestionMark;
527 self.index += 1;
528 break;
529 },
530 ':' => {
531 result.id = .Colon;
532 self.index += 1;
533 break;
534 },
535 '%' => {
536 state = .Percent;
537 },
538 '*' => {
539 state = .Asterisk;
540 },
541 '+' => {
542 state = .Plus;
543 },
544 '<' => {
545 if (self.prev_tok_id == .Keyword_include)
546 state = .MacroString
547 else
548 state = .AngleBracketLeft;
549 },
550 '>' => {
551 state = .AngleBracketRight;
552 },
553 '^' => {
554 state = .Caret;
555 },
556 '{' => {
557 result.id = .LBrace;
558 self.index += 1;
559 break;
560 },
561 '}' => {
562 result.id = .RBrace;
563 self.index += 1;
564 break;
565 },
566 '~' => {
567 result.id = .Tilde;
568 self.index += 1;
569 break;
570 },
571 '.' => {
572 state = .Period;
573 },
574 '-' => {
575 state = .Minus;
576 },
577 '/' => {
578 state = .Slash;
579 },
580 '&' => {
581 state = .Ampersand;
582 },
583 '#' => {
584 state = .Hash;
585 },
586 '0' => {
587 state = .Zero;
588 },
589 '1'...'9' => {
590 state = .IntegerLiteral;
591 },
592 '\\' => {
593 state = .BackSlash;
594 },
595 '\t', '\x0B', '\x0C', ' ' => {
596 result.start = self.index + 1;
597 },
598 else => {
599 // TODO handle invalid bytes better
600 result.id = .Invalid;
601 self.index += 1;
602 break;
603 },
604 },
605 .Cr => switch (c) {
606 '\n' => {
607 self.pp_directive = false;
608 result.id = .Nl;
609 self.index += 1;
610 break;
611 },
612 else => {
613 result.id = .Invalid;
614 break;
615 },
616 },
617 .BackSlash => switch (c) {
618 '\n' => {
619 state = .Start;
620 },
621 '\r' => {
622 state = .BackSlashCr;
623 },
624 '\t', '\x0B', '\x0C', ' ' => {
625 // TODO warn
626 },
627 else => {
628 result.id = .Invalid;
629 break;
630 },
631 },
632 .BackSlashCr => switch (c) {
633 '\n' => {
634 state = .Start;
635 },
636 else => {
637 result.id = .Invalid;
638 break;
639 },
640 },
641 .u => switch (c) {
642 '8' => {
643 state = .u8;
644 },
645 '\'' => {
646 result.id = .{ .CharLiteral = .Utf16 };
647 state = .CharLiteralStart;
648 },
649 '\"' => {
650 result.id = .{ .StringLiteral = .Utf16 };
651 state = .StringLiteral;
652 },
653 else => {
654 state = .Identifier;
655 },
656 },
657 .u8 => switch (c) {
658 '\"' => {
659 result.id = .{ .StringLiteral = .Utf8 };
660 state = .StringLiteral;
661 },
662 else => {
663 state = .Identifier;
664 },
665 },
666 .U => switch (c) {
667 '\'' => {
668 result.id = .{ .CharLiteral = .Utf32 };
669 state = .CharLiteralStart;
670 },
671 '\"' => {
672 result.id = .{ .StringLiteral = .Utf32 };
673 state = .StringLiteral;
674 },
675 else => {
676 state = .Identifier;
677 },
678 },
679 .L => switch (c) {
680 '\'' => {
681 result.id = .{ .CharLiteral = .Wide };
682 state = .CharLiteralStart;
683 },
684 '\"' => {
685 result.id = .{ .StringLiteral = .Wide };
686 state = .StringLiteral;
687 },
688 else => {
689 state = .Identifier;
690 },
691 },
692 .StringLiteral => switch (c) {
693 '\\' => {
694 string = true;
695 state = .EscapeSequence;
696 },
697 '"' => {
698 self.index += 1;
699 break;
700 },
701 '\n', '\r' => {
702 result.id = .Invalid;
703 break;
704 },
705 else => {},
706 },
707 .CharLiteralStart => switch (c) {
708 '\\' => {
709 string = false;
710 state = .EscapeSequence;
711 },
712 '\'', '\n' => {
713 result.id = .Invalid;
714 break;
715 },
716 else => {
717 state = .CharLiteral;
718 },
719 },
720 .CharLiteral => switch (c) {
721 '\\' => {
722 string = false;
723 state = .EscapeSequence;
724 },
725 '\'' => {
726 self.index += 1;
727 break;
728 },
729 '\n' => {
730 result.id = .Invalid;
731 break;
732 },
733 else => {},
734 },
735 .EscapeSequence => switch (c) {
736 '\'', '"', '?', '\\', 'a', 'b', 'f', 'n', 'r', 't', 'v', '\n' => {
737 state = if (string) .StringLiteral else .CharLiteral;
738 },
739 '\r' => {
740 state = .CrEscape;
741 },
742 '0'...'7' => {
743 counter = 1;
744 state = .OctalEscape;
745 },
746 'x' => {
747 state = .HexEscape;
748 },
749 'u' => {
750 counter = 4;
751 state = .OctalEscape;
752 },
753 'U' => {
754 counter = 8;
755 state = .OctalEscape;
756 },
757 else => {
758 result.id = .Invalid;
759 break;
760 },
761 },
762 .CrEscape => switch (c) {
763 '\n' => {
764 state = if (string) .StringLiteral else .CharLiteral;
765 },
766 else => {
767 result.id = .Invalid;
768 break;
769 },
770 },
771 .OctalEscape => switch (c) {
772 '0'...'7' => {
773 counter += 1;
774 if (counter == 3) {
775 state = if (string) .StringLiteral else .CharLiteral;
776 }
777 },
778 else => {
779 state = if (string) .StringLiteral else .CharLiteral;
780 },
781 },
782 .HexEscape => switch (c) {
783 '0'...'9', 'a'...'f', 'A'...'F' => {},
784 else => {
785 state = if (string) .StringLiteral else .CharLiteral;
786 },
787 },
788 .UnicodeEscape => switch (c) {
789 '0'...'9', 'a'...'f', 'A'...'F' => {
790 counter -= 1;
791 if (counter == 0) {
792 state = if (string) .StringLiteral else .CharLiteral;
793 }
794 },
795 else => {
796 if (counter != 0) {
797 result.id = .Invalid;
798 break;
799 }
800 state = if (string) .StringLiteral else .CharLiteral;
801 },
802 },
803 .Identifier => switch (c) {
804 'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
805 else => {
806 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
807 if (self.prev_tok_id == .Hash)
808 self.pp_directive = true;
809 break;
810 },
811 },
812 .Equal => switch (c) {
813 '=' => {
814 result.id = .EqualEqual;
815 self.index += 1;
816 break;
817 },
818 else => {
819 result.id = .Equal;
820 break;
821 },
822 },
823 .Bang => switch (c) {
824 '=' => {
825 result.id = .BangEqual;
826 self.index += 1;
827 break;
828 },
829 else => {
830 result.id = .Bang;
831 break;
832 },
833 },
834 .Pipe => switch (c) {
835 '=' => {
836 result.id = .PipeEqual;
837 self.index += 1;
838 break;
839 },
840 '|' => {
841 result.id = .PipePipe;
842 self.index += 1;
843 break;
844 },
845 else => {
846 result.id = .Pipe;
847 break;
848 },
849 },
850 .Percent => switch (c) {
851 '=' => {
852 result.id = .PercentEqual;
853 self.index += 1;
854 break;
855 },
856 else => {
857 result.id = .Percent;
858 break;
859 },
860 },
861 .Asterisk => switch (c) {
862 '=' => {
863 result.id = .AsteriskEqual;
864 self.index += 1;
865 break;
866 },
867 else => {
868 result.id = .Asterisk;
869 break;
870 },
871 },
872 .Plus => switch (c) {
873 '=' => {
874 result.id = .PlusEqual;
875 self.index += 1;
876 break;
877 },
878 '+' => {
879 result.id = .PlusPlus;
880 self.index += 1;
881 break;
882 },
883 else => {
884 result.id = .Plus;
885 break;
886 },
887 },
888 .MacroString => switch (c) {
889 '>' => {
890 result.id = .MacroString;
891 self.index += 1;
892 break;
893 },
894 else => {},
895 },
896 .AngleBracketLeft => switch (c) {
897 '<' => {
898 state = .AngleBracketAngleBracketLeft;
899 },
900 '=' => {
901 result.id = .AngleBracketLeftEqual;
902 self.index += 1;
903 break;
904 },
905 else => {
906 result.id = .AngleBracketLeft;
907 break;
908 },
909 },
910 .AngleBracketAngleBracketLeft => switch (c) {
911 '=' => {
912 result.id = .AngleBracketAngleBracketLeftEqual;
913 self.index += 1;
914 break;
915 },
916 else => {
917 result.id = .AngleBracketAngleBracketLeft;
918 break;
919 },
920 },
921 .AngleBracketRight => switch (c) {
922 '>' => {
923 state = .AngleBracketAngleBracketRight;
924 },
925 '=' => {
926 result.id = .AngleBracketRightEqual;
927 self.index += 1;
928 break;
929 },
930 else => {
931 result.id = .AngleBracketRight;
932 break;
933 },
934 },
935 .AngleBracketAngleBracketRight => switch (c) {
936 '=' => {
937 result.id = .AngleBracketAngleBracketRightEqual;
938 self.index += 1;
939 break;
940 },
941 else => {
942 result.id = .AngleBracketAngleBracketRight;
943 break;
944 },
945 },
946 .Caret => switch (c) {
947 '=' => {
948 result.id = .CaretEqual;
949 self.index += 1;
950 break;
951 },
952 else => {
953 result.id = .Caret;
954 break;
955 },
956 },
957 .Period => switch (c) {
958 '.' => {
959 state = .Period2;
960 },
961 '0'...'9' => {
962 state = .FloatFraction;
963 },
964 else => {
965 result.id = .Period;
966 break;
967 },
968 },
969 .Period2 => switch (c) {
970 '.' => {
971 result.id = .Ellipsis;
972 self.index += 1;
973 break;
974 },
975 else => {
976 result.id = .Period;
977 self.index -= 1;
978 break;
979 },
980 },
981 .Minus => switch (c) {
982 '>' => {
983 result.id = .Arrow;
984 self.index += 1;
985 break;
986 },
987 '=' => {
988 result.id = .MinusEqual;
989 self.index += 1;
990 break;
991 },
992 '-' => {
993 result.id = .MinusMinus;
994 self.index += 1;
995 break;
996 },
997 else => {
998 result.id = .Minus;
999 break;
1000 },
1001 },
1002 .Slash => switch (c) {
1003 '/' => {
1004 state = .LineComment;
1005 },
1006 '*' => {
1007 state = .MultiLineComment;
1008 },
1009 '=' => {
1010 result.id = .SlashEqual;
1011 self.index += 1;
1012 break;
1013 },
1014 else => {
1015 result.id = .Slash;
1016 break;
1017 },
1018 },
1019 .Ampersand => switch (c) {
1020 '&' => {
1021 result.id = .AmpersandAmpersand;
1022 self.index += 1;
1023 break;
1024 },
1025 '=' => {
1026 result.id = .AmpersandEqual;
1027 self.index += 1;
1028 break;
1029 },
1030 else => {
1031 result.id = .Ampersand;
1032 break;
1033 },
1034 },
1035 .Hash => switch (c) {
1036 '#' => {
1037 result.id = .HashHash;
1038 self.index += 1;
1039 break;
1040 },
1041 else => {
1042 result.id = .Hash;
1043 break;
1044 },
1045 },
1046 .LineComment => switch (c) {
1047 '\n' => {
1048 result.id = .LineComment;
1049 self.index += 1;
1050 break;
1051 },
1052 else => {},
1053 },
1054 .MultiLineComment => switch (c) {
1055 '*' => {
1056 state = .MultiLineCommentAsterisk;
1057 },
1058 else => {},
1059 },
1060 .MultiLineCommentAsterisk => switch (c) {
1061 '/' => {
1062 result.id = .MultiLineComment;
1063 self.index += 1;
1064 break;
1065 },
1066 else => {
1067 state = .MultiLineComment;
1068 },
1069 },
1070 .Zero => switch (c) {
1071 '0'...'9' => {
1072 state = .IntegerLiteralOct;
1073 },
1074 'b', 'B' => {
1075 state = .IntegerLiteralBinary;
1076 },
1077 'x', 'X' => {
1078 state = .IntegerLiteralHex;
1079 },
1080 else => {
1081 state = .IntegerSuffix;
1082 self.index -= 1;
1083 },
1084 },
1085 .IntegerLiteralOct => switch (c) {
1086 '0'...'7' => {},
1087 else => {
1088 state = .IntegerSuffix;
1089 self.index -= 1;
1090 },
1091 },
1092 .IntegerLiteralBinary => switch (c) {
1093 '0', '1' => {},
1094 else => {
1095 state = .IntegerSuffix;
1096 self.index -= 1;
1097 },
1098 },
1099 .IntegerLiteralHex => switch (c) {
1100 '0'...'9', 'a'...'f', 'A'...'F' => {},
1101 '.' => {
1102 state = .FloatFractionHex;
1103 },
1104 'p', 'P' => {
1105 state = .FloatExponent;
1106 },
1107 else => {
1108 state = .IntegerSuffix;
1109 self.index -= 1;
1110 },
1111 },
1112 .IntegerLiteral => switch (c) {
1113 '0'...'9' => {},
1114 '.' => {
1115 state = .FloatFraction;
1116 },
1117 'e', 'E' => {
1118 state = .FloatExponent;
1119 },
1120 else => {
1121 state = .IntegerSuffix;
1122 self.index -= 1;
1123 },
1124 },
1125 .IntegerSuffix => switch (c) {
1126 'u', 'U' => {
1127 state = .IntegerSuffixU;
1128 },
1129 'l', 'L' => {
1130 state = .IntegerSuffixL;
1131 },
1132 else => {
1133 result.id = .{ .IntegerLiteral = .None };
1134 break;
1135 },
1136 },
1137 .IntegerSuffixU => switch (c) {
1138 'l', 'L' => {
1139 state = .IntegerSuffixUL;
1140 },
1141 else => {
1142 result.id = .{ .IntegerLiteral = .U };
1143 break;
1144 },
1145 },
1146 .IntegerSuffixL => switch (c) {
1147 'l', 'L' => {
1148 state = .IntegerSuffixLL;
1149 },
1150 'u', 'U' => {
1151 result.id = .{ .IntegerLiteral = .LU };
1152 self.index += 1;
1153 break;
1154 },
1155 else => {
1156 result.id = .{ .IntegerLiteral = .L };
1157 break;
1158 },
1159 },
1160 .IntegerSuffixLL => switch (c) {
1161 'u', 'U' => {
1162 result.id = .{ .IntegerLiteral = .LLU };
1163 self.index += 1;
1164 break;
1165 },
1166 else => {
1167 result.id = .{ .IntegerLiteral = .LL };
1168 break;
1169 },
1170 },
1171 .IntegerSuffixUL => switch (c) {
1172 'l', 'L' => {
1173 result.id = .{ .IntegerLiteral = .LLU };
1174 self.index += 1;
1175 break;
1176 },
1177 else => {
1178 result.id = .{ .IntegerLiteral = .LU };
1179 break;
1180 },
1181 },
1182 .FloatFraction => switch (c) {
1183 '0'...'9' => {},
1184 'e', 'E' => {
1185 state = .FloatExponent;
1186 },
1187 else => {
1188 self.index -= 1;
1189 state = .FloatSuffix;
1190 },
1191 },
1192 .FloatFractionHex => switch (c) {
1193 '0'...'9', 'a'...'f', 'A'...'F' => {},
1194 'p', 'P' => {
1195 state = .FloatExponent;
1196 },
1197 else => {
1198 result.id = .Invalid;
1199 break;
1200 },
1201 },
1202 .FloatExponent => switch (c) {
1203 '+', '-' => {
1204 state = .FloatExponentDigits;
1205 },
1206 else => {
1207 self.index -= 1;
1208 state = .FloatExponentDigits;
1209 },
1210 },
1211 .FloatExponentDigits => switch (c) {
1212 '0'...'9' => {
1213 counter += 1;
1214 },
1215 else => {
1216 if (counter == 0) {
1217 result.id = .Invalid;
1218 break;
1219 }
1220 state = .FloatSuffix;
1221 },
1222 },
1223 .FloatSuffix => switch (c) {
1224 'l', 'L' => {
1225 result.id = .{ .FloatLiteral = .L };
1226 self.index += 1;
1227 break;
1228 },
1229 'f', 'F' => {
1230 result.id = .{ .FloatLiteral = .F };
1231 self.index += 1;
1232 break;
1233 },
1234 else => {
1235 result.id = .{ .FloatLiteral = .None };
1236 break;
1237 },
1238 },
1239 }
1240 } else if (self.index == self.source.buffer.len) {
1241 switch (state) {
1242 .Start => {},
1243 .u, .u8, .U, .L, .Identifier => {
1244 result.id = Token.getKeyword(self.source.buffer[result.start..self.index], self.prev_tok_id == .Hash and !self.pp_directive) orelse .Identifier;
1245 },
1246
1247 .Cr,
1248 .BackSlash,
1249 .BackSlashCr,
1250 .Period2,
1251 .StringLiteral,
1252 .CharLiteralStart,
1253 .CharLiteral,
1254 .EscapeSequence,
1255 .CrEscape,
1256 .OctalEscape,
1257 .HexEscape,
1258 .UnicodeEscape,
1259 .MultiLineComment,
1260 .MultiLineCommentAsterisk,
1261 .FloatFraction,
1262 .FloatFractionHex,
1263 .FloatExponent,
1264 .FloatExponentDigits,
1265 .MacroString,
1266 => result.id = .Invalid,
1267
1268 .IntegerLiteralOct,
1269 .IntegerLiteralBinary,
1270 .IntegerLiteralHex,
1271 .IntegerLiteral,
1272 .IntegerSuffix,
1273 .Zero,
1274 => result.id = .{ .IntegerLiteral = .None },
1275 .IntegerSuffixU => result.id = .{ .IntegerLiteral = .U },
1276 .IntegerSuffixL => result.id = .{ .IntegerLiteral = .L },
1277 .IntegerSuffixLL => result.id = .{ .IntegerLiteral = .LL },
1278 .IntegerSuffixUL => result.id = .{ .IntegerLiteral = .LU },
1279
1280 .FloatSuffix => result.id = .{ .FloatLiteral = .None },
1281 .Equal => result.id = .Equal,
1282 .Bang => result.id = .Bang,
1283 .Minus => result.id = .Minus,
1284 .Slash => result.id = .Slash,
1285 .Ampersand => result.id = .Ampersand,
1286 .Hash => result.id = .Hash,
1287 .Period => result.id = .Period,
1288 .Pipe => result.id = .Pipe,
1289 .AngleBracketAngleBracketRight => result.id = .AngleBracketAngleBracketRight,
1290 .AngleBracketRight => result.id = .AngleBracketRight,
1291 .AngleBracketAngleBracketLeft => result.id = .AngleBracketAngleBracketLeft,
1292 .AngleBracketLeft => result.id = .AngleBracketLeft,
1293 .Plus => result.id = .Plus,
1294 .Percent => result.id = .Percent,
1295 .Caret => result.id = .Caret,
1296 .Asterisk => result.id = .Asterisk,
1297 .LineComment => result.id = .LineComment,
1298 }
1299 }
1300
1301 self.prev_tok_id = result.id;
1302 result.end = self.index;
1303 return result;
1304 }
1305};
1306
1307test "operators" {
1308 expectTokens(
1309 \\ ! != | || |= = ==
1310 \\ ( ) { } [ ] . .. ...
1311 \\ ^ ^= + ++ += - -- -=
1312 \\ * *= % %= -> : ; / /=
1313 \\ , & && &= ? < <= <<
1314 \\ <<= > >= >> >>= ~ # ##
1315 \\
1316 , &[_]Token.Id{
1317 .Bang,
1318 .BangEqual,
1319 .Pipe,
1320 .PipePipe,
1321 .PipeEqual,
1322 .Equal,
1323 .EqualEqual,
1324 .Nl,
1325 .LParen,
1326 .RParen,
1327 .LBrace,
1328 .RBrace,
1329 .LBracket,
1330 .RBracket,
1331 .Period,
1332 .Period,
1333 .Period,
1334 .Ellipsis,
1335 .Nl,
1336 .Caret,
1337 .CaretEqual,
1338 .Plus,
1339 .PlusPlus,
1340 .PlusEqual,
1341 .Minus,
1342 .MinusMinus,
1343 .MinusEqual,
1344 .Nl,
1345 .Asterisk,
1346 .AsteriskEqual,
1347 .Percent,
1348 .PercentEqual,
1349 .Arrow,
1350 .Colon,
1351 .Semicolon,
1352 .Slash,
1353 .SlashEqual,
1354 .Nl,
1355 .Comma,
1356 .Ampersand,
1357 .AmpersandAmpersand,
1358 .AmpersandEqual,
1359 .QuestionMark,
1360 .AngleBracketLeft,
1361 .AngleBracketLeftEqual,
1362 .AngleBracketAngleBracketLeft,
1363 .Nl,
1364 .AngleBracketAngleBracketLeftEqual,
1365 .AngleBracketRight,
1366 .AngleBracketRightEqual,
1367 .AngleBracketAngleBracketRight,
1368 .AngleBracketAngleBracketRightEqual,
1369 .Tilde,
1370 .Hash,
1371 .HashHash,
1372 .Nl,
1373 });
1374}
1375
1376test "keywords" {
1377 expectTokens(
1378 \\auto break case char const continue default do
1379 \\double else enum extern float for goto if int
1380 \\long register return short signed sizeof static
1381 \\struct switch typedef union unsigned void volatile
1382 \\while _Bool _Complex _Imaginary inline restrict _Alignas
1383 \\_Alignof _Atomic _Generic _Noreturn _Static_assert _Thread_local
1384 \\
1385 , &[_]Token.Id{
1386 .Keyword_auto,
1387 .Keyword_break,
1388 .Keyword_case,
1389 .Keyword_char,
1390 .Keyword_const,
1391 .Keyword_continue,
1392 .Keyword_default,
1393 .Keyword_do,
1394 .Nl,
1395 .Keyword_double,
1396 .Keyword_else,
1397 .Keyword_enum,
1398 .Keyword_extern,
1399 .Keyword_float,
1400 .Keyword_for,
1401 .Keyword_goto,
1402 .Keyword_if,
1403 .Keyword_int,
1404 .Nl,
1405 .Keyword_long,
1406 .Keyword_register,
1407 .Keyword_return,
1408 .Keyword_short,
1409 .Keyword_signed,
1410 .Keyword_sizeof,
1411 .Keyword_static,
1412 .Nl,
1413 .Keyword_struct,
1414 .Keyword_switch,
1415 .Keyword_typedef,
1416 .Keyword_union,
1417 .Keyword_unsigned,
1418 .Keyword_void,
1419 .Keyword_volatile,
1420 .Nl,
1421 .Keyword_while,
1422 .Keyword_bool,
1423 .Keyword_complex,
1424 .Keyword_imaginary,
1425 .Keyword_inline,
1426 .Keyword_restrict,
1427 .Keyword_alignas,
1428 .Nl,
1429 .Keyword_alignof,
1430 .Keyword_atomic,
1431 .Keyword_generic,
1432 .Keyword_noreturn,
1433 .Keyword_static_assert,
1434 .Keyword_thread_local,
1435 .Nl,
1436 });
1437}
1438
1439test "preprocessor keywords" {
1440 expectTokens(
1441 \\#include <test>
1442 \\#define #include <1
1443 \\#ifdef
1444 \\#ifndef
1445 \\#error
1446 \\#pragma
1447 \\
1448 , &[_]Token.Id{
1449 .Hash,
1450 .Keyword_include,
1451 .MacroString,
1452 .Nl,
1453 .Hash,
1454 .Keyword_define,
1455 .Hash,
1456 .Identifier,
1457 .AngleBracketLeft,
1458 .{ .IntegerLiteral = .None },
1459 .Nl,
1460 .Hash,
1461 .Keyword_ifdef,
1462 .Nl,
1463 .Hash,
1464 .Keyword_ifndef,
1465 .Nl,
1466 .Hash,
1467 .Keyword_error,
1468 .Nl,
1469 .Hash,
1470 .Keyword_pragma,
1471 .Nl,
1472 });
1473}
1474
1475test "line continuation" {
1476 expectTokens(
1477 \\#define foo \
1478 \\ bar
1479 \\"foo\
1480 \\ bar"
1481 \\#define "foo"
1482 \\ "bar"
1483 \\#define "foo" \
1484 \\ "bar"
1485 , &[_]Token.Id{
1486 .Hash,
1487 .Keyword_define,
1488 .Identifier,
1489 .Identifier,
1490 .Nl,
1491 .{ .StringLiteral = .None },
1492 .Nl,
1493 .Hash,
1494 .Keyword_define,
1495 .{ .StringLiteral = .None },
1496 .Nl,
1497 .{ .StringLiteral = .None },
1498 .Nl,
1499 .Hash,
1500 .Keyword_define,
1501 .{ .StringLiteral = .None },
1502 .{ .StringLiteral = .None },
1503 });
1504}
1505
1506test "string prefix" {
1507 expectTokens(
1508 \\"foo"
1509 \\u"foo"
1510 \\u8"foo"
1511 \\U"foo"
1512 \\L"foo"
1513 \\'foo'
1514 \\u'foo'
1515 \\U'foo'
1516 \\L'foo'
1517 \\
1518 , &[_]Token.Id{
1519 .{ .StringLiteral = .None },
1520 .Nl,
1521 .{ .StringLiteral = .Utf16 },
1522 .Nl,
1523 .{ .StringLiteral = .Utf8 },
1524 .Nl,
1525 .{ .StringLiteral = .Utf32 },
1526 .Nl,
1527 .{ .StringLiteral = .Wide },
1528 .Nl,
1529 .{ .CharLiteral = .None },
1530 .Nl,
1531 .{ .CharLiteral = .Utf16 },
1532 .Nl,
1533 .{ .CharLiteral = .Utf32 },
1534 .Nl,
1535 .{ .CharLiteral = .Wide },
1536 .Nl,
1537 });
1538}
1539
1540test "num suffixes" {
1541 expectTokens(
1542 \\ 1.0f 1.0L 1.0 .0 1.
1543 \\ 0l 0lu 0ll 0llu 0
1544 \\ 1u 1ul 1ull 1
1545 \\
1546 , &[_]Token.Id{
1547 .{ .FloatLiteral = .F },
1548 .{ .FloatLiteral = .L },
1549 .{ .FloatLiteral = .None },
1550 .{ .FloatLiteral = .None },
1551 .{ .FloatLiteral = .None },
1552 .Nl,
1553 .{ .IntegerLiteral = .L },
1554 .{ .IntegerLiteral = .LU },
1555 .{ .IntegerLiteral = .LL },
1556 .{ .IntegerLiteral = .LLU },
1557 .{ .IntegerLiteral = .None },
1558 .Nl,
1559 .{ .IntegerLiteral = .U },
1560 .{ .IntegerLiteral = .LU },
1561 .{ .IntegerLiteral = .LLU },
1562 .{ .IntegerLiteral = .None },
1563 .Nl,
1564 });
1565}
1566
1567fn expectTokens(source: []const u8, expected_tokens: []const Token.Id) void {
1568 var tokenizer = Tokenizer{
1569 .source = &Source{
1570 .buffer = source,
1571 .file_name = undefined,
1572 .tokens = undefined,
1573 },
1574 };
1575 for (expected_tokens) |expected_token_id| {
1576 const token = tokenizer.next();
1577 if (!std.meta.eql(token.id, expected_token_id)) {
1578 std.debug.panic("expected {}, found {}\n", .{ @tagName(expected_token_id), @tagName(token.id) });
1579 }
1580 }
1581 const last_token = tokenizer.next();
1582 std.testing.expect(last_token.id == .Eof);
1583}