Commit 1fb308ceeea0259ad021d67945ea5adc10960a85 | zig

author	Andrew Kelley <andrew@ziglang.org> 2018-02-09 13:08:02-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2018-02-09 13:08:02-05:00
log	1fb308ceeea0259ad021d67945ea5adc10960a85
tree	eceb252e06a6ed0cc179bc4cdf5a698057da6761
parent	3919afcad26d2359efe52f98cd4f2f0573527369

self hosted compiler: move tokenization and parsing to std lib

12 files changed, 2110 insertions(+), 2102 deletions(-)

CMakeLists.txt
build.zig
src-self-hosted
std
- index.zig
- zig

CMakeLists.txt+4

 @@ -477,6 +477,10 @@ set(ZIG_STD_FILES
     "special/panic.zig"
     "special/test_runner.zig"
     "unicode.zig"
+    "zig/ast.zig"
+    "zig/index.zig"
+    "zig/parser.zig"
+    "zig/tokenizer.zig"
 )
 set(ZIG_C_HEADER_FILES

build.zig-4

 @@ -108,10 +108,6 @@ pub fn build(b: &Builder) !void {
         "std/special/compiler_rt/index.zig", "compiler-rt", "Run the compiler_rt tests",
         with_lldb));
-    test_step.dependOn(tests.addPkgTests(b, test_filter,
-        "src-self-hosted/main.zig", "fmt", "Run the fmt tests",
-        with_lldb));
     test_step.dependOn(tests.addCompareOutputTests(b, test_filter));
     test_step.dependOn(tests.addBuildExampleTests(b, test_filter));
     test_step.dependOn(tests.addCompileErrorTests(b, test_filter));

src-self-hosted/ast.zig deleted-271

...	@@ -1,271 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const ArrayList = std.ArrayList;
4	const Token = @import("tokenizer.zig").Token;
5	const mem = std.mem;
6
7	pub const Node = struct {
8	id: Id,
9
10	pub const Id = enum {
11	Root,
12	VarDecl,
13	Identifier,
14	FnProto,
15	ParamDecl,
16	Block,
17	InfixOp,
18	PrefixOp,
19	IntegerLiteral,
20	FloatLiteral,
21	};
22
23	pub fn iterate(base: &Node, index: usize) ?&Node {
24	return switch (base.id) {
25	Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
26	Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
27	Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
28	Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
29	Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
30	Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
31	Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
32	Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
33	Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
34	Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
35	};
36	}
37
38	pub fn destroy(base: &Node, allocator: &mem.Allocator) void {
39	return switch (base.id) {
40	Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
41	Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
42	Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
43	Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
44	Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
45	Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
46	Id.InfixOp => allocator.destroy(@fieldParentPtr(NodeInfixOp, "base", base)),
47	Id.PrefixOp => allocator.destroy(@fieldParentPtr(NodePrefixOp, "base", base)),
48	Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
49	Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
50	};
51	}
52	};
53
54	pub const NodeRoot = struct {
55	base: Node,
56	decls: ArrayList(&Node),
57
58	pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
59	if (index < self.decls.len) {
60	return self.decls.items[self.decls.len - index - 1];
61	}
62	return null;
63	}
64	};
65
66	pub const NodeVarDecl = struct {
67	base: Node,
68	visib_token: ?Token,
69	name_token: Token,
70	eq_token: Token,
71	mut_token: Token,
72	comptime_token: ?Token,
73	extern_token: ?Token,
74	lib_name: ?&Node,
75	type_node: ?&Node,
76	align_node: ?&Node,
77	init_node: ?&Node,
78
79	pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
80	var i = index;
81
82	if (self.type_node) \|type_node\| {
83	if (i < 1) return type_node;
84	i -= 1;
85	}
86
87	if (self.align_node) \|align_node\| {
88	if (i < 1) return align_node;
89	i -= 1;
90	}
91
92	if (self.init_node) \|init_node\| {
93	if (i < 1) return init_node;
94	i -= 1;
95	}
96
97	return null;
98	}
99	};
100
101	pub const NodeIdentifier = struct {
102	base: Node,
103	name_token: Token,
104
105	pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
106	return null;
107	}
108	};
109
110	pub const NodeFnProto = struct {
111	base: Node,
112	visib_token: ?Token,
113	fn_token: Token,
114	name_token: ?Token,
115	params: ArrayList(&Node),
116	return_type: &Node,
117	var_args_token: ?Token,
118	extern_token: ?Token,
119	inline_token: ?Token,
120	cc_token: ?Token,
121	body_node: ?&Node,
122	lib_name: ?&Node, // populated if this is an extern declaration
123	align_expr: ?&Node, // populated if align(A) is present
124
125	pub fn iterate(self: &NodeFnProto, index: usize) ?&Node {
126	var i = index;
127
128	if (self.body_node) \|body_node\| {
129	if (i < 1) return body_node;
130	i -= 1;
131	}
132
133	if (i < 1) return self.return_type;
134	i -= 1;
135
136	if (self.align_expr) \|align_expr\| {
137	if (i < 1) return align_expr;
138	i -= 1;
139	}
140
141	if (i < self.params.len) return self.params.items[self.params.len - i - 1];
142	i -= self.params.len;
143
144	if (self.lib_name) \|lib_name\| {
145	if (i < 1) return lib_name;
146	i -= 1;
147	}
148
149	return null;
150	}
151	};
152
153	pub const NodeParamDecl = struct {
154	base: Node,
155	comptime_token: ?Token,
156	noalias_token: ?Token,
157	name_token: ?Token,
158	type_node: &Node,
159	var_args_token: ?Token,
160
161	pub fn iterate(self: &NodeParamDecl, index: usize) ?&Node {
162	var i = index;
163
164	if (i < 1) return self.type_node;
165	i -= 1;
166
167	return null;
168	}
169	};
170
171	pub const NodeBlock = struct {
172	base: Node,
173	begin_token: Token,
174	end_token: Token,
175	statements: ArrayList(&Node),
176
177	pub fn iterate(self: &NodeBlock, index: usize) ?&Node {
178	var i = index;
179
180	if (i < self.statements.len) return self.statements.items[i];
181	i -= self.statements.len;
182
183	return null;
184	}
185	};
186
187	pub const NodeInfixOp = struct {
188	base: Node,
189	op_token: Token,
190	lhs: &Node,
191	op: InfixOp,
192	rhs: &Node,
193
194	const InfixOp = enum {
195	EqualEqual,
196	BangEqual,
197	};
198
199	pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
200	var i = index;
201
202	if (i < 1) return self.lhs;
203	i -= 1;
204
205	switch (self.op) {
206	InfixOp.EqualEqual => {},
207	InfixOp.BangEqual => {},
208	}
209
210	if (i < 1) return self.rhs;
211	i -= 1;
212
213	return null;
214	}
215	};
216
217	pub const NodePrefixOp = struct {
218	base: Node,
219	op_token: Token,
220	op: PrefixOp,
221	rhs: &Node,
222
223	const PrefixOp = union(enum) {
224	Return,
225	AddrOf: AddrOfInfo,
226	};
227	const AddrOfInfo = struct {
228	align_expr: ?&Node,
229	bit_offset_start_token: ?Token,
230	bit_offset_end_token: ?Token,
231	const_token: ?Token,
232	volatile_token: ?Token,
233	};
234
235	pub fn iterate(self: &NodePrefixOp, index: usize) ?&Node {
236	var i = index;
237
238	switch (self.op) {
239	PrefixOp.Return => {},
240	PrefixOp.AddrOf => \|addr_of_info\| {
241	if (addr_of_info.align_expr) \|align_expr\| {
242	if (i < 1) return align_expr;
243	i -= 1;
244	}
245	},
246	}
247
248	if (i < 1) return self.rhs;
249	i -= 1;
250
251	return null;
252	}
253	};
254
255	pub const NodeIntegerLiteral = struct {
256	base: Node,
257	token: Token,
258
259	pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
260	return null;
261	}
262	};
263
264	pub const NodeFloatLiteral = struct {
265	base: Node,
266	token: Token,
267
268	pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
269	return null;
270	}
271	};

src-self-hosted/main.zig-5

...	@@ -622,8 +622,3 @@ fn findZigLibDir(allocator: &mem.Allocator) ![]u8 {	...	@@ -622,8 +622,3 @@ fn findZigLibDir(allocator: &mem.Allocator) ![]u8 {
622		622
623	return error.FileNotFound;	623	return error.FileNotFound;
624	}	624	}
625
626	test "import tests" {
627	_ = @import("tokenizer.zig");
628	_ = @import("parser.zig");
629	}

src-self-hosted/module.zig+3-3

...	@@ -8,9 +8,9 @@ const c = @import("c.zig");	...	@@ -8,9 +8,9 @@ const c = @import("c.zig");
8	const builtin = @import("builtin");	8	const builtin = @import("builtin");
9	const Target = @import("target.zig").Target;	9	const Target = @import("target.zig").Target;
10	const warn = std.debug.warn;	10	const warn = std.debug.warn;
11	const Tokenizer = @import("tokenizer.zig").Tokenizer;	11	const Tokenizer = std.zig.Tokenizer;
12	const Token = @import("tokenizer.zig").Token;	12	const Token = std.zig.Token;
13	const Parser = @import("parser.zig").Parser;	13	const Parser = std.zig.Parser;
14	const ArrayList = std.ArrayList;	14	const ArrayList = std.ArrayList;
15		15
16	pub const Module = struct {	16	pub const Module = struct {

src-self-hosted/parser.zig deleted-1160

...	@@ -1,1160 +0,0 @@
1	const std = @import("std");
2	const assert = std.debug.assert;
3	const ArrayList = std.ArrayList;
4	const mem = std.mem;
5	const ast = @import("ast.zig");
6	const Tokenizer = @import("tokenizer.zig").Tokenizer;
7	const Token = @import("tokenizer.zig").Token;
8	const builtin = @import("builtin");
9	const io = std.io;
10
11	// TODO when we make parse errors into error types instead of printing directly,
12	// get rid of this
13	const warn = std.debug.warn;
14
15	pub const Parser = struct {
16	allocator: &mem.Allocator,
17	tokenizer: &Tokenizer,
18	put_back_tokens: [2]Token,
19	put_back_count: usize,
20	source_file_name: []const u8,
21
22	pub const Tree = struct {
23	root_node: &ast.NodeRoot,
24
25	pub fn deinit(self: &const Tree) void {
26	// TODO free the whole arena
27	}
28	};
29
30	// This memory contents are used only during a function call. It's used to repurpose memory;
31	// we reuse the same bytes for the stack data structure used by parsing, tree rendering, and
32	// source rendering.
33	const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
34	utility_bytes: []align(utility_bytes_align) u8,
35
36	/// `allocator` should be an arena allocator. Parser never calls free on anything. After you're
37	/// done with a Parser, free the arena. After the arena is freed, no member functions of Parser
38	/// may be called.
39	pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) Parser {
40	return Parser {
41	.allocator = allocator,
42	.tokenizer = tokenizer,
43	.put_back_tokens = undefined,
44	.put_back_count = 0,
45	.source_file_name = source_file_name,
46	.utility_bytes = []align(utility_bytes_align) u8{},
47	};
48	}
49
50	pub fn deinit(self: &Parser) void {
51	self.allocator.free(self.utility_bytes);
52	}
53
54	const TopLevelDeclCtx = struct {
55	visib_token: ?Token,
56	extern_token: ?Token,
57	};
58
59	const DestPtr = union(enum) {
60	Field: &&ast.Node,
61	NullableField: &?&ast.Node,
62	List: &ArrayList(&ast.Node),
63
64	pub fn store(self: &const DestPtr, value: &ast.Node) !void {
65	switch (*self) {
66	DestPtr.Field => \|ptr\| *ptr = value,
67	DestPtr.NullableField => \|ptr\| *ptr = value,
68	DestPtr.List => \|list\| try list.append(value),
69	}
70	}
71	};
72
73	const State = union(enum) {
74	TopLevel,
75	TopLevelExtern: ?Token,
76	TopLevelDecl: TopLevelDeclCtx,
77	Expression: DestPtr,
78	ExpectOperand,
79	Operand: &ast.Node,
80	AfterOperand,
81	InfixOp: &ast.NodeInfixOp,
82	PrefixOp: &ast.NodePrefixOp,
83	AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
84	TypeExpr: DestPtr,
85	VarDecl: &ast.NodeVarDecl,
86	VarDeclAlign: &ast.NodeVarDecl,
87	VarDeclEq: &ast.NodeVarDecl,
88	ExpectToken: @TagType(Token.Id),
89	FnProto: &ast.NodeFnProto,
90	FnProtoAlign: &ast.NodeFnProto,
91	ParamDecl: &ast.NodeFnProto,
92	ParamDeclComma,
93	FnDef: &ast.NodeFnProto,
94	Block: &ast.NodeBlock,
95	Statement: &ast.NodeBlock,
96	};
97
98	/// Returns an AST tree, allocated with the parser's allocator.
99	/// Result should be freed with `freeAst` when done.
100	pub fn parse(self: &Parser) !Tree {
101	var stack = self.initUtilityArrayList(State);
102	defer self.deinitUtilityArrayList(stack);
103
104	const root_node = try self.createRoot();
105	// TODO errdefer arena free root node
106
107	try stack.append(State.TopLevel);
108
109	while (true) {
110	//{
111	// const token = self.getNextToken();
112	// warn("{} ", @tagName(token.id));
113	// self.putBackToken(token);
114	// var i: usize = stack.len;
115	// while (i != 0) {
116	// i -= 1;
117	// warn("{} ", @tagName(stack.items[i]));
118	// }
119	// warn("\n");
120	//}
121
122	// This gives us 1 free append that can't fail
123	const state = stack.pop();
124
125	switch (state) {
126	State.TopLevel => {
127	const token = self.getNextToken();
128	switch (token.id) {
129	Token.Id.Keyword_pub, Token.Id.Keyword_export => {
130	stack.append(State { .TopLevelExtern = token }) catch unreachable;
131	continue;
132	},
133	Token.Id.Eof => return Tree {.root_node = root_node},
134	else => {
135	self.putBackToken(token);
136	// TODO shouldn't need this cast
137	stack.append(State { .TopLevelExtern = null }) catch unreachable;
138	continue;
139	},
140	}
141	},
142	State.TopLevelExtern => \|visib_token\| {
143	const token = self.getNextToken();
144	if (token.id == Token.Id.Keyword_extern) {
145	stack.append(State {
146	.TopLevelDecl = TopLevelDeclCtx {
147	.visib_token = visib_token,
148	.extern_token = token,
149	},
150	}) catch unreachable;
151	continue;
152	}
153	self.putBackToken(token);
154	stack.append(State {
155	.TopLevelDecl = TopLevelDeclCtx {
156	.visib_token = visib_token,
157	.extern_token = null,
158	},
159	}) catch unreachable;
160	continue;
161	},
162	State.TopLevelDecl => \|ctx\| {
163	const token = self.getNextToken();
164	switch (token.id) {
165	Token.Id.Keyword_var, Token.Id.Keyword_const => {
166	stack.append(State.TopLevel) catch unreachable;
167	// TODO shouldn't need these casts
168	const var_decl_node = try self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
169	token, (?Token)(null), ctx.extern_token);
170	try stack.append(State { .VarDecl = var_decl_node });
171	continue;
172	},
173	Token.Id.Keyword_fn => {
174	stack.append(State.TopLevel) catch unreachable;
175	// TODO shouldn't need these casts
176	const fn_proto = try self.createAttachFnProto(&root_node.decls, token,
177	ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
178	try stack.append(State { .FnDef = fn_proto });
179	try stack.append(State { .FnProto = fn_proto });
180	continue;
181	},
182	Token.Id.StringLiteral => {
183	@panic("TODO extern with string literal");
184	},
185	Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
186	stack.append(State.TopLevel) catch unreachable;
187	const fn_token = try self.eatToken(Token.Id.Keyword_fn);
188	// TODO shouldn't need this cast
189	const fn_proto = try self.createAttachFnProto(&root_node.decls, fn_token,
190	ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
191	try stack.append(State { .FnDef = fn_proto });
192	try stack.append(State { .FnProto = fn_proto });
193	continue;
194	},
195	else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
196	}
197	},
198	State.VarDecl => \|var_decl\| {
199	var_decl.name_token = try self.eatToken(Token.Id.Identifier);
200	stack.append(State { .VarDeclAlign = var_decl }) catch unreachable;
201
202	const next_token = self.getNextToken();
203	if (next_token.id == Token.Id.Colon) {
204	try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
205	continue;
206	}
207
208	self.putBackToken(next_token);
209	continue;
210	},
211	State.VarDeclAlign => \|var_decl\| {
212	stack.append(State { .VarDeclEq = var_decl }) catch unreachable;
213
214	const next_token = self.getNextToken();
215	if (next_token.id == Token.Id.Keyword_align) {
216	_ = try self.eatToken(Token.Id.LParen);
217	try stack.append(State { .ExpectToken = Token.Id.RParen });
218	try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} });
219	continue;
220	}
221
222	self.putBackToken(next_token);
223	continue;
224	},
225	State.VarDeclEq => \|var_decl\| {
226	const token = self.getNextToken();
227	if (token.id == Token.Id.Equal) {
228	var_decl.eq_token = token;
229	stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
230	try stack.append(State {
231	.Expression = DestPtr {.NullableField = &var_decl.init_node},
232	});
233	continue;
234	}
235	if (token.id == Token.Id.Semicolon) {
236	continue;
237	}
238	return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
239	},
240	State.ExpectToken => \|token_id\| {
241	_ = try self.eatToken(token_id);
242	continue;
243	},
244
245	State.Expression => \|dest_ptr\| {
246	// save the dest_ptr for later
247	stack.append(state) catch unreachable;
248	try stack.append(State.ExpectOperand);
249	continue;
250	},
251	State.ExpectOperand => {
252	// we'll either get an operand (like 1 or x),
253	// or a prefix operator (like ~ or return).
254	const token = self.getNextToken();
255	switch (token.id) {
256	Token.Id.Keyword_return => {
257	try stack.append(State { .PrefixOp = try self.createPrefixOp(token,
258	ast.NodePrefixOp.PrefixOp.Return) });
259	try stack.append(State.ExpectOperand);
260	continue;
261	},
262	Token.Id.Ampersand => {
263	const prefix_op = try self.createPrefixOp(token, ast.NodePrefixOp.PrefixOp{
264	.AddrOf = ast.NodePrefixOp.AddrOfInfo {
265	.align_expr = null,
266	.bit_offset_start_token = null,
267	.bit_offset_end_token = null,
268	.const_token = null,
269	.volatile_token = null,
270	}
271	});
272	try stack.append(State { .PrefixOp = prefix_op });
273	try stack.append(State.ExpectOperand);
274	try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf });
275	continue;
276	},
277	Token.Id.Identifier => {
278	try stack.append(State {
279	.Operand = &(try self.createIdentifier(token)).base
280	});
281	try stack.append(State.AfterOperand);
282	continue;
283	},
284	Token.Id.IntegerLiteral => {
285	try stack.append(State {
286	.Operand = &(try self.createIntegerLiteral(token)).base
287	});
288	try stack.append(State.AfterOperand);
289	continue;
290	},
291	Token.Id.FloatLiteral => {
292	try stack.append(State {
293	.Operand = &(try self.createFloatLiteral(token)).base
294	});
295	try stack.append(State.AfterOperand);
296	continue;
297	},
298	else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
299	}
300	},
301
302	State.AfterOperand => {
303	// we'll either get an infix operator (like != or ^),
304	// or a postfix operator (like () or {}),
305	// otherwise this expression is done (like on a ; or else).
306	var token = self.getNextToken();
307	switch (token.id) {
308	Token.Id.EqualEqual => {
309	try stack.append(State {
310	.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.EqualEqual)
311	});
312	try stack.append(State.ExpectOperand);
313	continue;
314	},
315	Token.Id.BangEqual => {
316	try stack.append(State {
317	.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.BangEqual)
318	});
319	try stack.append(State.ExpectOperand);
320	continue;
321	},
322	else => {
323	// no postfix/infix operator after this operand.
324	self.putBackToken(token);
325	// reduce the stack
326	var expression: &ast.Node = stack.pop().Operand;
327	while (true) {
328	switch (stack.pop()) {
329	State.Expression => \|dest_ptr\| {
330	// we're done
331	try dest_ptr.store(expression);
332	break;
333	},
334	State.InfixOp => \|infix_op\| {
335	infix_op.rhs = expression;
336	infix_op.lhs = stack.pop().Operand;
337	expression = &infix_op.base;
338	continue;
339	},
340	State.PrefixOp => \|prefix_op\| {
341	prefix_op.rhs = expression;
342	expression = &prefix_op.base;
343	continue;
344	},
345	else => unreachable,
346	}
347	}
348	continue;
349	},
350	}
351	},
352
353	State.AddrOfModifiers => \|addr_of_info\| {
354	var token = self.getNextToken();
355	switch (token.id) {
356	Token.Id.Keyword_align => {
357	stack.append(state) catch unreachable;
358	if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers");
359	_ = try self.eatToken(Token.Id.LParen);
360	try stack.append(State { .ExpectToken = Token.Id.RParen });
361	try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} });
362	continue;
363	},
364	Token.Id.Keyword_const => {
365	stack.append(state) catch unreachable;
366	if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const");
367	addr_of_info.const_token = token;
368	continue;
369	},
370	Token.Id.Keyword_volatile => {
371	stack.append(state) catch unreachable;
372	if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile");
373	addr_of_info.volatile_token = token;
374	continue;
375	},
376	else => {
377	self.putBackToken(token);
378	continue;
379	},
380	}
381	},
382
383	State.TypeExpr => \|dest_ptr\| {
384	const token = self.getNextToken();
385	if (token.id == Token.Id.Keyword_var) {
386	@panic("TODO param with type var");
387	}
388	self.putBackToken(token);
389
390	stack.append(State { .Expression = dest_ptr }) catch unreachable;
391	continue;
392	},
393
394	State.FnProto => \|fn_proto\| {
395	stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable;
396	try stack.append(State { .ParamDecl = fn_proto });
397	try stack.append(State { .ExpectToken = Token.Id.LParen });
398
399	const next_token = self.getNextToken();
400	if (next_token.id == Token.Id.Identifier) {
401	fn_proto.name_token = next_token;
402	continue;
403	}
404	self.putBackToken(next_token);
405	continue;
406	},
407
408	State.FnProtoAlign => \|fn_proto\| {
409	const token = self.getNextToken();
410	if (token.id == Token.Id.Keyword_align) {
411	@panic("TODO fn proto align");
412	}
413	self.putBackToken(token);
414	stack.append(State {
415	.TypeExpr = DestPtr {.Field = &fn_proto.return_type},
416	}) catch unreachable;
417	continue;
418	},
419
420	State.ParamDecl => \|fn_proto\| {
421	var token = self.getNextToken();
422	if (token.id == Token.Id.RParen) {
423	continue;
424	}
425	const param_decl = try self.createAttachParamDecl(&fn_proto.params);
426	if (token.id == Token.Id.Keyword_comptime) {
427	param_decl.comptime_token = token;
428	token = self.getNextToken();
429	} else if (token.id == Token.Id.Keyword_noalias) {
430	param_decl.noalias_token = token;
431	token = self.getNextToken();
432	}
433	if (token.id == Token.Id.Identifier) {
434	const next_token = self.getNextToken();
435	if (next_token.id == Token.Id.Colon) {
436	param_decl.name_token = token;
437	token = self.getNextToken();
438	} else {
439	self.putBackToken(next_token);
440	}
441	}
442	if (token.id == Token.Id.Ellipsis3) {
443	param_decl.var_args_token = token;
444	stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable;
445	continue;
446	} else {
447	self.putBackToken(token);
448	}
449
450	stack.append(State { .ParamDecl = fn_proto }) catch unreachable;
451	try stack.append(State.ParamDeclComma);
452	try stack.append(State {
453	.TypeExpr = DestPtr {.Field = &param_decl.type_node}
454	});
455	continue;
456	},
457
458	State.ParamDeclComma => {
459	const token = self.getNextToken();
460	switch (token.id) {
461	Token.Id.RParen => {
462	_ = stack.pop(); // pop off the ParamDecl
463	continue;
464	},
465	Token.Id.Comma => continue,
466	else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
467	}
468	},
469
470	State.FnDef => \|fn_proto\| {
471	const token = self.getNextToken();
472	switch(token.id) {
473	Token.Id.LBrace => {
474	const block = try self.createBlock(token);
475	fn_proto.body_node = &block.base;
476	stack.append(State { .Block = block }) catch unreachable;
477	continue;
478	},
479	Token.Id.Semicolon => continue,
480	else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
481	}
482	},
483
484	State.Block => \|block\| {
485	const token = self.getNextToken();
486	switch (token.id) {
487	Token.Id.RBrace => {
488	block.end_token = token;
489	continue;
490	},
491	else => {
492	self.putBackToken(token);
493	stack.append(State { .Block = block }) catch unreachable;
494	try stack.append(State { .Statement = block });
495	continue;
496	},
497	}
498	},
499
500	State.Statement => \|block\| {
501	{
502	// Look for comptime var, comptime const
503	const comptime_token = self.getNextToken();
504	if (comptime_token.id == Token.Id.Keyword_comptime) {
505	const mut_token = self.getNextToken();
506	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
507	// TODO shouldn't need these casts
508	const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
509	mut_token, (?Token)(comptime_token), (?Token)(null));
510	try stack.append(State { .VarDecl = var_decl });
511	continue;
512	}
513	self.putBackToken(mut_token);
514	}
515	self.putBackToken(comptime_token);
516	}
517	{
518	// Look for const, var
519	const mut_token = self.getNextToken();
520	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
521	// TODO shouldn't need these casts
522	const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
523	mut_token, (?Token)(null), (?Token)(null));
524	try stack.append(State { .VarDecl = var_decl });
525	continue;
526	}
527	self.putBackToken(mut_token);
528	}
529
530	stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
531	try stack.append(State { .Expression = DestPtr{.List = &block.statements} });
532	continue;
533	},
534
535	// These are data, not control flow.
536	State.InfixOp => unreachable,
537	State.PrefixOp => unreachable,
538	State.Operand => unreachable,
539	}
540	@import("std").debug.panic("{}", @tagName(state));
541	//unreachable;
542	}
543	}
544
545	fn createRoot(self: &Parser) !&ast.NodeRoot {
546	const node = try self.allocator.create(ast.NodeRoot);
547
548	*node = ast.NodeRoot {
549	.base = ast.Node {.id = ast.Node.Id.Root},
550	.decls = ArrayList(&ast.Node).init(self.allocator),
551	};
552	return node;
553	}
554
555	fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
556	extern_token: &const ?Token) !&ast.NodeVarDecl
557	{
558	const node = try self.allocator.create(ast.NodeVarDecl);
559
560	*node = ast.NodeVarDecl {
561	.base = ast.Node {.id = ast.Node.Id.VarDecl},
562	.visib_token = *visib_token,
563	.mut_token = *mut_token,
564	.comptime_token = *comptime_token,
565	.extern_token = *extern_token,
566	.type_node = null,
567	.align_node = null,
568	.init_node = null,
569	.lib_name = null,
570	// initialized later
571	.name_token = undefined,
572	.eq_token = undefined,
573	};
574	return node;
575	}
576
577	fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
578	cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto
579	{
580	const node = try self.allocator.create(ast.NodeFnProto);
581
582	*node = ast.NodeFnProto {
583	.base = ast.Node {.id = ast.Node.Id.FnProto},
584	.visib_token = *visib_token,
585	.name_token = null,
586	.fn_token = *fn_token,
587	.params = ArrayList(&ast.Node).init(self.allocator),
588	.return_type = undefined,
589	.var_args_token = null,
590	.extern_token = *extern_token,
591	.inline_token = *inline_token,
592	.cc_token = *cc_token,
593	.body_node = null,
594	.lib_name = null,
595	.align_expr = null,
596	};
597	return node;
598	}
599
600	fn createParamDecl(self: &Parser) !&ast.NodeParamDecl {
601	const node = try self.allocator.create(ast.NodeParamDecl);
602
603	*node = ast.NodeParamDecl {
604	.base = ast.Node {.id = ast.Node.Id.ParamDecl},
605	.comptime_token = null,
606	.noalias_token = null,
607	.name_token = null,
608	.type_node = undefined,
609	.var_args_token = null,
610	};
611	return node;
612	}
613
614	fn createBlock(self: &Parser, begin_token: &const Token) !&ast.NodeBlock {
615	const node = try self.allocator.create(ast.NodeBlock);
616
617	*node = ast.NodeBlock {
618	.base = ast.Node {.id = ast.Node.Id.Block},
619	.begin_token = *begin_token,
620	.end_token = undefined,
621	.statements = ArrayList(&ast.Node).init(self.allocator),
622	};
623	return node;
624	}
625
626	fn createInfixOp(self: &Parser, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp {
627	const node = try self.allocator.create(ast.NodeInfixOp);
628
629	*node = ast.NodeInfixOp {
630	.base = ast.Node {.id = ast.Node.Id.InfixOp},
631	.op_token = *op_token,
632	.lhs = undefined,
633	.op = *op,
634	.rhs = undefined,
635	};
636	return node;
637	}
638
639	fn createPrefixOp(self: &Parser, op_token: &const Token, op: &const ast.NodePrefixOp.PrefixOp) !&ast.NodePrefixOp {
640	const node = try self.allocator.create(ast.NodePrefixOp);
641
642	*node = ast.NodePrefixOp {
643	.base = ast.Node {.id = ast.Node.Id.PrefixOp},
644	.op_token = *op_token,
645	.op = *op,
646	.rhs = undefined,
647	};
648	return node;
649	}
650
651	fn createIdentifier(self: &Parser, name_token: &const Token) !&ast.NodeIdentifier {
652	const node = try self.allocator.create(ast.NodeIdentifier);
653
654	*node = ast.NodeIdentifier {
655	.base = ast.Node {.id = ast.Node.Id.Identifier},
656	.name_token = *name_token,
657	};
658	return node;
659	}
660
661	fn createIntegerLiteral(self: &Parser, token: &const Token) !&ast.NodeIntegerLiteral {
662	const node = try self.allocator.create(ast.NodeIntegerLiteral);
663
664	*node = ast.NodeIntegerLiteral {
665	.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
666	.token = *token,
667	};
668	return node;
669	}
670
671	fn createFloatLiteral(self: &Parser, token: &const Token) !&ast.NodeFloatLiteral {
672	const node = try self.allocator.create(ast.NodeFloatLiteral);
673
674	*node = ast.NodeFloatLiteral {
675	.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
676	.token = *token,
677	};
678	return node;
679	}
680
681	fn createAttachIdentifier(self: &Parser, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {
682	const node = try self.createIdentifier(name_token);
683	try dest_ptr.store(&node.base);
684	return node;
685	}
686
687	fn createAttachParamDecl(self: &Parser, list: &ArrayList(&ast.Node)) !&ast.NodeParamDecl {
688	const node = try self.createParamDecl();
689	try list.append(&node.base);
690	return node;
691	}
692
693	fn createAttachFnProto(self: &Parser, list: &ArrayList(&ast.Node), fn_token: &const Token,
694	extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
695	inline_token: &const ?Token) !&ast.NodeFnProto
696	{
697	const node = try self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
698	try list.append(&node.base);
699	return node;
700	}
701
702	fn createAttachVarDecl(self: &Parser, list: &ArrayList(&ast.Node), visib_token: &const ?Token,
703	mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl
704	{
705	const node = try self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
706	try list.append(&node.base);
707	return node;
708	}
709
710	fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) error {
711	const loc = self.tokenizer.getTokenLocation(token);
712	warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
713	warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
714	{
715	var i: usize = 0;
716	while (i < loc.column) : (i += 1) {
717	warn(" ");
718	}
719	}
720	{
721	const caret_count = token.end - token.start;
722	var i: usize = 0;
723	while (i < caret_count) : (i += 1) {
724	warn("~");
725	}
726	}
727	warn("\n");
728	return error.ParseError;
729	}
730
731	fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void {
732	if (token.id != id) {
733	return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
734	}
735	}
736
737	fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token {
738	const token = self.getNextToken();
739	try self.expectToken(token, id);
740	return token;
741	}
742
743	fn putBackToken(self: &Parser, token: &const Token) void {
744	self.put_back_tokens[self.put_back_count] = *token;
745	self.put_back_count += 1;
746	}
747
748	fn getNextToken(self: &Parser) Token {
749	if (self.put_back_count != 0) {
750	const put_back_index = self.put_back_count - 1;
751	const put_back_token = self.put_back_tokens[put_back_index];
752	self.put_back_count = put_back_index;
753	return put_back_token;
754	} else {
755	return self.tokenizer.next();
756	}
757	}
758
759	const RenderAstFrame = struct {
760	node: &ast.Node,
761	indent: usize,
762	};
763
764	pub fn renderAst(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
765	var stack = self.initUtilityArrayList(RenderAstFrame);
766	defer self.deinitUtilityArrayList(stack);
767
768	try stack.append(RenderAstFrame {
769	.node = &root_node.base,
770	.indent = 0,
771	});
772
773	while (stack.popOrNull()) \|frame\| {
774	{
775	var i: usize = 0;
776	while (i < frame.indent) : (i += 1) {
777	try stream.print(" ");
778	}
779	}
780	try stream.print("{}\n", @tagName(frame.node.id));
781	var child_i: usize = 0;
782	while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
783	try stack.append(RenderAstFrame {
784	.node = child,
785	.indent = frame.indent + 2,
786	});
787	}
788	}
789	}
790
791	const RenderState = union(enum) {
792	TopLevelDecl: &ast.Node,
793	FnProtoRParen: &ast.NodeFnProto,
794	ParamDecl: &ast.Node,
795	Text: []const u8,
796	Expression: &ast.Node,
797	VarDecl: &ast.NodeVarDecl,
798	Statement: &ast.Node,
799	PrintIndent,
800	Indent: usize,
801	};
802
803	pub fn renderSource(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
804	var stack = self.initUtilityArrayList(RenderState);
805	defer self.deinitUtilityArrayList(stack);
806
807	{
808	var i = root_node.decls.len;
809	while (i != 0) {
810	i -= 1;
811	const decl = root_node.decls.items[i];
812	try stack.append(RenderState {.TopLevelDecl = decl});
813	}
814	}
815
816	const indent_delta = 4;
817	var indent: usize = 0;
818	while (stack.popOrNull()) \|state\| {
819	switch (state) {
820	RenderState.TopLevelDecl => \|decl\| {
821	switch (decl.id) {
822	ast.Node.Id.FnProto => {
823	const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl);
824	if (fn_proto.visib_token) \|visib_token\| {
825	switch (visib_token.id) {
826	Token.Id.Keyword_pub => try stream.print("pub "),
827	Token.Id.Keyword_export => try stream.print("export "),
828	else => unreachable,
829	}
830	}
831	if (fn_proto.extern_token) \|extern_token\| {
832	try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
833	}
834	try stream.print("fn");
835
836	if (fn_proto.name_token) \|name_token\| {
837	try stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
838	}
839
840	try stream.print("(");
841
842	try stack.append(RenderState { .Text = "\n" });
843	if (fn_proto.body_node == null) {
844	try stack.append(RenderState { .Text = ";" });
845	}
846
847	try stack.append(RenderState { .FnProtoRParen = fn_proto});
848	var i = fn_proto.params.len;
849	while (i != 0) {
850	i -= 1;
851	const param_decl_node = fn_proto.params.items[i];
852	try stack.append(RenderState { .ParamDecl = param_decl_node});
853	if (i != 0) {
854	try stack.append(RenderState { .Text = ", " });
855	}
856	}
857	},
858	ast.Node.Id.VarDecl => {
859	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
860	try stack.append(RenderState { .Text = "\n"});
861	try stack.append(RenderState { .VarDecl = var_decl});
862
863	},
864	else => unreachable,
865	}
866	},
867
868	RenderState.VarDecl => \|var_decl\| {
869	if (var_decl.visib_token) \|visib_token\| {
870	try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
871	}
872	if (var_decl.extern_token) \|extern_token\| {
873	try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
874	if (var_decl.lib_name != null) {
875	@panic("TODO");
876	}
877	}
878	if (var_decl.comptime_token) \|comptime_token\| {
879	try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
880	}
881	try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
882	try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
883
884	try stack.append(RenderState { .Text = ";" });
885	if (var_decl.init_node) \|init_node\| {
886	try stack.append(RenderState { .Expression = init_node });
887	try stack.append(RenderState { .Text = " = " });
888	}
889	if (var_decl.align_node) \|align_node\| {
890	try stack.append(RenderState { .Text = ")" });
891	try stack.append(RenderState { .Expression = align_node });
892	try stack.append(RenderState { .Text = " align(" });
893	}
894	if (var_decl.type_node) \|type_node\| {
895	try stream.print(": ");
896	try stack.append(RenderState { .Expression = type_node });
897	}
898	},
899
900	RenderState.ParamDecl => \|base\| {
901	const param_decl = @fieldParentPtr(ast.NodeParamDecl, "base", base);
902	if (param_decl.comptime_token) \|comptime_token\| {
903	try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
904	}
905	if (param_decl.noalias_token) \|noalias_token\| {
906	try stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
907	}
908	if (param_decl.name_token) \|name_token\| {
909	try stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
910	}
911	if (param_decl.var_args_token) \|var_args_token\| {
912	try stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
913	} else {
914	try stack.append(RenderState { .Expression = param_decl.type_node});
915	}
916	},
917	RenderState.Text => \|bytes\| {
918	try stream.write(bytes);
919	},
920	RenderState.Expression => \|base\| switch (base.id) {
921	ast.Node.Id.Identifier => {
922	const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base);
923	try stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
924	},
925	ast.Node.Id.Block => {
926	const block = @fieldParentPtr(ast.NodeBlock, "base", base);
927	try stream.write("{");
928	try stack.append(RenderState { .Text = "}"});
929	try stack.append(RenderState.PrintIndent);
930	try stack.append(RenderState { .Indent = indent});
931	try stack.append(RenderState { .Text = "\n"});
932	var i = block.statements.len;
933	while (i != 0) {
934	i -= 1;
935	const statement_node = block.statements.items[i];
936	try stack.append(RenderState { .Statement = statement_node});
937	try stack.append(RenderState.PrintIndent);
938	try stack.append(RenderState { .Indent = indent + indent_delta});
939	try stack.append(RenderState { .Text = "\n" });
940	}
941	},
942	ast.Node.Id.InfixOp => {
943	const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);
944	try stack.append(RenderState { .Expression = prefix_op_node.rhs });
945	switch (prefix_op_node.op) {
946	ast.NodeInfixOp.InfixOp.EqualEqual => {
947	try stack.append(RenderState { .Text = " == "});
948	},
949	ast.NodeInfixOp.InfixOp.BangEqual => {
950	try stack.append(RenderState { .Text = " != "});
951	},
952	else => unreachable,
953	}
954	try stack.append(RenderState { .Expression = prefix_op_node.lhs });
955	},
956	ast.Node.Id.PrefixOp => {
957	const prefix_op_node = @fieldParentPtr(ast.NodePrefixOp, "base", base);
958	try stack.append(RenderState { .Expression = prefix_op_node.rhs });
959	switch (prefix_op_node.op) {
960	ast.NodePrefixOp.PrefixOp.Return => {
961	try stream.write("return ");
962	},
963	ast.NodePrefixOp.PrefixOp.AddrOf => \|addr_of_info\| {
964	try stream.write("&");
965	if (addr_of_info.volatile_token != null) {
966	try stack.append(RenderState { .Text = "volatile "});
967	}
968	if (addr_of_info.const_token != null) {
969	try stack.append(RenderState { .Text = "const "});
970	}
971	if (addr_of_info.align_expr) \|align_expr\| {
972	try stream.print("align(");
973	try stack.append(RenderState { .Text = ") "});
974	try stack.append(RenderState { .Expression = align_expr});
975	}
976	},
977	else => unreachable,
978	}
979	},
980	ast.Node.Id.IntegerLiteral => {
981	const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
982	try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));
983	},
984	ast.Node.Id.FloatLiteral => {
985	const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base);
986	try stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token));
987	},
988	else => unreachable,
989	},
990	RenderState.FnProtoRParen => \|fn_proto\| {
991	try stream.print(")");
992	if (fn_proto.align_expr != null) {
993	@panic("TODO");
994	}
995	try stream.print(" ");
996	if (fn_proto.body_node) \|body_node\| {
997	try stack.append(RenderState { .Expression = body_node});
998	try stack.append(RenderState { .Text = " "});
999	}
1000	try stack.append(RenderState { .Expression = fn_proto.return_type});
1001	},
1002	RenderState.Statement => \|base\| {
1003	switch (base.id) {
1004	ast.Node.Id.VarDecl => {
1005	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
1006	try stack.append(RenderState { .VarDecl = var_decl});
1007	},
1008	else => {
1009	try stack.append(RenderState { .Text = ";"});
1010	try stack.append(RenderState { .Expression = base});
1011	},
1012	}
1013	},
1014	RenderState.Indent => \|new_indent\| indent = new_indent,
1015	RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent),
1016	}
1017	}
1018	}
1019
1020	fn initUtilityArrayList(self: &Parser, comptime T: type) ArrayList(T) {
1021	const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
1022	self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
1023	const typed_slice = ([]T)(self.utility_bytes);
1024	return ArrayList(T) {
1025	.allocator = self.allocator,
1026	.items = typed_slice,
1027	.len = 0,
1028	};
1029	}
1030
1031	fn deinitUtilityArrayList(self: &Parser, list: var) void {
1032	self.utility_bytes = ([]align(utility_bytes_align) u8)(list.items);
1033	}
1034
1035	};
1036
1037	var fixed_buffer_mem: [100 * 1024]u8 = undefined;
1038
1039	fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
1040	var padded_source: [0x100]u8 = undefined;
1041	std.mem.copy(u8, padded_source[0..source.len], source);
1042	padded_source[source.len + 0] = '\n';
1043	padded_source[source.len + 1] = '\n';
1044	padded_source[source.len + 2] = '\n';
1045
1046	var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
1047	var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
1048	defer parser.deinit();
1049
1050	const tree = try parser.parse();
1051	defer tree.deinit();
1052
1053	var buffer = try std.Buffer.initSize(allocator, 0);
1054	var buffer_out_stream = io.BufferOutStream.init(&buffer);
1055	try parser.renderSource(&buffer_out_stream.stream, tree.root_node);
1056	return buffer.toOwnedSlice();
1057	}
1058
1059	// TODO test for memory leaks
1060	// TODO test for valid frees
1061	fn testCanonical(source: []const u8) !void {
1062	const needed_alloc_count = x: {
1063	// Try it once with unlimited memory, make sure it works
1064	var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
1065	var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize));
1066	const result_source = try testParse(source, &failing_allocator.allocator);
1067	if (!mem.eql(u8, result_source, source)) {
1068	warn("\n====== expected this output: =========\n");
1069	warn("{}", source);
1070	warn("\n======== instead found this: =========\n");
1071	warn("{}", result_source);
1072	warn("\n======================================\n");
1073	return error.TestFailed;
1074	}
1075	failing_allocator.allocator.free(result_source);
1076	break :x failing_allocator.index;
1077	};
1078
1079	var fail_index: usize = 0;
1080	while (fail_index < needed_alloc_count) : (fail_index += 1) {
1081	var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
1082	var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index);
1083	if (testParse(source, &failing_allocator.allocator)) \|_\| {
1084	return error.NondeterministicMemoryUsage;
1085	} else \|err\| {
1086	assert(err == error.OutOfMemory);
1087	// TODO make this pass
1088	//if (failing_allocator.allocated_bytes != failing_allocator.freed_bytes) {
1089	// warn("\nfail_index: {}/{}\nallocated bytes: {}\nfreed bytes: {}\nallocations: {}\ndeallocations: {}\n",
1090	// fail_index, needed_alloc_count,
1091	// failing_allocator.allocated_bytes, failing_allocator.freed_bytes,
1092	// failing_allocator.index, failing_allocator.deallocations);
1093	// return error.MemoryLeakDetected;
1094	//}
1095	}
1096	}
1097	}
1098
1099	test "zig fmt" {
1100	try testCanonical(
1101	\\extern fn puts(s: &const u8) c_int;
1102	\\
1103	);
1104
1105	try testCanonical(
1106	\\const a = b;
1107	\\pub const a = b;
1108	\\var a = b;
1109	\\pub var a = b;
1110	\\const a: i32 = b;
1111	\\pub const a: i32 = b;
1112	\\var a: i32 = b;
1113	\\pub var a: i32 = b;
1114	\\
1115	);
1116
1117	try testCanonical(
1118	\\extern var foo: c_int;
1119	\\
1120	);
1121
1122	try testCanonical(
1123	\\var foo: c_int align(1);
1124	\\
1125	);
1126
1127	try testCanonical(
1128	\\fn main(argc: c_int, argv: &&u8) c_int {
1129	\\ const a = b;
1130	\\}
1131	\\
1132	);
1133
1134	try testCanonical(
1135	\\fn foo(argc: c_int, argv: &&u8) c_int {
1136	\\ return 0;
1137	\\}
1138	\\
1139	);
1140
1141	try testCanonical(
1142	\\extern fn f1(s: &align(&u8) u8) c_int;
1143	\\
1144	);
1145
1146	try testCanonical(
1147	\\extern fn f1(s: &&align(1) &const &volatile u8) c_int;
1148	\\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int;
1149	\\extern fn f3(s: &align(1) const volatile u8) c_int;
1150	\\
1151	);
1152
1153	try testCanonical(
1154	\\fn f1(a: bool, b: bool) bool {
1155	\\ a != b;
1156	\\ return a == b;
1157	\\}
1158	\\
1159	);
1160	}

src-self-hosted/tokenizer.zig deleted-659

...	@@ -1,659 +0,0 @@
1	const std = @import("std");
2	const mem = std.mem;
3
4	pub const Token = struct {
5	id: Id,
6	start: usize,
7	end: usize,
8
9	const KeywordId = struct {
10	bytes: []const u8,
11	id: Id,
12	};
13
14	const keywords = []KeywordId {
15	KeywordId{.bytes="align", .id = Id.Keyword_align},
16	KeywordId{.bytes="and", .id = Id.Keyword_and},
17	KeywordId{.bytes="asm", .id = Id.Keyword_asm},
18	KeywordId{.bytes="break", .id = Id.Keyword_break},
19	KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
20	KeywordId{.bytes="const", .id = Id.Keyword_const},
21	KeywordId{.bytes="continue", .id = Id.Keyword_continue},
22	KeywordId{.bytes="defer", .id = Id.Keyword_defer},
23	KeywordId{.bytes="else", .id = Id.Keyword_else},
24	KeywordId{.bytes="enum", .id = Id.Keyword_enum},
25	KeywordId{.bytes="error", .id = Id.Keyword_error},
26	KeywordId{.bytes="export", .id = Id.Keyword_export},
27	KeywordId{.bytes="extern", .id = Id.Keyword_extern},
28	KeywordId{.bytes="false", .id = Id.Keyword_false},
29	KeywordId{.bytes="fn", .id = Id.Keyword_fn},
30	KeywordId{.bytes="for", .id = Id.Keyword_for},
31	KeywordId{.bytes="goto", .id = Id.Keyword_goto},
32	KeywordId{.bytes="if", .id = Id.Keyword_if},
33	KeywordId{.bytes="inline", .id = Id.Keyword_inline},
34	KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
35	KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
36	KeywordId{.bytes="null", .id = Id.Keyword_null},
37	KeywordId{.bytes="or", .id = Id.Keyword_or},
38	KeywordId{.bytes="packed", .id = Id.Keyword_packed},
39	KeywordId{.bytes="pub", .id = Id.Keyword_pub},
40	KeywordId{.bytes="return", .id = Id.Keyword_return},
41	KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
42	KeywordId{.bytes="struct", .id = Id.Keyword_struct},
43	KeywordId{.bytes="switch", .id = Id.Keyword_switch},
44	KeywordId{.bytes="test", .id = Id.Keyword_test},
45	KeywordId{.bytes="this", .id = Id.Keyword_this},
46	KeywordId{.bytes="true", .id = Id.Keyword_true},
47	KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
48	KeywordId{.bytes="union", .id = Id.Keyword_union},
49	KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
50	KeywordId{.bytes="use", .id = Id.Keyword_use},
51	KeywordId{.bytes="var", .id = Id.Keyword_var},
52	KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
53	KeywordId{.bytes="while", .id = Id.Keyword_while},
54	};
55
56	fn getKeyword(bytes: []const u8) ?Id {
57	for (keywords) \|kw\| {
58	if (mem.eql(u8, kw.bytes, bytes)) {
59	return kw.id;
60	}
61	}
62	return null;
63	}
64
65	const StrLitKind = enum {Normal, C};
66
67	pub const Id = union(enum) {
68	Invalid,
69	Identifier,
70	StringLiteral: StrLitKind,
71	Eof,
72	Builtin,
73	Bang,
74	Equal,
75	EqualEqual,
76	BangEqual,
77	LParen,
78	RParen,
79	Semicolon,
80	Percent,
81	LBrace,
82	RBrace,
83	Period,
84	Ellipsis2,
85	Ellipsis3,
86	Minus,
87	Arrow,
88	Colon,
89	Slash,
90	Comma,
91	Ampersand,
92	AmpersandEqual,
93	IntegerLiteral,
94	FloatLiteral,
95	Keyword_align,
96	Keyword_and,
97	Keyword_asm,
98	Keyword_break,
99	Keyword_comptime,
100	Keyword_const,
101	Keyword_continue,
102	Keyword_defer,
103	Keyword_else,
104	Keyword_enum,
105	Keyword_error,
106	Keyword_export,
107	Keyword_extern,
108	Keyword_false,
109	Keyword_fn,
110	Keyword_for,
111	Keyword_goto,
112	Keyword_if,
113	Keyword_inline,
114	Keyword_nakedcc,
115	Keyword_noalias,
116	Keyword_null,
117	Keyword_or,
118	Keyword_packed,
119	Keyword_pub,
120	Keyword_return,
121	Keyword_stdcallcc,
122	Keyword_struct,
123	Keyword_switch,
124	Keyword_test,
125	Keyword_this,
126	Keyword_true,
127	Keyword_undefined,
128	Keyword_union,
129	Keyword_unreachable,
130	Keyword_use,
131	Keyword_var,
132	Keyword_volatile,
133	Keyword_while,
134	};
135	};
136
137	pub const Tokenizer = struct {
138	buffer: []const u8,
139	index: usize,
140	pending_invalid_token: ?Token,
141
142	pub const Location = struct {
143	line: usize,
144	column: usize,
145	line_start: usize,
146	line_end: usize,
147	};
148
149	pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {
150	var loc = Location {
151	.line = 0,
152	.column = 0,
153	.line_start = 0,
154	.line_end = 0,
155	};
156	for (self.buffer) \|c, i\| {
157	if (i == token.start) {
158	loc.line_end = i;
159	while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
160	return loc;
161	}
162	if (c == '\n') {
163	loc.line += 1;
164	loc.column = 0;
165	loc.line_start = i + 1;
166	} else {
167	loc.column += 1;
168	}
169	}
170	return loc;
171	}
172
173	/// For debugging purposes
174	pub fn dump(self: &Tokenizer, token: &const Token) void {
175	std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
176	}
177
178	/// buffer must end with "\n\n\n". This is so that attempting to decode
179	/// a the 3 trailing bytes of a 4-byte utf8 sequence is never a buffer overflow.
180	pub fn init(buffer: []const u8) Tokenizer {
181	std.debug.assert(buffer[buffer.len - 1] == '\n');
182	std.debug.assert(buffer[buffer.len - 2] == '\n');
183	std.debug.assert(buffer[buffer.len - 3] == '\n');
184	return Tokenizer {
185	.buffer = buffer,
186	.index = 0,
187	.pending_invalid_token = null,
188	};
189	}
190
191	const State = enum {
192	Start,
193	Identifier,
194	Builtin,
195	C,
196	StringLiteral,
197	StringLiteralBackslash,
198	Equal,
199	Bang,
200	Minus,
201	Slash,
202	LineComment,
203	Zero,
204	IntegerLiteral,
205	IntegerLiteralWithRadix,
206	NumberDot,
207	FloatFraction,
208	FloatExponentUnsigned,
209	FloatExponentNumber,
210	Ampersand,
211	Period,
212	Period2,
213	};
214
215	pub fn next(self: &Tokenizer) Token {
216	if (self.pending_invalid_token) \|token\| {
217	self.pending_invalid_token = null;
218	return token;
219	}
220	var state = State.Start;
221	var result = Token {
222	.id = Token.Id.Eof,
223	.start = self.index,
224	.end = undefined,
225	};
226	while (self.index < self.buffer.len) : (self.index += 1) {
227	const c = self.buffer[self.index];
228	switch (state) {
229	State.Start => switch (c) {
230	' ', '\n' => {
231	result.start = self.index + 1;
232	},
233	'c' => {
234	state = State.C;
235	result.id = Token.Id.Identifier;
236	},
237	'"' => {
238	state = State.StringLiteral;
239	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
240	},
241	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
242	state = State.Identifier;
243	result.id = Token.Id.Identifier;
244	},
245	'@' => {
246	state = State.Builtin;
247	result.id = Token.Id.Builtin;
248	},
249	'=' => {
250	state = State.Equal;
251	},
252	'!' => {
253	state = State.Bang;
254	},
255	'(' => {
256	result.id = Token.Id.LParen;
257	self.index += 1;
258	break;
259	},
260	')' => {
261	result.id = Token.Id.RParen;
262	self.index += 1;
263	break;
264	},
265	';' => {
266	result.id = Token.Id.Semicolon;
267	self.index += 1;
268	break;
269	},
270	',' => {
271	result.id = Token.Id.Comma;
272	self.index += 1;
273	break;
274	},
275	':' => {
276	result.id = Token.Id.Colon;
277	self.index += 1;
278	break;
279	},
280	'%' => {
281	result.id = Token.Id.Percent;
282	self.index += 1;
283	break;
284	},
285	'{' => {
286	result.id = Token.Id.LBrace;
287	self.index += 1;
288	break;
289	},
290	'}' => {
291	result.id = Token.Id.RBrace;
292	self.index += 1;
293	break;
294	},
295	'.' => {
296	state = State.Period;
297	},
298	'-' => {
299	state = State.Minus;
300	},
301	'/' => {
302	state = State.Slash;
303	},
304	'&' => {
305	state = State.Ampersand;
306	},
307	'0' => {
308	state = State.Zero;
309	result.id = Token.Id.IntegerLiteral;
310	},
311	'1'...'9' => {
312	state = State.IntegerLiteral;
313	result.id = Token.Id.IntegerLiteral;
314	},
315	else => {
316	result.id = Token.Id.Invalid;
317	self.index += 1;
318	break;
319	},
320	},
321	State.Ampersand => switch (c) {
322	'=' => {
323	result.id = Token.Id.AmpersandEqual;
324	self.index += 1;
325	break;
326	},
327	else => {
328	result.id = Token.Id.Ampersand;
329	break;
330	},
331	},
332	State.Identifier => switch (c) {
333	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
334	else => {
335	if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
336	result.id = id;
337	}
338	break;
339	},
340	},
341	State.Builtin => switch (c) {
342	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
343	else => break,
344	},
345	State.C => switch (c) {
346	'\\' => @panic("TODO"),
347	'"' => {
348	state = State.StringLiteral;
349	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
350	},
351	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
352	state = State.Identifier;
353	},
354	else => break,
355	},
356	State.StringLiteral => switch (c) {
357	'\\' => {
358	state = State.StringLiteralBackslash;
359	},
360	'"' => {
361	self.index += 1;
362	break;
363	},
364	'\n' => break, // Look for this error later.
365	else => self.checkLiteralCharacter(),
366	},
367
368	State.StringLiteralBackslash => switch (c) {
369	'\n' => break, // Look for this error later.
370	else => {
371	state = State.StringLiteral;
372	},
373	},
374
375	State.Bang => switch (c) {
376	'=' => {
377	result.id = Token.Id.BangEqual;
378	self.index += 1;
379	break;
380	},
381	else => {
382	result.id = Token.Id.Bang;
383	break;
384	},
385	},
386
387	State.Equal => switch (c) {
388	'=' => {
389	result.id = Token.Id.EqualEqual;
390	self.index += 1;
391	break;
392	},
393	else => {
394	result.id = Token.Id.Equal;
395	break;
396	},
397	},
398
399	State.Minus => switch (c) {
400	'>' => {
401	result.id = Token.Id.Arrow;
402	self.index += 1;
403	break;
404	},
405	else => {
406	result.id = Token.Id.Minus;
407	break;
408	},
409	},
410
411	State.Period => switch (c) {
412	'.' => {
413	state = State.Period2;
414	},
415	else => {
416	result.id = Token.Id.Period;
417	break;
418	},
419	},
420
421	State.Period2 => switch (c) {
422	'.' => {
423	result.id = Token.Id.Ellipsis3;
424	self.index += 1;
425	break;
426	},
427	else => {
428	result.id = Token.Id.Ellipsis2;
429	break;
430	},
431	},
432
433	State.Slash => switch (c) {
434	'/' => {
435	result.id = undefined;
436	state = State.LineComment;
437	},
438	else => {
439	result.id = Token.Id.Slash;
440	break;
441	},
442	},
443	State.LineComment => switch (c) {
444	'\n' => {
445	state = State.Start;
446	result = Token {
447	.id = Token.Id.Eof,
448	.start = self.index + 1,
449	.end = undefined,
450	};
451	},
452	else => self.checkLiteralCharacter(),
453	},
454	State.Zero => switch (c) {
455	'b', 'o', 'x' => {
456	state = State.IntegerLiteralWithRadix;
457	},
458	else => {
459	// reinterpret as a normal number
460	self.index -= 1;
461	state = State.IntegerLiteral;
462	},
463	},
464	State.IntegerLiteral => switch (c) {
465	'.' => {
466	state = State.NumberDot;
467	},
468	'p', 'P', 'e', 'E' => {
469	state = State.FloatExponentUnsigned;
470	},
471	'0'...'9' => {},
472	else => break,
473	},
474	State.IntegerLiteralWithRadix => switch (c) {
475	'.' => {
476	state = State.NumberDot;
477	},
478	'p', 'P' => {
479	state = State.FloatExponentUnsigned;
480	},
481	'0'...'9', 'a'...'f', 'A'...'F' => {},
482	else => break,
483	},
484	State.NumberDot => switch (c) {
485	'.' => {
486	self.index -= 1;
487	state = State.Start;
488	break;
489	},
490	else => {
491	self.index -= 1;
492	result.id = Token.Id.FloatLiteral;
493	state = State.FloatFraction;
494	},
495	},
496	State.FloatFraction => switch (c) {
497	'p', 'P' => {
498	state = State.FloatExponentUnsigned;
499	},
500	'0'...'9', 'a'...'f', 'A'...'F' => {},
501	else => break,
502	},
503	State.FloatExponentUnsigned => switch (c) {
504	'+', '-' => {
505	state = State.FloatExponentNumber;
506	},
507	else => {
508	// reinterpret as a normal exponent number
509	self.index -= 1;
510	state = State.FloatExponentNumber;
511	}
512	},
513	State.FloatExponentNumber => switch (c) {
514	'0'...'9', 'a'...'f', 'A'...'F' => {},
515	else => break,
516	},
517	}
518	}
519	result.end = self.index;
520
521	if (result.id == Token.Id.Eof) {
522	if (self.pending_invalid_token) \|token\| {
523	self.pending_invalid_token = null;
524	return token;
525	}
526	}
527
528	return result;
529	}
530
531	pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) []const u8 {
532	return self.buffer[token.start..token.end];
533	}
534
535	fn checkLiteralCharacter(self: &Tokenizer) void {
536	if (self.pending_invalid_token != null) return;
537	const invalid_length = self.getInvalidCharacterLength();
538	if (invalid_length == 0) return;
539	self.pending_invalid_token = Token {
540	.id = Token.Id.Invalid,
541	.start = self.index,
542	.end = self.index + invalid_length,
543	};
544	}
545
546	fn getInvalidCharacterLength(self: &Tokenizer) u3 {
547	const c0 = self.buffer[self.index];
548	if (c0 < 0x80) {
549	if (c0 < 0x20 or c0 == 0x7f) {
550	// ascii control codes are never allowed
551	// (note that \n was checked before we got here)
552	return 1;
553	}
554	// looks fine to me.
555	return 0;
556	} else {
557	// check utf8-encoded character.
558	const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
559	// the last 3 bytes in the buffer are guaranteed to be '\n',
560	// which means we don't need to do any bounds checking here.
561	const bytes = self.buffer[self.index..self.index + length];
562	switch (length) {
563	2 => {
564	const value = std.unicode.utf8Decode2(bytes) catch return length;
565	if (value == 0x85) return length; // U+0085 (NEL)
566	},
567	3 => {
568	const value = std.unicode.utf8Decode3(bytes) catch return length;
569	if (value == 0x2028) return length; // U+2028 (LS)
570	if (value == 0x2029) return length; // U+2029 (PS)
571	},
572	4 => {
573	_ = std.unicode.utf8Decode4(bytes) catch return length;
574	},
575	else => unreachable,
576	}
577	self.index += length - 1;
578	return 0;
579	}
580	}
581	};
582
583
584
585	test "tokenizer" {
586	testTokenize("test", []Token.Id {
587	Token.Id.Keyword_test,
588	});
589	}
590
591	test "tokenizer - invalid token characters" {
592	testTokenize("#", []Token.Id{Token.Id.Invalid});
593	testTokenize("`", []Token.Id{Token.Id.Invalid});
594	}
595
596	test "tokenizer - invalid literal/comment characters" {
597	testTokenize("\"\x00\"", []Token.Id {
598	Token.Id { .StringLiteral = Token.StrLitKind.Normal },
599	Token.Id.Invalid,
600	});
601	testTokenize("//\x00", []Token.Id {
602	Token.Id.Invalid,
603	});
604	testTokenize("//\x1f", []Token.Id {
605	Token.Id.Invalid,
606	});
607	testTokenize("//\x7f", []Token.Id {
608	Token.Id.Invalid,
609	});
610	}
611
612	test "tokenizer - utf8" {
613	testTokenize("//\xc2\x80", []Token.Id{});
614	testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});
615	}
616
617	test "tokenizer - invalid utf8" {
618	testTokenize("//\x80", []Token.Id{Token.Id.Invalid});
619	testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});
620	testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});
621	testTokenize("//\xff", []Token.Id{Token.Id.Invalid});
622	testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});
623	testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});
624	testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});
625	testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});
626	}
627
628	test "tokenizer - illegal unicode codepoints" {
629	// unicode newline characters.U+0085, U+2028, U+2029
630	testTokenize("//\xc2\x84", []Token.Id{});
631	testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});
632	testTokenize("//\xc2\x86", []Token.Id{});
633	testTokenize("//\xe2\x80\xa7", []Token.Id{});
634	testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});
635	testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});
636	testTokenize("//\xe2\x80\xaa", []Token.Id{});
637	}
638
639	fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
640	// (test authors, just make this bigger if you need it)
641	var padded_source: [0x100]u8 = undefined;
642	std.mem.copy(u8, padded_source[0..source.len], source);
643	padded_source[source.len + 0] = '\n';
644	padded_source[source.len + 1] = '\n';
645	padded_source[source.len + 2] = '\n';
646
647	var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
648	for (expected_tokens) \|expected_token_id\| {
649	const token = tokenizer.next();
650	std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
651	switch (expected_token_id) {
652	Token.Id.StringLiteral => \|expected_kind\| {
653	std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => \|kind\| kind, else => unreachable });
654	},
655	else => {},
656	}
657	}
658	std.debug.assert(tokenizer.next().id == Token.Id.Eof);
659	}

std/index.zig+2

...	@@ -28,6 +28,7 @@ pub const os = @import("os/index.zig");	...	@@ -28,6 +28,7 @@ pub const os = @import("os/index.zig");
28	pub const rand = @import("rand.zig");	28	pub const rand = @import("rand.zig");
29	pub const sort = @import("sort.zig");	29	pub const sort = @import("sort.zig");
30	pub const unicode = @import("unicode.zig");	30	pub const unicode = @import("unicode.zig");
		31	pub const zig = @import("zig/index.zig");
31		32
32	test "std" {	33	test "std" {
33	// run tests from these	34	// run tests from these
...	@@ -58,4 +59,5 @@ test "std" {	...	@@ -58,4 +59,5 @@ test "std" {
58	_ = @import("rand.zig");	59	_ = @import("rand.zig");
59	_ = @import("sort.zig");	60	_ = @import("sort.zig");
60	_ = @import("unicode.zig");	61	_ = @import("unicode.zig");
		62	_ = @import("zig/index.zig");
61	}	63	}

std/zig/ast.zig created+271

...	@@ -0,0 +1,271 @@
1	const std = @import("../index.zig");
2	const assert = std.debug.assert;
3	const ArrayList = std.ArrayList;
4	const Token = std.zig.Token;
5	const mem = std.mem;
6
7	pub const Node = struct {
8	id: Id,
9
10	pub const Id = enum {
11	Root,
12	VarDecl,
13	Identifier,
14	FnProto,
15	ParamDecl,
16	Block,
17	InfixOp,
18	PrefixOp,
19	IntegerLiteral,
20	FloatLiteral,
21	};
22
23	pub fn iterate(base: &Node, index: usize) ?&Node {
24	return switch (base.id) {
25	Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
26	Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
27	Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
28	Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
29	Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
30	Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
31	Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
32	Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
33	Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
34	Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
35	};
36	}
37
38	pub fn destroy(base: &Node, allocator: &mem.Allocator) void {
39	return switch (base.id) {
40	Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
41	Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
42	Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
43	Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
44	Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
45	Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
46	Id.InfixOp => allocator.destroy(@fieldParentPtr(NodeInfixOp, "base", base)),
47	Id.PrefixOp => allocator.destroy(@fieldParentPtr(NodePrefixOp, "base", base)),
48	Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
49	Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
50	};
51	}
52	};
53
54	pub const NodeRoot = struct {
55	base: Node,
56	decls: ArrayList(&Node),
57
58	pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
59	if (index < self.decls.len) {
60	return self.decls.items[self.decls.len - index - 1];
61	}
62	return null;
63	}
64	};
65
66	pub const NodeVarDecl = struct {
67	base: Node,
68	visib_token: ?Token,
69	name_token: Token,
70	eq_token: Token,
71	mut_token: Token,
72	comptime_token: ?Token,
73	extern_token: ?Token,
74	lib_name: ?&Node,
75	type_node: ?&Node,
76	align_node: ?&Node,
77	init_node: ?&Node,
78
79	pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
80	var i = index;
81
82	if (self.type_node) \|type_node\| {
83	if (i < 1) return type_node;
84	i -= 1;
85	}
86
87	if (self.align_node) \|align_node\| {
88	if (i < 1) return align_node;
89	i -= 1;
90	}
91
92	if (self.init_node) \|init_node\| {
93	if (i < 1) return init_node;
94	i -= 1;
95	}
96
97	return null;
98	}
99	};
100
101	pub const NodeIdentifier = struct {
102	base: Node,
103	name_token: Token,
104
105	pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
106	return null;
107	}
108	};
109
110	pub const NodeFnProto = struct {
111	base: Node,
112	visib_token: ?Token,
113	fn_token: Token,
114	name_token: ?Token,
115	params: ArrayList(&Node),
116	return_type: &Node,
117	var_args_token: ?Token,
118	extern_token: ?Token,
119	inline_token: ?Token,
120	cc_token: ?Token,
121	body_node: ?&Node,
122	lib_name: ?&Node, // populated if this is an extern declaration
123	align_expr: ?&Node, // populated if align(A) is present
124
125	pub fn iterate(self: &NodeFnProto, index: usize) ?&Node {
126	var i = index;
127
128	if (self.body_node) \|body_node\| {
129	if (i < 1) return body_node;
130	i -= 1;
131	}
132
133	if (i < 1) return self.return_type;
134	i -= 1;
135
136	if (self.align_expr) \|align_expr\| {
137	if (i < 1) return align_expr;
138	i -= 1;
139	}
140
141	if (i < self.params.len) return self.params.items[self.params.len - i - 1];
142	i -= self.params.len;
143
144	if (self.lib_name) \|lib_name\| {
145	if (i < 1) return lib_name;
146	i -= 1;
147	}
148
149	return null;
150	}
151	};
152
153	pub const NodeParamDecl = struct {
154	base: Node,
155	comptime_token: ?Token,
156	noalias_token: ?Token,
157	name_token: ?Token,
158	type_node: &Node,
159	var_args_token: ?Token,
160
161	pub fn iterate(self: &NodeParamDecl, index: usize) ?&Node {
162	var i = index;
163
164	if (i < 1) return self.type_node;
165	i -= 1;
166
167	return null;
168	}
169	};
170
171	pub const NodeBlock = struct {
172	base: Node,
173	begin_token: Token,
174	end_token: Token,
175	statements: ArrayList(&Node),
176
177	pub fn iterate(self: &NodeBlock, index: usize) ?&Node {
178	var i = index;
179
180	if (i < self.statements.len) return self.statements.items[i];
181	i -= self.statements.len;
182
183	return null;
184	}
185	};
186
187	pub const NodeInfixOp = struct {
188	base: Node,
189	op_token: Token,
190	lhs: &Node,
191	op: InfixOp,
192	rhs: &Node,
193
194	const InfixOp = enum {
195	EqualEqual,
196	BangEqual,
197	};
198
199	pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
200	var i = index;
201
202	if (i < 1) return self.lhs;
203	i -= 1;
204
205	switch (self.op) {
206	InfixOp.EqualEqual => {},
207	InfixOp.BangEqual => {},
208	}
209
210	if (i < 1) return self.rhs;
211	i -= 1;
212
213	return null;
214	}
215	};
216
217	pub const NodePrefixOp = struct {
218	base: Node,
219	op_token: Token,
220	op: PrefixOp,
221	rhs: &Node,
222
223	const PrefixOp = union(enum) {
224	Return,
225	AddrOf: AddrOfInfo,
226	};
227	const AddrOfInfo = struct {
228	align_expr: ?&Node,
229	bit_offset_start_token: ?Token,
230	bit_offset_end_token: ?Token,
231	const_token: ?Token,
232	volatile_token: ?Token,
233	};
234
235	pub fn iterate(self: &NodePrefixOp, index: usize) ?&Node {
236	var i = index;
237
238	switch (self.op) {
239	PrefixOp.Return => {},
240	PrefixOp.AddrOf => \|addr_of_info\| {
241	if (addr_of_info.align_expr) \|align_expr\| {
242	if (i < 1) return align_expr;
243	i -= 1;
244	}
245	},
246	}
247
248	if (i < 1) return self.rhs;
249	i -= 1;
250
251	return null;
252	}
253	};
254
255	pub const NodeIntegerLiteral = struct {
256	base: Node,
257	token: Token,
258
259	pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
260	return null;
261	}
262	};
263
264	pub const NodeFloatLiteral = struct {
265	base: Node,
266	token: Token,
267
268	pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
269	return null;
270	}
271	};

std/zig/index.zig created+11

...	@@ -0,0 +1,11 @@
1	const tokenizer = @import("tokenizer.zig");
2	pub const Token = tokenizer.Token;
3	pub const Tokenizer = tokenizer.Tokenizer;
4	pub const Parser = @import("parser.zig").Parser;
5	pub const ast = @import("ast.zig");
6
7	test "std.zig tests" {
8	_ = @import("tokenizer.zig");
9	_ = @import("parser.zig");
10	_ = @import("ast.zig");
11	}

std/zig/parser.zig created+1160

...	@@ -0,0 +1,1160 @@
1	const std = @import("../index.zig");
2	const assert = std.debug.assert;
3	const ArrayList = std.ArrayList;
4	const mem = std.mem;
5	const ast = std.zig.ast;
6	const Tokenizer = std.zig.Tokenizer;
7	const Token = std.zig.Token;
8	const builtin = @import("builtin");
9	const io = std.io;
10
11	// TODO when we make parse errors into error types instead of printing directly,
12	// get rid of this
13	const warn = std.debug.warn;
14
15	pub const Parser = struct {
16	allocator: &mem.Allocator,
17	tokenizer: &Tokenizer,
18	put_back_tokens: [2]Token,
19	put_back_count: usize,
20	source_file_name: []const u8,
21
22	pub const Tree = struct {
23	root_node: &ast.NodeRoot,
24
25	pub fn deinit(self: &const Tree) void {
26	// TODO free the whole arena
27	}
28	};
29
30	// This memory contents are used only during a function call. It's used to repurpose memory;
31	// we reuse the same bytes for the stack data structure used by parsing, tree rendering, and
32	// source rendering.
33	const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
34	utility_bytes: []align(utility_bytes_align) u8,
35
36	/// `allocator` should be an arena allocator. Parser never calls free on anything. After you're
37	/// done with a Parser, free the arena. After the arena is freed, no member functions of Parser
38	/// may be called.
39	pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) Parser {
40	return Parser {
41	.allocator = allocator,
42	.tokenizer = tokenizer,
43	.put_back_tokens = undefined,
44	.put_back_count = 0,
45	.source_file_name = source_file_name,
46	.utility_bytes = []align(utility_bytes_align) u8{},
47	};
48	}
49
50	pub fn deinit(self: &Parser) void {
51	self.allocator.free(self.utility_bytes);
52	}
53
54	const TopLevelDeclCtx = struct {
55	visib_token: ?Token,
56	extern_token: ?Token,
57	};
58
59	const DestPtr = union(enum) {
60	Field: &&ast.Node,
61	NullableField: &?&ast.Node,
62	List: &ArrayList(&ast.Node),
63
64	pub fn store(self: &const DestPtr, value: &ast.Node) !void {
65	switch (*self) {
66	DestPtr.Field => \|ptr\| *ptr = value,
67	DestPtr.NullableField => \|ptr\| *ptr = value,
68	DestPtr.List => \|list\| try list.append(value),
69	}
70	}
71	};
72
73	const State = union(enum) {
74	TopLevel,
75	TopLevelExtern: ?Token,
76	TopLevelDecl: TopLevelDeclCtx,
77	Expression: DestPtr,
78	ExpectOperand,
79	Operand: &ast.Node,
80	AfterOperand,
81	InfixOp: &ast.NodeInfixOp,
82	PrefixOp: &ast.NodePrefixOp,
83	AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
84	TypeExpr: DestPtr,
85	VarDecl: &ast.NodeVarDecl,
86	VarDeclAlign: &ast.NodeVarDecl,
87	VarDeclEq: &ast.NodeVarDecl,
88	ExpectToken: @TagType(Token.Id),
89	FnProto: &ast.NodeFnProto,
90	FnProtoAlign: &ast.NodeFnProto,
91	ParamDecl: &ast.NodeFnProto,
92	ParamDeclComma,
93	FnDef: &ast.NodeFnProto,
94	Block: &ast.NodeBlock,
95	Statement: &ast.NodeBlock,
96	};
97
98	/// Returns an AST tree, allocated with the parser's allocator.
99	/// Result should be freed with `freeAst` when done.
100	pub fn parse(self: &Parser) !Tree {
101	var stack = self.initUtilityArrayList(State);
102	defer self.deinitUtilityArrayList(stack);
103
104	const root_node = try self.createRoot();
105	// TODO errdefer arena free root node
106
107	try stack.append(State.TopLevel);
108
109	while (true) {
110	//{
111	// const token = self.getNextToken();
112	// warn("{} ", @tagName(token.id));
113	// self.putBackToken(token);
114	// var i: usize = stack.len;
115	// while (i != 0) {
116	// i -= 1;
117	// warn("{} ", @tagName(stack.items[i]));
118	// }
119	// warn("\n");
120	//}
121
122	// This gives us 1 free append that can't fail
123	const state = stack.pop();
124
125	switch (state) {
126	State.TopLevel => {
127	const token = self.getNextToken();
128	switch (token.id) {
129	Token.Id.Keyword_pub, Token.Id.Keyword_export => {
130	stack.append(State { .TopLevelExtern = token }) catch unreachable;
131	continue;
132	},
133	Token.Id.Eof => return Tree {.root_node = root_node},
134	else => {
135	self.putBackToken(token);
136	// TODO shouldn't need this cast
137	stack.append(State { .TopLevelExtern = null }) catch unreachable;
138	continue;
139	},
140	}
141	},
142	State.TopLevelExtern => \|visib_token\| {
143	const token = self.getNextToken();
144	if (token.id == Token.Id.Keyword_extern) {
145	stack.append(State {
146	.TopLevelDecl = TopLevelDeclCtx {
147	.visib_token = visib_token,
148	.extern_token = token,
149	},
150	}) catch unreachable;
151	continue;
152	}
153	self.putBackToken(token);
154	stack.append(State {
155	.TopLevelDecl = TopLevelDeclCtx {
156	.visib_token = visib_token,
157	.extern_token = null,
158	},
159	}) catch unreachable;
160	continue;
161	},
162	State.TopLevelDecl => \|ctx\| {
163	const token = self.getNextToken();
164	switch (token.id) {
165	Token.Id.Keyword_var, Token.Id.Keyword_const => {
166	stack.append(State.TopLevel) catch unreachable;
167	// TODO shouldn't need these casts
168	const var_decl_node = try self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
169	token, (?Token)(null), ctx.extern_token);
170	try stack.append(State { .VarDecl = var_decl_node });
171	continue;
172	},
173	Token.Id.Keyword_fn => {
174	stack.append(State.TopLevel) catch unreachable;
175	// TODO shouldn't need these casts
176	const fn_proto = try self.createAttachFnProto(&root_node.decls, token,
177	ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
178	try stack.append(State { .FnDef = fn_proto });
179	try stack.append(State { .FnProto = fn_proto });
180	continue;
181	},
182	Token.Id.StringLiteral => {
183	@panic("TODO extern with string literal");
184	},
185	Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
186	stack.append(State.TopLevel) catch unreachable;
187	const fn_token = try self.eatToken(Token.Id.Keyword_fn);
188	// TODO shouldn't need this cast
189	const fn_proto = try self.createAttachFnProto(&root_node.decls, fn_token,
190	ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
191	try stack.append(State { .FnDef = fn_proto });
192	try stack.append(State { .FnProto = fn_proto });
193	continue;
194	},
195	else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
196	}
197	},
198	State.VarDecl => \|var_decl\| {
199	var_decl.name_token = try self.eatToken(Token.Id.Identifier);
200	stack.append(State { .VarDeclAlign = var_decl }) catch unreachable;
201
202	const next_token = self.getNextToken();
203	if (next_token.id == Token.Id.Colon) {
204	try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
205	continue;
206	}
207
208	self.putBackToken(next_token);
209	continue;
210	},
211	State.VarDeclAlign => \|var_decl\| {
212	stack.append(State { .VarDeclEq = var_decl }) catch unreachable;
213
214	const next_token = self.getNextToken();
215	if (next_token.id == Token.Id.Keyword_align) {
216	_ = try self.eatToken(Token.Id.LParen);
217	try stack.append(State { .ExpectToken = Token.Id.RParen });
218	try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} });
219	continue;
220	}
221
222	self.putBackToken(next_token);
223	continue;
224	},
225	State.VarDeclEq => \|var_decl\| {
226	const token = self.getNextToken();
227	if (token.id == Token.Id.Equal) {
228	var_decl.eq_token = token;
229	stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
230	try stack.append(State {
231	.Expression = DestPtr {.NullableField = &var_decl.init_node},
232	});
233	continue;
234	}
235	if (token.id == Token.Id.Semicolon) {
236	continue;
237	}
238	return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
239	},
240	State.ExpectToken => \|token_id\| {
241	_ = try self.eatToken(token_id);
242	continue;
243	},
244
245	State.Expression => \|dest_ptr\| {
246	// save the dest_ptr for later
247	stack.append(state) catch unreachable;
248	try stack.append(State.ExpectOperand);
249	continue;
250	},
251	State.ExpectOperand => {
252	// we'll either get an operand (like 1 or x),
253	// or a prefix operator (like ~ or return).
254	const token = self.getNextToken();
255	switch (token.id) {
256	Token.Id.Keyword_return => {
257	try stack.append(State { .PrefixOp = try self.createPrefixOp(token,
258	ast.NodePrefixOp.PrefixOp.Return) });
259	try stack.append(State.ExpectOperand);
260	continue;
261	},
262	Token.Id.Ampersand => {
263	const prefix_op = try self.createPrefixOp(token, ast.NodePrefixOp.PrefixOp{
264	.AddrOf = ast.NodePrefixOp.AddrOfInfo {
265	.align_expr = null,
266	.bit_offset_start_token = null,
267	.bit_offset_end_token = null,
268	.const_token = null,
269	.volatile_token = null,
270	}
271	});
272	try stack.append(State { .PrefixOp = prefix_op });
273	try stack.append(State.ExpectOperand);
274	try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf });
275	continue;
276	},
277	Token.Id.Identifier => {
278	try stack.append(State {
279	.Operand = &(try self.createIdentifier(token)).base
280	});
281	try stack.append(State.AfterOperand);
282	continue;
283	},
284	Token.Id.IntegerLiteral => {
285	try stack.append(State {
286	.Operand = &(try self.createIntegerLiteral(token)).base
287	});
288	try stack.append(State.AfterOperand);
289	continue;
290	},
291	Token.Id.FloatLiteral => {
292	try stack.append(State {
293	.Operand = &(try self.createFloatLiteral(token)).base
294	});
295	try stack.append(State.AfterOperand);
296	continue;
297	},
298	else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
299	}
300	},
301
302	State.AfterOperand => {
303	// we'll either get an infix operator (like != or ^),
304	// or a postfix operator (like () or {}),
305	// otherwise this expression is done (like on a ; or else).
306	var token = self.getNextToken();
307	switch (token.id) {
308	Token.Id.EqualEqual => {
309	try stack.append(State {
310	.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.EqualEqual)
311	});
312	try stack.append(State.ExpectOperand);
313	continue;
314	},
315	Token.Id.BangEqual => {
316	try stack.append(State {
317	.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.BangEqual)
318	});
319	try stack.append(State.ExpectOperand);
320	continue;
321	},
322	else => {
323	// no postfix/infix operator after this operand.
324	self.putBackToken(token);
325	// reduce the stack
326	var expression: &ast.Node = stack.pop().Operand;
327	while (true) {
328	switch (stack.pop()) {
329	State.Expression => \|dest_ptr\| {
330	// we're done
331	try dest_ptr.store(expression);
332	break;
333	},
334	State.InfixOp => \|infix_op\| {
335	infix_op.rhs = expression;
336	infix_op.lhs = stack.pop().Operand;
337	expression = &infix_op.base;
338	continue;
339	},
340	State.PrefixOp => \|prefix_op\| {
341	prefix_op.rhs = expression;
342	expression = &prefix_op.base;
343	continue;
344	},
345	else => unreachable,
346	}
347	}
348	continue;
349	},
350	}
351	},
352
353	State.AddrOfModifiers => \|addr_of_info\| {
354	var token = self.getNextToken();
355	switch (token.id) {
356	Token.Id.Keyword_align => {
357	stack.append(state) catch unreachable;
358	if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers");
359	_ = try self.eatToken(Token.Id.LParen);
360	try stack.append(State { .ExpectToken = Token.Id.RParen });
361	try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} });
362	continue;
363	},
364	Token.Id.Keyword_const => {
365	stack.append(state) catch unreachable;
366	if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const");
367	addr_of_info.const_token = token;
368	continue;
369	},
370	Token.Id.Keyword_volatile => {
371	stack.append(state) catch unreachable;
372	if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile");
373	addr_of_info.volatile_token = token;
374	continue;
375	},
376	else => {
377	self.putBackToken(token);
378	continue;
379	},
380	}
381	},
382
383	State.TypeExpr => \|dest_ptr\| {
384	const token = self.getNextToken();
385	if (token.id == Token.Id.Keyword_var) {
386	@panic("TODO param with type var");
387	}
388	self.putBackToken(token);
389
390	stack.append(State { .Expression = dest_ptr }) catch unreachable;
391	continue;
392	},
393
394	State.FnProto => \|fn_proto\| {
395	stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable;
396	try stack.append(State { .ParamDecl = fn_proto });
397	try stack.append(State { .ExpectToken = Token.Id.LParen });
398
399	const next_token = self.getNextToken();
400	if (next_token.id == Token.Id.Identifier) {
401	fn_proto.name_token = next_token;
402	continue;
403	}
404	self.putBackToken(next_token);
405	continue;
406	},
407
408	State.FnProtoAlign => \|fn_proto\| {
409	const token = self.getNextToken();
410	if (token.id == Token.Id.Keyword_align) {
411	@panic("TODO fn proto align");
412	}
413	self.putBackToken(token);
414	stack.append(State {
415	.TypeExpr = DestPtr {.Field = &fn_proto.return_type},
416	}) catch unreachable;
417	continue;
418	},
419
420	State.ParamDecl => \|fn_proto\| {
421	var token = self.getNextToken();
422	if (token.id == Token.Id.RParen) {
423	continue;
424	}
425	const param_decl = try self.createAttachParamDecl(&fn_proto.params);
426	if (token.id == Token.Id.Keyword_comptime) {
427	param_decl.comptime_token = token;
428	token = self.getNextToken();
429	} else if (token.id == Token.Id.Keyword_noalias) {
430	param_decl.noalias_token = token;
431	token = self.getNextToken();
432	}
433	if (token.id == Token.Id.Identifier) {
434	const next_token = self.getNextToken();
435	if (next_token.id == Token.Id.Colon) {
436	param_decl.name_token = token;
437	token = self.getNextToken();
438	} else {
439	self.putBackToken(next_token);
440	}
441	}
442	if (token.id == Token.Id.Ellipsis3) {
443	param_decl.var_args_token = token;
444	stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable;
445	continue;
446	} else {
447	self.putBackToken(token);
448	}
449
450	stack.append(State { .ParamDecl = fn_proto }) catch unreachable;
451	try stack.append(State.ParamDeclComma);
452	try stack.append(State {
453	.TypeExpr = DestPtr {.Field = &param_decl.type_node}
454	});
455	continue;
456	},
457
458	State.ParamDeclComma => {
459	const token = self.getNextToken();
460	switch (token.id) {
461	Token.Id.RParen => {
462	_ = stack.pop(); // pop off the ParamDecl
463	continue;
464	},
465	Token.Id.Comma => continue,
466	else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
467	}
468	},
469
470	State.FnDef => \|fn_proto\| {
471	const token = self.getNextToken();
472	switch(token.id) {
473	Token.Id.LBrace => {
474	const block = try self.createBlock(token);
475	fn_proto.body_node = &block.base;
476	stack.append(State { .Block = block }) catch unreachable;
477	continue;
478	},
479	Token.Id.Semicolon => continue,
480	else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
481	}
482	},
483
484	State.Block => \|block\| {
485	const token = self.getNextToken();
486	switch (token.id) {
487	Token.Id.RBrace => {
488	block.end_token = token;
489	continue;
490	},
491	else => {
492	self.putBackToken(token);
493	stack.append(State { .Block = block }) catch unreachable;
494	try stack.append(State { .Statement = block });
495	continue;
496	},
497	}
498	},
499
500	State.Statement => \|block\| {
501	{
502	// Look for comptime var, comptime const
503	const comptime_token = self.getNextToken();
504	if (comptime_token.id == Token.Id.Keyword_comptime) {
505	const mut_token = self.getNextToken();
506	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
507	// TODO shouldn't need these casts
508	const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
509	mut_token, (?Token)(comptime_token), (?Token)(null));
510	try stack.append(State { .VarDecl = var_decl });
511	continue;
512	}
513	self.putBackToken(mut_token);
514	}
515	self.putBackToken(comptime_token);
516	}
517	{
518	// Look for const, var
519	const mut_token = self.getNextToken();
520	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
521	// TODO shouldn't need these casts
522	const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
523	mut_token, (?Token)(null), (?Token)(null));
524	try stack.append(State { .VarDecl = var_decl });
525	continue;
526	}
527	self.putBackToken(mut_token);
528	}
529
530	stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
531	try stack.append(State { .Expression = DestPtr{.List = &block.statements} });
532	continue;
533	},
534
535	// These are data, not control flow.
536	State.InfixOp => unreachable,
537	State.PrefixOp => unreachable,
538	State.Operand => unreachable,
539	}
540	@import("std").debug.panic("{}", @tagName(state));
541	//unreachable;
542	}
543	}
544
545	fn createRoot(self: &Parser) !&ast.NodeRoot {
546	const node = try self.allocator.create(ast.NodeRoot);
547
548	*node = ast.NodeRoot {
549	.base = ast.Node {.id = ast.Node.Id.Root},
550	.decls = ArrayList(&ast.Node).init(self.allocator),
551	};
552	return node;
553	}
554
555	fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
556	extern_token: &const ?Token) !&ast.NodeVarDecl
557	{
558	const node = try self.allocator.create(ast.NodeVarDecl);
559
560	*node = ast.NodeVarDecl {
561	.base = ast.Node {.id = ast.Node.Id.VarDecl},
562	.visib_token = *visib_token,
563	.mut_token = *mut_token,
564	.comptime_token = *comptime_token,
565	.extern_token = *extern_token,
566	.type_node = null,
567	.align_node = null,
568	.init_node = null,
569	.lib_name = null,
570	// initialized later
571	.name_token = undefined,
572	.eq_token = undefined,
573	};
574	return node;
575	}
576
577	fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
578	cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto
579	{
580	const node = try self.allocator.create(ast.NodeFnProto);
581
582	*node = ast.NodeFnProto {
583	.base = ast.Node {.id = ast.Node.Id.FnProto},
584	.visib_token = *visib_token,
585	.name_token = null,
586	.fn_token = *fn_token,
587	.params = ArrayList(&ast.Node).init(self.allocator),
588	.return_type = undefined,
589	.var_args_token = null,
590	.extern_token = *extern_token,
591	.inline_token = *inline_token,
592	.cc_token = *cc_token,
593	.body_node = null,
594	.lib_name = null,
595	.align_expr = null,
596	};
597	return node;
598	}
599
600	fn createParamDecl(self: &Parser) !&ast.NodeParamDecl {
601	const node = try self.allocator.create(ast.NodeParamDecl);
602
603	*node = ast.NodeParamDecl {
604	.base = ast.Node {.id = ast.Node.Id.ParamDecl},
605	.comptime_token = null,
606	.noalias_token = null,
607	.name_token = null,
608	.type_node = undefined,
609	.var_args_token = null,
610	};
611	return node;
612	}
613
614	fn createBlock(self: &Parser, begin_token: &const Token) !&ast.NodeBlock {
615	const node = try self.allocator.create(ast.NodeBlock);
616
617	*node = ast.NodeBlock {
618	.base = ast.Node {.id = ast.Node.Id.Block},
619	.begin_token = *begin_token,
620	.end_token = undefined,
621	.statements = ArrayList(&ast.Node).init(self.allocator),
622	};
623	return node;
624	}
625
626	fn createInfixOp(self: &Parser, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp {
627	const node = try self.allocator.create(ast.NodeInfixOp);
628
629	*node = ast.NodeInfixOp {
630	.base = ast.Node {.id = ast.Node.Id.InfixOp},
631	.op_token = *op_token,
632	.lhs = undefined,
633	.op = *op,
634	.rhs = undefined,
635	};
636	return node;
637	}
638
639	fn createPrefixOp(self: &Parser, op_token: &const Token, op: &const ast.NodePrefixOp.PrefixOp) !&ast.NodePrefixOp {
640	const node = try self.allocator.create(ast.NodePrefixOp);
641
642	*node = ast.NodePrefixOp {
643	.base = ast.Node {.id = ast.Node.Id.PrefixOp},
644	.op_token = *op_token,
645	.op = *op,
646	.rhs = undefined,
647	};
648	return node;
649	}
650
651	fn createIdentifier(self: &Parser, name_token: &const Token) !&ast.NodeIdentifier {
652	const node = try self.allocator.create(ast.NodeIdentifier);
653
654	*node = ast.NodeIdentifier {
655	.base = ast.Node {.id = ast.Node.Id.Identifier},
656	.name_token = *name_token,
657	};
658	return node;
659	}
660
661	fn createIntegerLiteral(self: &Parser, token: &const Token) !&ast.NodeIntegerLiteral {
662	const node = try self.allocator.create(ast.NodeIntegerLiteral);
663
664	*node = ast.NodeIntegerLiteral {
665	.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
666	.token = *token,
667	};
668	return node;
669	}
670
671	fn createFloatLiteral(self: &Parser, token: &const Token) !&ast.NodeFloatLiteral {
672	const node = try self.allocator.create(ast.NodeFloatLiteral);
673
674	*node = ast.NodeFloatLiteral {
675	.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
676	.token = *token,
677	};
678	return node;
679	}
680
681	fn createAttachIdentifier(self: &Parser, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {
682	const node = try self.createIdentifier(name_token);
683	try dest_ptr.store(&node.base);
684	return node;
685	}
686
687	fn createAttachParamDecl(self: &Parser, list: &ArrayList(&ast.Node)) !&ast.NodeParamDecl {
688	const node = try self.createParamDecl();
689	try list.append(&node.base);
690	return node;
691	}
692
693	fn createAttachFnProto(self: &Parser, list: &ArrayList(&ast.Node), fn_token: &const Token,
694	extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
695	inline_token: &const ?Token) !&ast.NodeFnProto
696	{
697	const node = try self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
698	try list.append(&node.base);
699	return node;
700	}
701
702	fn createAttachVarDecl(self: &Parser, list: &ArrayList(&ast.Node), visib_token: &const ?Token,
703	mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl
704	{
705	const node = try self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
706	try list.append(&node.base);
707	return node;
708	}
709
710	fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) error {
711	const loc = self.tokenizer.getTokenLocation(token);
712	warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
713	warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
714	{
715	var i: usize = 0;
716	while (i < loc.column) : (i += 1) {
717	warn(" ");
718	}
719	}
720	{
721	const caret_count = token.end - token.start;
722	var i: usize = 0;
723	while (i < caret_count) : (i += 1) {
724	warn("~");
725	}
726	}
727	warn("\n");
728	return error.ParseError;
729	}
730
731	fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void {
732	if (token.id != id) {
733	return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
734	}
735	}
736
737	fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token {
738	const token = self.getNextToken();
739	try self.expectToken(token, id);
740	return token;
741	}
742
743	fn putBackToken(self: &Parser, token: &const Token) void {
744	self.put_back_tokens[self.put_back_count] = *token;
745	self.put_back_count += 1;
746	}
747
748	fn getNextToken(self: &Parser) Token {
749	if (self.put_back_count != 0) {
750	const put_back_index = self.put_back_count - 1;
751	const put_back_token = self.put_back_tokens[put_back_index];
752	self.put_back_count = put_back_index;
753	return put_back_token;
754	} else {
755	return self.tokenizer.next();
756	}
757	}
758
759	const RenderAstFrame = struct {
760	node: &ast.Node,
761	indent: usize,
762	};
763
764	pub fn renderAst(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
765	var stack = self.initUtilityArrayList(RenderAstFrame);
766	defer self.deinitUtilityArrayList(stack);
767
768	try stack.append(RenderAstFrame {
769	.node = &root_node.base,
770	.indent = 0,
771	});
772
773	while (stack.popOrNull()) \|frame\| {
774	{
775	var i: usize = 0;
776	while (i < frame.indent) : (i += 1) {
777	try stream.print(" ");
778	}
779	}
780	try stream.print("{}\n", @tagName(frame.node.id));
781	var child_i: usize = 0;
782	while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
783	try stack.append(RenderAstFrame {
784	.node = child,
785	.indent = frame.indent + 2,
786	});
787	}
788	}
789	}
790
791	const RenderState = union(enum) {
792	TopLevelDecl: &ast.Node,
793	FnProtoRParen: &ast.NodeFnProto,
794	ParamDecl: &ast.Node,
795	Text: []const u8,
796	Expression: &ast.Node,
797	VarDecl: &ast.NodeVarDecl,
798	Statement: &ast.Node,
799	PrintIndent,
800	Indent: usize,
801	};
802
803	pub fn renderSource(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
804	var stack = self.initUtilityArrayList(RenderState);
805	defer self.deinitUtilityArrayList(stack);
806
807	{
808	var i = root_node.decls.len;
809	while (i != 0) {
810	i -= 1;
811	const decl = root_node.decls.items[i];
812	try stack.append(RenderState {.TopLevelDecl = decl});
813	}
814	}
815
816	const indent_delta = 4;
817	var indent: usize = 0;
818	while (stack.popOrNull()) \|state\| {
819	switch (state) {
820	RenderState.TopLevelDecl => \|decl\| {
821	switch (decl.id) {
822	ast.Node.Id.FnProto => {
823	const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl);
824	if (fn_proto.visib_token) \|visib_token\| {
825	switch (visib_token.id) {
826	Token.Id.Keyword_pub => try stream.print("pub "),
827	Token.Id.Keyword_export => try stream.print("export "),
828	else => unreachable,
829	}
830	}
831	if (fn_proto.extern_token) \|extern_token\| {
832	try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
833	}
834	try stream.print("fn");
835
836	if (fn_proto.name_token) \|name_token\| {
837	try stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
838	}
839
840	try stream.print("(");
841
842	try stack.append(RenderState { .Text = "\n" });
843	if (fn_proto.body_node == null) {
844	try stack.append(RenderState { .Text = ";" });
845	}
846
847	try stack.append(RenderState { .FnProtoRParen = fn_proto});
848	var i = fn_proto.params.len;
849	while (i != 0) {
850	i -= 1;
851	const param_decl_node = fn_proto.params.items[i];
852	try stack.append(RenderState { .ParamDecl = param_decl_node});
853	if (i != 0) {
854	try stack.append(RenderState { .Text = ", " });
855	}
856	}
857	},
858	ast.Node.Id.VarDecl => {
859	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
860	try stack.append(RenderState { .Text = "\n"});
861	try stack.append(RenderState { .VarDecl = var_decl});
862
863	},
864	else => unreachable,
865	}
866	},
867
868	RenderState.VarDecl => \|var_decl\| {
869	if (var_decl.visib_token) \|visib_token\| {
870	try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
871	}
872	if (var_decl.extern_token) \|extern_token\| {
873	try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
874	if (var_decl.lib_name != null) {
875	@panic("TODO");
876	}
877	}
878	if (var_decl.comptime_token) \|comptime_token\| {
879	try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
880	}
881	try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
882	try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
883
884	try stack.append(RenderState { .Text = ";" });
885	if (var_decl.init_node) \|init_node\| {
886	try stack.append(RenderState { .Expression = init_node });
887	try stack.append(RenderState { .Text = " = " });
888	}
889	if (var_decl.align_node) \|align_node\| {
890	try stack.append(RenderState { .Text = ")" });
891	try stack.append(RenderState { .Expression = align_node });
892	try stack.append(RenderState { .Text = " align(" });
893	}
894	if (var_decl.type_node) \|type_node\| {
895	try stream.print(": ");
896	try stack.append(RenderState { .Expression = type_node });
897	}
898	},
899
900	RenderState.ParamDecl => \|base\| {
901	const param_decl = @fieldParentPtr(ast.NodeParamDecl, "base", base);
902	if (param_decl.comptime_token) \|comptime_token\| {
903	try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
904	}
905	if (param_decl.noalias_token) \|noalias_token\| {
906	try stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
907	}
908	if (param_decl.name_token) \|name_token\| {
909	try stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
910	}
911	if (param_decl.var_args_token) \|var_args_token\| {
912	try stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
913	} else {
914	try stack.append(RenderState { .Expression = param_decl.type_node});
915	}
916	},
917	RenderState.Text => \|bytes\| {
918	try stream.write(bytes);
919	},
920	RenderState.Expression => \|base\| switch (base.id) {
921	ast.Node.Id.Identifier => {
922	const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base);
923	try stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
924	},
925	ast.Node.Id.Block => {
926	const block = @fieldParentPtr(ast.NodeBlock, "base", base);
927	try stream.write("{");
928	try stack.append(RenderState { .Text = "}"});
929	try stack.append(RenderState.PrintIndent);
930	try stack.append(RenderState { .Indent = indent});
931	try stack.append(RenderState { .Text = "\n"});
932	var i = block.statements.len;
933	while (i != 0) {
934	i -= 1;
935	const statement_node = block.statements.items[i];
936	try stack.append(RenderState { .Statement = statement_node});
937	try stack.append(RenderState.PrintIndent);
938	try stack.append(RenderState { .Indent = indent + indent_delta});
939	try stack.append(RenderState { .Text = "\n" });
940	}
941	},
942	ast.Node.Id.InfixOp => {
943	const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);
944	try stack.append(RenderState { .Expression = prefix_op_node.rhs });
945	switch (prefix_op_node.op) {
946	ast.NodeInfixOp.InfixOp.EqualEqual => {
947	try stack.append(RenderState { .Text = " == "});
948	},
949	ast.NodeInfixOp.InfixOp.BangEqual => {
950	try stack.append(RenderState { .Text = " != "});
951	},
952	else => unreachable,
953	}
954	try stack.append(RenderState { .Expression = prefix_op_node.lhs });
955	},
956	ast.Node.Id.PrefixOp => {
957	const prefix_op_node = @fieldParentPtr(ast.NodePrefixOp, "base", base);
958	try stack.append(RenderState { .Expression = prefix_op_node.rhs });
959	switch (prefix_op_node.op) {
960	ast.NodePrefixOp.PrefixOp.Return => {
961	try stream.write("return ");
962	},
963	ast.NodePrefixOp.PrefixOp.AddrOf => \|addr_of_info\| {
964	try stream.write("&");
965	if (addr_of_info.volatile_token != null) {
966	try stack.append(RenderState { .Text = "volatile "});
967	}
968	if (addr_of_info.const_token != null) {
969	try stack.append(RenderState { .Text = "const "});
970	}
971	if (addr_of_info.align_expr) \|align_expr\| {
972	try stream.print("align(");
973	try stack.append(RenderState { .Text = ") "});
974	try stack.append(RenderState { .Expression = align_expr});
975	}
976	},
977	else => unreachable,
978	}
979	},
980	ast.Node.Id.IntegerLiteral => {
981	const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
982	try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));
983	},
984	ast.Node.Id.FloatLiteral => {
985	const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base);
986	try stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token));
987	},
988	else => unreachable,
989	},
990	RenderState.FnProtoRParen => \|fn_proto\| {
991	try stream.print(")");
992	if (fn_proto.align_expr != null) {
993	@panic("TODO");
994	}
995	try stream.print(" ");
996	if (fn_proto.body_node) \|body_node\| {
997	try stack.append(RenderState { .Expression = body_node});
998	try stack.append(RenderState { .Text = " "});
999	}
1000	try stack.append(RenderState { .Expression = fn_proto.return_type});
1001	},
1002	RenderState.Statement => \|base\| {
1003	switch (base.id) {
1004	ast.Node.Id.VarDecl => {
1005	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
1006	try stack.append(RenderState { .VarDecl = var_decl});
1007	},
1008	else => {
1009	try stack.append(RenderState { .Text = ";"});
1010	try stack.append(RenderState { .Expression = base});
1011	},
1012	}
1013	},
1014	RenderState.Indent => \|new_indent\| indent = new_indent,
1015	RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent),
1016	}
1017	}
1018	}
1019
1020	fn initUtilityArrayList(self: &Parser, comptime T: type) ArrayList(T) {
1021	const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
1022	self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
1023	const typed_slice = ([]T)(self.utility_bytes);
1024	return ArrayList(T) {
1025	.allocator = self.allocator,
1026	.items = typed_slice,
1027	.len = 0,
1028	};
1029	}
1030
1031	fn deinitUtilityArrayList(self: &Parser, list: var) void {
1032	self.utility_bytes = ([]align(utility_bytes_align) u8)(list.items);
1033	}
1034
1035	};
1036
1037	var fixed_buffer_mem: [100 * 1024]u8 = undefined;
1038
1039	fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
1040	var padded_source: [0x100]u8 = undefined;
1041	std.mem.copy(u8, padded_source[0..source.len], source);
1042	padded_source[source.len + 0] = '\n';
1043	padded_source[source.len + 1] = '\n';
1044	padded_source[source.len + 2] = '\n';
1045
1046	var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
1047	var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
1048	defer parser.deinit();
1049
1050	const tree = try parser.parse();
1051	defer tree.deinit();
1052
1053	var buffer = try std.Buffer.initSize(allocator, 0);
1054	var buffer_out_stream = io.BufferOutStream.init(&buffer);
1055	try parser.renderSource(&buffer_out_stream.stream, tree.root_node);
1056	return buffer.toOwnedSlice();
1057	}
1058
1059	// TODO test for memory leaks
1060	// TODO test for valid frees
1061	fn testCanonical(source: []const u8) !void {
1062	const needed_alloc_count = x: {
1063	// Try it once with unlimited memory, make sure it works
1064	var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
1065	var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize));
1066	const result_source = try testParse(source, &failing_allocator.allocator);
1067	if (!mem.eql(u8, result_source, source)) {
1068	warn("\n====== expected this output: =========\n");
1069	warn("{}", source);
1070	warn("\n======== instead found this: =========\n");
1071	warn("{}", result_source);
1072	warn("\n======================================\n");
1073	return error.TestFailed;
1074	}
1075	failing_allocator.allocator.free(result_source);
1076	break :x failing_allocator.index;
1077	};
1078
1079	var fail_index: usize = 0;
1080	while (fail_index < needed_alloc_count) : (fail_index += 1) {
1081	var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
1082	var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index);
1083	if (testParse(source, &failing_allocator.allocator)) \|_\| {
1084	return error.NondeterministicMemoryUsage;
1085	} else \|err\| {
1086	assert(err == error.OutOfMemory);
1087	// TODO make this pass
1088	//if (failing_allocator.allocated_bytes != failing_allocator.freed_bytes) {
1089	// warn("\nfail_index: {}/{}\nallocated bytes: {}\nfreed bytes: {}\nallocations: {}\ndeallocations: {}\n",
1090	// fail_index, needed_alloc_count,
1091	// failing_allocator.allocated_bytes, failing_allocator.freed_bytes,
1092	// failing_allocator.index, failing_allocator.deallocations);
1093	// return error.MemoryLeakDetected;
1094	//}
1095	}
1096	}
1097	}
1098
1099	test "zig fmt" {
1100	try testCanonical(
1101	\\extern fn puts(s: &const u8) c_int;
1102	\\
1103	);
1104
1105	try testCanonical(
1106	\\const a = b;
1107	\\pub const a = b;
1108	\\var a = b;
1109	\\pub var a = b;
1110	\\const a: i32 = b;
1111	\\pub const a: i32 = b;
1112	\\var a: i32 = b;
1113	\\pub var a: i32 = b;
1114	\\
1115	);
1116
1117	try testCanonical(
1118	\\extern var foo: c_int;
1119	\\
1120	);
1121
1122	try testCanonical(
1123	\\var foo: c_int align(1);
1124	\\
1125	);
1126
1127	try testCanonical(
1128	\\fn main(argc: c_int, argv: &&u8) c_int {
1129	\\ const a = b;
1130	\\}
1131	\\
1132	);
1133
1134	try testCanonical(
1135	\\fn foo(argc: c_int, argv: &&u8) c_int {
1136	\\ return 0;
1137	\\}
1138	\\
1139	);
1140
1141	try testCanonical(
1142	\\extern fn f1(s: &align(&u8) u8) c_int;
1143	\\
1144	);
1145
1146	try testCanonical(
1147	\\extern fn f1(s: &&align(1) &const &volatile u8) c_int;
1148	\\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int;
1149	\\extern fn f3(s: &align(1) const volatile u8) c_int;
1150	\\
1151	);
1152
1153	try testCanonical(
1154	\\fn f1(a: bool, b: bool) bool {
1155	\\ a != b;
1156	\\ return a == b;
1157	\\}
1158	\\
1159	);
1160	}

std/zig/tokenizer.zig created+659

...	@@ -0,0 +1,659 @@
1	const std = @import("../index.zig");
2	const mem = std.mem;
3
4	pub const Token = struct {
5	id: Id,
6	start: usize,
7	end: usize,
8
9	const KeywordId = struct {
10	bytes: []const u8,
11	id: Id,
12	};
13
14	const keywords = []KeywordId {
15	KeywordId{.bytes="align", .id = Id.Keyword_align},
16	KeywordId{.bytes="and", .id = Id.Keyword_and},
17	KeywordId{.bytes="asm", .id = Id.Keyword_asm},
18	KeywordId{.bytes="break", .id = Id.Keyword_break},
19	KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
20	KeywordId{.bytes="const", .id = Id.Keyword_const},
21	KeywordId{.bytes="continue", .id = Id.Keyword_continue},
22	KeywordId{.bytes="defer", .id = Id.Keyword_defer},
23	KeywordId{.bytes="else", .id = Id.Keyword_else},
24	KeywordId{.bytes="enum", .id = Id.Keyword_enum},
25	KeywordId{.bytes="error", .id = Id.Keyword_error},
26	KeywordId{.bytes="export", .id = Id.Keyword_export},
27	KeywordId{.bytes="extern", .id = Id.Keyword_extern},
28	KeywordId{.bytes="false", .id = Id.Keyword_false},
29	KeywordId{.bytes="fn", .id = Id.Keyword_fn},
30	KeywordId{.bytes="for", .id = Id.Keyword_for},
31	KeywordId{.bytes="goto", .id = Id.Keyword_goto},
32	KeywordId{.bytes="if", .id = Id.Keyword_if},
33	KeywordId{.bytes="inline", .id = Id.Keyword_inline},
34	KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
35	KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
36	KeywordId{.bytes="null", .id = Id.Keyword_null},
37	KeywordId{.bytes="or", .id = Id.Keyword_or},
38	KeywordId{.bytes="packed", .id = Id.Keyword_packed},
39	KeywordId{.bytes="pub", .id = Id.Keyword_pub},
40	KeywordId{.bytes="return", .id = Id.Keyword_return},
41	KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
42	KeywordId{.bytes="struct", .id = Id.Keyword_struct},
43	KeywordId{.bytes="switch", .id = Id.Keyword_switch},
44	KeywordId{.bytes="test", .id = Id.Keyword_test},
45	KeywordId{.bytes="this", .id = Id.Keyword_this},
46	KeywordId{.bytes="true", .id = Id.Keyword_true},
47	KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
48	KeywordId{.bytes="union", .id = Id.Keyword_union},
49	KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
50	KeywordId{.bytes="use", .id = Id.Keyword_use},
51	KeywordId{.bytes="var", .id = Id.Keyword_var},
52	KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
53	KeywordId{.bytes="while", .id = Id.Keyword_while},
54	};
55
56	fn getKeyword(bytes: []const u8) ?Id {
57	for (keywords) \|kw\| {
58	if (mem.eql(u8, kw.bytes, bytes)) {
59	return kw.id;
60	}
61	}
62	return null;
63	}
64
65	const StrLitKind = enum {Normal, C};
66
67	pub const Id = union(enum) {
68	Invalid,
69	Identifier,
70	StringLiteral: StrLitKind,
71	Eof,
72	Builtin,
73	Bang,
74	Equal,
75	EqualEqual,
76	BangEqual,
77	LParen,
78	RParen,
79	Semicolon,
80	Percent,
81	LBrace,
82	RBrace,
83	Period,
84	Ellipsis2,
85	Ellipsis3,
86	Minus,
87	Arrow,
88	Colon,
89	Slash,
90	Comma,
91	Ampersand,
92	AmpersandEqual,
93	IntegerLiteral,
94	FloatLiteral,
95	Keyword_align,
96	Keyword_and,
97	Keyword_asm,
98	Keyword_break,
99	Keyword_comptime,
100	Keyword_const,
101	Keyword_continue,
102	Keyword_defer,
103	Keyword_else,
104	Keyword_enum,
105	Keyword_error,
106	Keyword_export,
107	Keyword_extern,
108	Keyword_false,
109	Keyword_fn,
110	Keyword_for,
111	Keyword_goto,
112	Keyword_if,
113	Keyword_inline,
114	Keyword_nakedcc,
115	Keyword_noalias,
116	Keyword_null,
117	Keyword_or,
118	Keyword_packed,
119	Keyword_pub,
120	Keyword_return,
121	Keyword_stdcallcc,
122	Keyword_struct,
123	Keyword_switch,
124	Keyword_test,
125	Keyword_this,
126	Keyword_true,
127	Keyword_undefined,
128	Keyword_union,
129	Keyword_unreachable,
130	Keyword_use,
131	Keyword_var,
132	Keyword_volatile,
133	Keyword_while,
134	};
135	};
136
137	pub const Tokenizer = struct {
138	buffer: []const u8,
139	index: usize,
140	pending_invalid_token: ?Token,
141
142	pub const Location = struct {
143	line: usize,
144	column: usize,
145	line_start: usize,
146	line_end: usize,
147	};
148
149	pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {
150	var loc = Location {
151	.line = 0,
152	.column = 0,
153	.line_start = 0,
154	.line_end = 0,
155	};
156	for (self.buffer) \|c, i\| {
157	if (i == token.start) {
158	loc.line_end = i;
159	while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
160	return loc;
161	}
162	if (c == '\n') {
163	loc.line += 1;
164	loc.column = 0;
165	loc.line_start = i + 1;
166	} else {
167	loc.column += 1;
168	}
169	}
170	return loc;
171	}
172
173	/// For debugging purposes
174	pub fn dump(self: &Tokenizer, token: &const Token) void {
175	std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
176	}
177
178	/// buffer must end with "\n\n\n". This is so that attempting to decode
179	/// a the 3 trailing bytes of a 4-byte utf8 sequence is never a buffer overflow.
180	pub fn init(buffer: []const u8) Tokenizer {
181	std.debug.assert(buffer[buffer.len - 1] == '\n');
182	std.debug.assert(buffer[buffer.len - 2] == '\n');
183	std.debug.assert(buffer[buffer.len - 3] == '\n');
184	return Tokenizer {
185	.buffer = buffer,
186	.index = 0,
187	.pending_invalid_token = null,
188	};
189	}
190
191	const State = enum {
192	Start,
193	Identifier,
194	Builtin,
195	C,
196	StringLiteral,
197	StringLiteralBackslash,
198	Equal,
199	Bang,
200	Minus,
201	Slash,
202	LineComment,
203	Zero,
204	IntegerLiteral,
205	IntegerLiteralWithRadix,
206	NumberDot,
207	FloatFraction,
208	FloatExponentUnsigned,
209	FloatExponentNumber,
210	Ampersand,
211	Period,
212	Period2,
213	};
214
215	pub fn next(self: &Tokenizer) Token {
216	if (self.pending_invalid_token) \|token\| {
217	self.pending_invalid_token = null;
218	return token;
219	}
220	var state = State.Start;
221	var result = Token {
222	.id = Token.Id.Eof,
223	.start = self.index,
224	.end = undefined,
225	};
226	while (self.index < self.buffer.len) : (self.index += 1) {
227	const c = self.buffer[self.index];
228	switch (state) {
229	State.Start => switch (c) {
230	' ', '\n' => {
231	result.start = self.index + 1;
232	},
233	'c' => {
234	state = State.C;
235	result.id = Token.Id.Identifier;
236	},
237	'"' => {
238	state = State.StringLiteral;
239	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
240	},
241	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
242	state = State.Identifier;
243	result.id = Token.Id.Identifier;
244	},
245	'@' => {
246	state = State.Builtin;
247	result.id = Token.Id.Builtin;
248	},
249	'=' => {
250	state = State.Equal;
251	},
252	'!' => {
253	state = State.Bang;
254	},
255	'(' => {
256	result.id = Token.Id.LParen;
257	self.index += 1;
258	break;
259	},
260	')' => {
261	result.id = Token.Id.RParen;
262	self.index += 1;
263	break;
264	},
265	';' => {
266	result.id = Token.Id.Semicolon;
267	self.index += 1;
268	break;
269	},
270	',' => {
271	result.id = Token.Id.Comma;
272	self.index += 1;
273	break;
274	},
275	':' => {
276	result.id = Token.Id.Colon;
277	self.index += 1;
278	break;
279	},
280	'%' => {
281	result.id = Token.Id.Percent;
282	self.index += 1;
283	break;
284	},
285	'{' => {
286	result.id = Token.Id.LBrace;
287	self.index += 1;
288	break;
289	},
290	'}' => {
291	result.id = Token.Id.RBrace;
292	self.index += 1;
293	break;
294	},
295	'.' => {
296	state = State.Period;
297	},
298	'-' => {
299	state = State.Minus;
300	},
301	'/' => {
302	state = State.Slash;
303	},
304	'&' => {
305	state = State.Ampersand;
306	},
307	'0' => {
308	state = State.Zero;
309	result.id = Token.Id.IntegerLiteral;
310	},
311	'1'...'9' => {
312	state = State.IntegerLiteral;
313	result.id = Token.Id.IntegerLiteral;
314	},
315	else => {
316	result.id = Token.Id.Invalid;
317	self.index += 1;
318	break;
319	},
320	},
321	State.Ampersand => switch (c) {
322	'=' => {
323	result.id = Token.Id.AmpersandEqual;
324	self.index += 1;
325	break;
326	},
327	else => {
328	result.id = Token.Id.Ampersand;
329	break;
330	},
331	},
332	State.Identifier => switch (c) {
333	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
334	else => {
335	if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
336	result.id = id;
337	}
338	break;
339	},
340	},
341	State.Builtin => switch (c) {
342	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
343	else => break,
344	},
345	State.C => switch (c) {
346	'\\' => @panic("TODO"),
347	'"' => {
348	state = State.StringLiteral;
349	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
350	},
351	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
352	state = State.Identifier;
353	},
354	else => break,
355	},
356	State.StringLiteral => switch (c) {
357	'\\' => {
358	state = State.StringLiteralBackslash;
359	},
360	'"' => {
361	self.index += 1;
362	break;
363	},
364	'\n' => break, // Look for this error later.
365	else => self.checkLiteralCharacter(),
366	},
367
368	State.StringLiteralBackslash => switch (c) {
369	'\n' => break, // Look for this error later.
370	else => {
371	state = State.StringLiteral;
372	},
373	},
374
375	State.Bang => switch (c) {
376	'=' => {
377	result.id = Token.Id.BangEqual;
378	self.index += 1;
379	break;
380	},
381	else => {
382	result.id = Token.Id.Bang;
383	break;
384	},
385	},
386
387	State.Equal => switch (c) {
388	'=' => {
389	result.id = Token.Id.EqualEqual;
390	self.index += 1;
391	break;
392	},
393	else => {
394	result.id = Token.Id.Equal;
395	break;
396	},
397	},
398
399	State.Minus => switch (c) {
400	'>' => {
401	result.id = Token.Id.Arrow;
402	self.index += 1;
403	break;
404	},
405	else => {
406	result.id = Token.Id.Minus;
407	break;
408	},
409	},
410
411	State.Period => switch (c) {
412	'.' => {
413	state = State.Period2;
414	},
415	else => {
416	result.id = Token.Id.Period;
417	break;
418	},
419	},
420
421	State.Period2 => switch (c) {
422	'.' => {
423	result.id = Token.Id.Ellipsis3;
424	self.index += 1;
425	break;
426	},
427	else => {
428	result.id = Token.Id.Ellipsis2;
429	break;
430	},
431	},
432
433	State.Slash => switch (c) {
434	'/' => {
435	result.id = undefined;
436	state = State.LineComment;
437	},
438	else => {
439	result.id = Token.Id.Slash;
440	break;
441	},
442	},
443	State.LineComment => switch (c) {
444	'\n' => {
445	state = State.Start;
446	result = Token {
447	.id = Token.Id.Eof,
448	.start = self.index + 1,
449	.end = undefined,
450	};
451	},
452	else => self.checkLiteralCharacter(),
453	},
454	State.Zero => switch (c) {
455	'b', 'o', 'x' => {
456	state = State.IntegerLiteralWithRadix;
457	},
458	else => {
459	// reinterpret as a normal number
460	self.index -= 1;
461	state = State.IntegerLiteral;
462	},
463	},
464	State.IntegerLiteral => switch (c) {
465	'.' => {
466	state = State.NumberDot;
467	},
468	'p', 'P', 'e', 'E' => {
469	state = State.FloatExponentUnsigned;
470	},
471	'0'...'9' => {},
472	else => break,
473	},
474	State.IntegerLiteralWithRadix => switch (c) {
475	'.' => {
476	state = State.NumberDot;
477	},
478	'p', 'P' => {
479	state = State.FloatExponentUnsigned;
480	},
481	'0'...'9', 'a'...'f', 'A'...'F' => {},
482	else => break,
483	},
484	State.NumberDot => switch (c) {
485	'.' => {
486	self.index -= 1;
487	state = State.Start;
488	break;
489	},
490	else => {
491	self.index -= 1;
492	result.id = Token.Id.FloatLiteral;
493	state = State.FloatFraction;
494	},
495	},
496	State.FloatFraction => switch (c) {
497	'p', 'P' => {
498	state = State.FloatExponentUnsigned;
499	},
500	'0'...'9', 'a'...'f', 'A'...'F' => {},
501	else => break,
502	},
503	State.FloatExponentUnsigned => switch (c) {
504	'+', '-' => {
505	state = State.FloatExponentNumber;
506	},
507	else => {
508	// reinterpret as a normal exponent number
509	self.index -= 1;
510	state = State.FloatExponentNumber;
511	}
512	},
513	State.FloatExponentNumber => switch (c) {
514	'0'...'9', 'a'...'f', 'A'...'F' => {},
515	else => break,
516	},
517	}
518	}
519	result.end = self.index;
520
521	if (result.id == Token.Id.Eof) {
522	if (self.pending_invalid_token) \|token\| {
523	self.pending_invalid_token = null;
524	return token;
525	}
526	}
527
528	return result;
529	}
530
531	pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) []const u8 {
532	return self.buffer[token.start..token.end];
533	}
534
535	fn checkLiteralCharacter(self: &Tokenizer) void {
536	if (self.pending_invalid_token != null) return;
537	const invalid_length = self.getInvalidCharacterLength();
538	if (invalid_length == 0) return;
539	self.pending_invalid_token = Token {
540	.id = Token.Id.Invalid,
541	.start = self.index,
542	.end = self.index + invalid_length,
543	};
544	}
545
546	fn getInvalidCharacterLength(self: &Tokenizer) u3 {
547	const c0 = self.buffer[self.index];
548	if (c0 < 0x80) {
549	if (c0 < 0x20 or c0 == 0x7f) {
550	// ascii control codes are never allowed
551	// (note that \n was checked before we got here)
552	return 1;
553	}
554	// looks fine to me.
555	return 0;
556	} else {
557	// check utf8-encoded character.
558	const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
559	// the last 3 bytes in the buffer are guaranteed to be '\n',
560	// which means we don't need to do any bounds checking here.
561	const bytes = self.buffer[self.index..self.index + length];
562	switch (length) {
563	2 => {
564	const value = std.unicode.utf8Decode2(bytes) catch return length;
565	if (value == 0x85) return length; // U+0085 (NEL)
566	},
567	3 => {
568	const value = std.unicode.utf8Decode3(bytes) catch return length;
569	if (value == 0x2028) return length; // U+2028 (LS)
570	if (value == 0x2029) return length; // U+2029 (PS)
571	},
572	4 => {
573	_ = std.unicode.utf8Decode4(bytes) catch return length;
574	},
575	else => unreachable,
576	}
577	self.index += length - 1;
578	return 0;
579	}
580	}
581	};
582
583
584
585	test "tokenizer" {
586	testTokenize("test", []Token.Id {
587	Token.Id.Keyword_test,
588	});
589	}
590
591	test "tokenizer - invalid token characters" {
592	testTokenize("#", []Token.Id{Token.Id.Invalid});
593	testTokenize("`", []Token.Id{Token.Id.Invalid});
594	}
595
596	test "tokenizer - invalid literal/comment characters" {
597	testTokenize("\"\x00\"", []Token.Id {
598	Token.Id { .StringLiteral = Token.StrLitKind.Normal },
599	Token.Id.Invalid,
600	});
601	testTokenize("//\x00", []Token.Id {
602	Token.Id.Invalid,
603	});
604	testTokenize("//\x1f", []Token.Id {
605	Token.Id.Invalid,
606	});
607	testTokenize("//\x7f", []Token.Id {
608	Token.Id.Invalid,
609	});
610	}
611
612	test "tokenizer - utf8" {
613	testTokenize("//\xc2\x80", []Token.Id{});
614	testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});
615	}
616
617	test "tokenizer - invalid utf8" {
618	testTokenize("//\x80", []Token.Id{Token.Id.Invalid});
619	testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});
620	testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});
621	testTokenize("//\xff", []Token.Id{Token.Id.Invalid});
622	testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});
623	testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});
624	testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});
625	testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});
626	}
627
628	test "tokenizer - illegal unicode codepoints" {
629	// unicode newline characters.U+0085, U+2028, U+2029
630	testTokenize("//\xc2\x84", []Token.Id{});
631	testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});
632	testTokenize("//\xc2\x86", []Token.Id{});
633	testTokenize("//\xe2\x80\xa7", []Token.Id{});
634	testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});
635	testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});
636	testTokenize("//\xe2\x80\xaa", []Token.Id{});
637	}
638
639	fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
640	// (test authors, just make this bigger if you need it)
641	var padded_source: [0x100]u8 = undefined;
642	std.mem.copy(u8, padded_source[0..source.len], source);
643	padded_source[source.len + 0] = '\n';
644	padded_source[source.len + 1] = '\n';
645	padded_source[source.len + 2] = '\n';
646
647	var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
648	for (expected_tokens) \|expected_token_id\| {
649	const token = tokenizer.next();
650	std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
651	switch (expected_token_id) {
652	Token.Id.StringLiteral => \|expected_kind\| {
653	std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => \|kind\| kind, else => unreachable });
654	},
655	else => {},
656	}
657	}
658	std.debug.assert(tokenizer.next().id == Token.Id.Eof);
659	}