Commit 1fb308ceeea0259ad021d67945ea5adc10960a85 | zig

author	Andrew Kelley <andrew@ziglang.org> 2018-02-09 13:08:02-05:00
committer	Andrew Kelley <andrew@ziglang.org> 2018-02-09 13:08:02-05:00
log	1fb308ceeea0259ad021d67945ea5adc10960a85
tree	eceb252e06a6ed0cc179bc4cdf5a698057da6761
parent	3919afcad26d2359efe52f98cd4f2f0573527369

self hosted compiler: move tokenization and parsing to std lib

12 files changed, 2110 insertions(+), 2102 deletions(-)

CMakeLists.txt
build.zig
src-self-hosted
std
- index.zig
- zig

CMakeLists.txt+4

...	...	@@ -477,6 +477,10 @@ set(ZIG_STD_FILES
477	477	"special/panic.zig"
478	478	"special/test_runner.zig"
479	479	"unicode.zig"
	480	"zig/ast.zig"
	481	"zig/index.zig"
	482	"zig/parser.zig"
	483	"zig/tokenizer.zig"
480	484	)
481	485
482	486	set(ZIG_C_HEADER_FILES

build.zig-4

...	...	@@ -108,10 +108,6 @@ pub fn build(b: &Builder) !void {
108	108	"std/special/compiler_rt/index.zig", "compiler-rt", "Run the compiler_rt tests",
109	109	with_lldb));
110	110
111		test_step.dependOn(tests.addPkgTests(b, test_filter,
112		"src-self-hosted/main.zig", "fmt", "Run the fmt tests",
113		with_lldb));
114
115	111	test_step.dependOn(tests.addCompareOutputTests(b, test_filter));
116	112	test_step.dependOn(tests.addBuildExampleTests(b, test_filter));
117	113	test_step.dependOn(tests.addCompileErrorTests(b, test_filter));

src-self-hosted/ast.zig deleted-271

...	...	@@ -1,271 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const ArrayList = std.ArrayList;
4		const Token = @import("tokenizer.zig").Token;
5		const mem = std.mem;
6
7		pub const Node = struct {
8		id: Id,
9
10		pub const Id = enum {
11		Root,
12		VarDecl,
13		Identifier,
14		FnProto,
15		ParamDecl,
16		Block,
17		InfixOp,
18		PrefixOp,
19		IntegerLiteral,
20		FloatLiteral,
21		};
22
23		pub fn iterate(base: &Node, index: usize) ?&Node {
24		return switch (base.id) {
25		Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
26		Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
27		Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
28		Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
29		Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
30		Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
31		Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
32		Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
33		Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
34		Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
35		};
36		}
37
38		pub fn destroy(base: &Node, allocator: &mem.Allocator) void {
39		return switch (base.id) {
40		Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
41		Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
42		Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
43		Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
44		Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
45		Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
46		Id.InfixOp => allocator.destroy(@fieldParentPtr(NodeInfixOp, "base", base)),
47		Id.PrefixOp => allocator.destroy(@fieldParentPtr(NodePrefixOp, "base", base)),
48		Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
49		Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
50		};
51		}
52		};
53
54		pub const NodeRoot = struct {
55		base: Node,
56		decls: ArrayList(&Node),
57
58		pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
59		if (index < self.decls.len) {
60		return self.decls.items[self.decls.len - index - 1];
61		}
62		return null;
63		}
64		};
65
66		pub const NodeVarDecl = struct {
67		base: Node,
68		visib_token: ?Token,
69		name_token: Token,
70		eq_token: Token,
71		mut_token: Token,
72		comptime_token: ?Token,
73		extern_token: ?Token,
74		lib_name: ?&Node,
75		type_node: ?&Node,
76		align_node: ?&Node,
77		init_node: ?&Node,
78
79		pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
80		var i = index;
81
82		if (self.type_node) \|type_node\| {
83		if (i < 1) return type_node;
84		i -= 1;
85		}
86
87		if (self.align_node) \|align_node\| {
88		if (i < 1) return align_node;
89		i -= 1;
90		}
91
92		if (self.init_node) \|init_node\| {
93		if (i < 1) return init_node;
94		i -= 1;
95		}
96
97		return null;
98		}
99		};
100
101		pub const NodeIdentifier = struct {
102		base: Node,
103		name_token: Token,
104
105		pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
106		return null;
107		}
108		};
109
110		pub const NodeFnProto = struct {
111		base: Node,
112		visib_token: ?Token,
113		fn_token: Token,
114		name_token: ?Token,
115		params: ArrayList(&Node),
116		return_type: &Node,
117		var_args_token: ?Token,
118		extern_token: ?Token,
119		inline_token: ?Token,
120		cc_token: ?Token,
121		body_node: ?&Node,
122		lib_name: ?&Node, // populated if this is an extern declaration
123		align_expr: ?&Node, // populated if align(A) is present
124
125		pub fn iterate(self: &NodeFnProto, index: usize) ?&Node {
126		var i = index;
127
128		if (self.body_node) \|body_node\| {
129		if (i < 1) return body_node;
130		i -= 1;
131		}
132
133		if (i < 1) return self.return_type;
134		i -= 1;
135
136		if (self.align_expr) \|align_expr\| {
137		if (i < 1) return align_expr;
138		i -= 1;
139		}
140
141		if (i < self.params.len) return self.params.items[self.params.len - i - 1];
142		i -= self.params.len;
143
144		if (self.lib_name) \|lib_name\| {
145		if (i < 1) return lib_name;
146		i -= 1;
147		}
148
149		return null;
150		}
151		};
152
153		pub const NodeParamDecl = struct {
154		base: Node,
155		comptime_token: ?Token,
156		noalias_token: ?Token,
157		name_token: ?Token,
158		type_node: &Node,
159		var_args_token: ?Token,
160
161		pub fn iterate(self: &NodeParamDecl, index: usize) ?&Node {
162		var i = index;
163
164		if (i < 1) return self.type_node;
165		i -= 1;
166
167		return null;
168		}
169		};
170
171		pub const NodeBlock = struct {
172		base: Node,
173		begin_token: Token,
174		end_token: Token,
175		statements: ArrayList(&Node),
176
177		pub fn iterate(self: &NodeBlock, index: usize) ?&Node {
178		var i = index;
179
180		if (i < self.statements.len) return self.statements.items[i];
181		i -= self.statements.len;
182
183		return null;
184		}
185		};
186
187		pub const NodeInfixOp = struct {
188		base: Node,
189		op_token: Token,
190		lhs: &Node,
191		op: InfixOp,
192		rhs: &Node,
193
194		const InfixOp = enum {
195		EqualEqual,
196		BangEqual,
197		};
198
199		pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
200		var i = index;
201
202		if (i < 1) return self.lhs;
203		i -= 1;
204
205		switch (self.op) {
206		InfixOp.EqualEqual => {},
207		InfixOp.BangEqual => {},
208		}
209
210		if (i < 1) return self.rhs;
211		i -= 1;
212
213		return null;
214		}
215		};
216
217		pub const NodePrefixOp = struct {
218		base: Node,
219		op_token: Token,
220		op: PrefixOp,
221		rhs: &Node,
222
223		const PrefixOp = union(enum) {
224		Return,
225		AddrOf: AddrOfInfo,
226		};
227		const AddrOfInfo = struct {
228		align_expr: ?&Node,
229		bit_offset_start_token: ?Token,
230		bit_offset_end_token: ?Token,
231		const_token: ?Token,
232		volatile_token: ?Token,
233		};
234
235		pub fn iterate(self: &NodePrefixOp, index: usize) ?&Node {
236		var i = index;
237
238		switch (self.op) {
239		PrefixOp.Return => {},
240		PrefixOp.AddrOf => \|addr_of_info\| {
241		if (addr_of_info.align_expr) \|align_expr\| {
242		if (i < 1) return align_expr;
243		i -= 1;
244		}
245		},
246		}
247
248		if (i < 1) return self.rhs;
249		i -= 1;
250
251		return null;
252		}
253		};
254
255		pub const NodeIntegerLiteral = struct {
256		base: Node,
257		token: Token,
258
259		pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
260		return null;
261		}
262		};
263
264		pub const NodeFloatLiteral = struct {
265		base: Node,
266		token: Token,
267
268		pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
269		return null;
270		}
271		};

src-self-hosted/main.zig-5

...	...	@@ -622,8 +622,3 @@ fn findZigLibDir(allocator: &mem.Allocator) ![]u8 {
622	622
623	623	return error.FileNotFound;
624	624	}
625
626		test "import tests" {
627		_ = @import("tokenizer.zig");
628		_ = @import("parser.zig");
629		}

src-self-hosted/module.zig+3-3

...	...	@@ -8,9 +8,9 @@ const c = @import("c.zig");
8	8	const builtin = @import("builtin");
9	9	const Target = @import("target.zig").Target;
10	10	const warn = std.debug.warn;
11		const Tokenizer = @import("tokenizer.zig").Tokenizer;
12		const Token = @import("tokenizer.zig").Token;
13		const Parser = @import("parser.zig").Parser;
	11	const Tokenizer = std.zig.Tokenizer;
	12	const Token = std.zig.Token;
	13	const Parser = std.zig.Parser;
14	14	const ArrayList = std.ArrayList;
15	15
16	16	pub const Module = struct {

src-self-hosted/parser.zig deleted-1160

...	...	@@ -1,1160 +0,0 @@
1		const std = @import("std");
2		const assert = std.debug.assert;
3		const ArrayList = std.ArrayList;
4		const mem = std.mem;
5		const ast = @import("ast.zig");
6		const Tokenizer = @import("tokenizer.zig").Tokenizer;
7		const Token = @import("tokenizer.zig").Token;
8		const builtin = @import("builtin");
9		const io = std.io;
10
11		// TODO when we make parse errors into error types instead of printing directly,
12		// get rid of this
13		const warn = std.debug.warn;
14
15		pub const Parser = struct {
16		allocator: &mem.Allocator,
17		tokenizer: &Tokenizer,
18		put_back_tokens: [2]Token,
19		put_back_count: usize,
20		source_file_name: []const u8,
21
22		pub const Tree = struct {
23		root_node: &ast.NodeRoot,
24
25		pub fn deinit(self: &const Tree) void {
26		// TODO free the whole arena
27		}
28		};
29
30		// This memory contents are used only during a function call. It's used to repurpose memory;
31		// we reuse the same bytes for the stack data structure used by parsing, tree rendering, and
32		// source rendering.
33		const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
34		utility_bytes: []align(utility_bytes_align) u8,
35
36		/// `allocator` should be an arena allocator. Parser never calls free on anything. After you're
37		/// done with a Parser, free the arena. After the arena is freed, no member functions of Parser
38		/// may be called.
39		pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) Parser {
40		return Parser {
41		.allocator = allocator,
42		.tokenizer = tokenizer,
43		.put_back_tokens = undefined,
44		.put_back_count = 0,
45		.source_file_name = source_file_name,
46		.utility_bytes = []align(utility_bytes_align) u8{},
47		};
48		}
49
50		pub fn deinit(self: &Parser) void {
51		self.allocator.free(self.utility_bytes);
52		}
53
54		const TopLevelDeclCtx = struct {
55		visib_token: ?Token,
56		extern_token: ?Token,
57		};
58
59		const DestPtr = union(enum) {
60		Field: &&ast.Node,
61		NullableField: &?&ast.Node,
62		List: &ArrayList(&ast.Node),
63
64		pub fn store(self: &const DestPtr, value: &ast.Node) !void {
65		switch (*self) {
66		DestPtr.Field => \|ptr\| *ptr = value,
67		DestPtr.NullableField => \|ptr\| *ptr = value,
68		DestPtr.List => \|list\| try list.append(value),
69		}
70		}
71		};
72
73		const State = union(enum) {
74		TopLevel,
75		TopLevelExtern: ?Token,
76		TopLevelDecl: TopLevelDeclCtx,
77		Expression: DestPtr,
78		ExpectOperand,
79		Operand: &ast.Node,
80		AfterOperand,
81		InfixOp: &ast.NodeInfixOp,
82		PrefixOp: &ast.NodePrefixOp,
83		AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
84		TypeExpr: DestPtr,
85		VarDecl: &ast.NodeVarDecl,
86		VarDeclAlign: &ast.NodeVarDecl,
87		VarDeclEq: &ast.NodeVarDecl,
88		ExpectToken: @TagType(Token.Id),
89		FnProto: &ast.NodeFnProto,
90		FnProtoAlign: &ast.NodeFnProto,
91		ParamDecl: &ast.NodeFnProto,
92		ParamDeclComma,
93		FnDef: &ast.NodeFnProto,
94		Block: &ast.NodeBlock,
95		Statement: &ast.NodeBlock,
96		};
97
98		/// Returns an AST tree, allocated with the parser's allocator.
99		/// Result should be freed with `freeAst` when done.
100		pub fn parse(self: &Parser) !Tree {
101		var stack = self.initUtilityArrayList(State);
102		defer self.deinitUtilityArrayList(stack);
103
104		const root_node = try self.createRoot();
105		// TODO errdefer arena free root node
106
107		try stack.append(State.TopLevel);
108
109		while (true) {
110		//{
111		// const token = self.getNextToken();
112		// warn("{} ", @tagName(token.id));
113		// self.putBackToken(token);
114		// var i: usize = stack.len;
115		// while (i != 0) {
116		// i -= 1;
117		// warn("{} ", @tagName(stack.items[i]));
118		// }
119		// warn("\n");
120		//}
121
122		// This gives us 1 free append that can't fail
123		const state = stack.pop();
124
125		switch (state) {
126		State.TopLevel => {
127		const token = self.getNextToken();
128		switch (token.id) {
129		Token.Id.Keyword_pub, Token.Id.Keyword_export => {
130		stack.append(State { .TopLevelExtern = token }) catch unreachable;
131		continue;
132		},
133		Token.Id.Eof => return Tree {.root_node = root_node},
134		else => {
135		self.putBackToken(token);
136		// TODO shouldn't need this cast
137		stack.append(State { .TopLevelExtern = null }) catch unreachable;
138		continue;
139		},
140		}
141		},
142		State.TopLevelExtern => \|visib_token\| {
143		const token = self.getNextToken();
144		if (token.id == Token.Id.Keyword_extern) {
145		stack.append(State {
146		.TopLevelDecl = TopLevelDeclCtx {
147		.visib_token = visib_token,
148		.extern_token = token,
149		},
150		}) catch unreachable;
151		continue;
152		}
153		self.putBackToken(token);
154		stack.append(State {
155		.TopLevelDecl = TopLevelDeclCtx {
156		.visib_token = visib_token,
157		.extern_token = null,
158		},
159		}) catch unreachable;
160		continue;
161		},
162		State.TopLevelDecl => \|ctx\| {
163		const token = self.getNextToken();
164		switch (token.id) {
165		Token.Id.Keyword_var, Token.Id.Keyword_const => {
166		stack.append(State.TopLevel) catch unreachable;
167		// TODO shouldn't need these casts
168		const var_decl_node = try self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
169		token, (?Token)(null), ctx.extern_token);
170		try stack.append(State { .VarDecl = var_decl_node });
171		continue;
172		},
173		Token.Id.Keyword_fn => {
174		stack.append(State.TopLevel) catch unreachable;
175		// TODO shouldn't need these casts
176		const fn_proto = try self.createAttachFnProto(&root_node.decls, token,
177		ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
178		try stack.append(State { .FnDef = fn_proto });
179		try stack.append(State { .FnProto = fn_proto });
180		continue;
181		},
182		Token.Id.StringLiteral => {
183		@panic("TODO extern with string literal");
184		},
185		Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
186		stack.append(State.TopLevel) catch unreachable;
187		const fn_token = try self.eatToken(Token.Id.Keyword_fn);
188		// TODO shouldn't need this cast
189		const fn_proto = try self.createAttachFnProto(&root_node.decls, fn_token,
190		ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
191		try stack.append(State { .FnDef = fn_proto });
192		try stack.append(State { .FnProto = fn_proto });
193		continue;
194		},
195		else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
196		}
197		},
198		State.VarDecl => \|var_decl\| {
199		var_decl.name_token = try self.eatToken(Token.Id.Identifier);
200		stack.append(State { .VarDeclAlign = var_decl }) catch unreachable;
201
202		const next_token = self.getNextToken();
203		if (next_token.id == Token.Id.Colon) {
204		try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
205		continue;
206		}
207
208		self.putBackToken(next_token);
209		continue;
210		},
211		State.VarDeclAlign => \|var_decl\| {
212		stack.append(State { .VarDeclEq = var_decl }) catch unreachable;
213
214		const next_token = self.getNextToken();
215		if (next_token.id == Token.Id.Keyword_align) {
216		_ = try self.eatToken(Token.Id.LParen);
217		try stack.append(State { .ExpectToken = Token.Id.RParen });
218		try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} });
219		continue;
220		}
221
222		self.putBackToken(next_token);
223		continue;
224		},
225		State.VarDeclEq => \|var_decl\| {
226		const token = self.getNextToken();
227		if (token.id == Token.Id.Equal) {
228		var_decl.eq_token = token;
229		stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
230		try stack.append(State {
231		.Expression = DestPtr {.NullableField = &var_decl.init_node},
232		});
233		continue;
234		}
235		if (token.id == Token.Id.Semicolon) {
236		continue;
237		}
238		return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
239		},
240		State.ExpectToken => \|token_id\| {
241		_ = try self.eatToken(token_id);
242		continue;
243		},
244
245		State.Expression => \|dest_ptr\| {
246		// save the dest_ptr for later
247		stack.append(state) catch unreachable;
248		try stack.append(State.ExpectOperand);
249		continue;
250		},
251		State.ExpectOperand => {
252		// we'll either get an operand (like 1 or x),
253		// or a prefix operator (like ~ or return).
254		const token = self.getNextToken();
255		switch (token.id) {
256		Token.Id.Keyword_return => {
257		try stack.append(State { .PrefixOp = try self.createPrefixOp(token,
258		ast.NodePrefixOp.PrefixOp.Return) });
259		try stack.append(State.ExpectOperand);
260		continue;
261		},
262		Token.Id.Ampersand => {
263		const prefix_op = try self.createPrefixOp(token, ast.NodePrefixOp.PrefixOp{
264		.AddrOf = ast.NodePrefixOp.AddrOfInfo {
265		.align_expr = null,
266		.bit_offset_start_token = null,
267		.bit_offset_end_token = null,
268		.const_token = null,
269		.volatile_token = null,
270		}
271		});
272		try stack.append(State { .PrefixOp = prefix_op });
273		try stack.append(State.ExpectOperand);
274		try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf });
275		continue;
276		},
277		Token.Id.Identifier => {
278		try stack.append(State {
279		.Operand = &(try self.createIdentifier(token)).base
280		});
281		try stack.append(State.AfterOperand);
282		continue;
283		},
284		Token.Id.IntegerLiteral => {
285		try stack.append(State {
286		.Operand = &(try self.createIntegerLiteral(token)).base
287		});
288		try stack.append(State.AfterOperand);
289		continue;
290		},
291		Token.Id.FloatLiteral => {
292		try stack.append(State {
293		.Operand = &(try self.createFloatLiteral(token)).base
294		});
295		try stack.append(State.AfterOperand);
296		continue;
297		},
298		else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
299		}
300		},
301
302		State.AfterOperand => {
303		// we'll either get an infix operator (like != or ^),
304		// or a postfix operator (like () or {}),
305		// otherwise this expression is done (like on a ; or else).
306		var token = self.getNextToken();
307		switch (token.id) {
308		Token.Id.EqualEqual => {
309		try stack.append(State {
310		.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.EqualEqual)
311		});
312		try stack.append(State.ExpectOperand);
313		continue;
314		},
315		Token.Id.BangEqual => {
316		try stack.append(State {
317		.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.BangEqual)
318		});
319		try stack.append(State.ExpectOperand);
320		continue;
321		},
322		else => {
323		// no postfix/infix operator after this operand.
324		self.putBackToken(token);
325		// reduce the stack
326		var expression: &ast.Node = stack.pop().Operand;
327		while (true) {
328		switch (stack.pop()) {
329		State.Expression => \|dest_ptr\| {
330		// we're done
331		try dest_ptr.store(expression);
332		break;
333		},
334		State.InfixOp => \|infix_op\| {
335		infix_op.rhs = expression;
336		infix_op.lhs = stack.pop().Operand;
337		expression = &infix_op.base;
338		continue;
339		},
340		State.PrefixOp => \|prefix_op\| {
341		prefix_op.rhs = expression;
342		expression = &prefix_op.base;
343		continue;
344		},
345		else => unreachable,
346		}
347		}
348		continue;
349		},
350		}
351		},
352
353		State.AddrOfModifiers => \|addr_of_info\| {
354		var token = self.getNextToken();
355		switch (token.id) {
356		Token.Id.Keyword_align => {
357		stack.append(state) catch unreachable;
358		if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers");
359		_ = try self.eatToken(Token.Id.LParen);
360		try stack.append(State { .ExpectToken = Token.Id.RParen });
361		try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} });
362		continue;
363		},
364		Token.Id.Keyword_const => {
365		stack.append(state) catch unreachable;
366		if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const");
367		addr_of_info.const_token = token;
368		continue;
369		},
370		Token.Id.Keyword_volatile => {
371		stack.append(state) catch unreachable;
372		if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile");
373		addr_of_info.volatile_token = token;
374		continue;
375		},
376		else => {
377		self.putBackToken(token);
378		continue;
379		},
380		}
381		},
382
383		State.TypeExpr => \|dest_ptr\| {
384		const token = self.getNextToken();
385		if (token.id == Token.Id.Keyword_var) {
386		@panic("TODO param with type var");
387		}
388		self.putBackToken(token);
389
390		stack.append(State { .Expression = dest_ptr }) catch unreachable;
391		continue;
392		},
393
394		State.FnProto => \|fn_proto\| {
395		stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable;
396		try stack.append(State { .ParamDecl = fn_proto });
397		try stack.append(State { .ExpectToken = Token.Id.LParen });
398
399		const next_token = self.getNextToken();
400		if (next_token.id == Token.Id.Identifier) {
401		fn_proto.name_token = next_token;
402		continue;
403		}
404		self.putBackToken(next_token);
405		continue;
406		},
407
408		State.FnProtoAlign => \|fn_proto\| {
409		const token = self.getNextToken();
410		if (token.id == Token.Id.Keyword_align) {
411		@panic("TODO fn proto align");
412		}
413		self.putBackToken(token);
414		stack.append(State {
415		.TypeExpr = DestPtr {.Field = &fn_proto.return_type},
416		}) catch unreachable;
417		continue;
418		},
419
420		State.ParamDecl => \|fn_proto\| {
421		var token = self.getNextToken();
422		if (token.id == Token.Id.RParen) {
423		continue;
424		}
425		const param_decl = try self.createAttachParamDecl(&fn_proto.params);
426		if (token.id == Token.Id.Keyword_comptime) {
427		param_decl.comptime_token = token;
428		token = self.getNextToken();
429		} else if (token.id == Token.Id.Keyword_noalias) {
430		param_decl.noalias_token = token;
431		token = self.getNextToken();
432		}
433		if (token.id == Token.Id.Identifier) {
434		const next_token = self.getNextToken();
435		if (next_token.id == Token.Id.Colon) {
436		param_decl.name_token = token;
437		token = self.getNextToken();
438		} else {
439		self.putBackToken(next_token);
440		}
441		}
442		if (token.id == Token.Id.Ellipsis3) {
443		param_decl.var_args_token = token;
444		stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable;
445		continue;
446		} else {
447		self.putBackToken(token);
448		}
449
450		stack.append(State { .ParamDecl = fn_proto }) catch unreachable;
451		try stack.append(State.ParamDeclComma);
452		try stack.append(State {
453		.TypeExpr = DestPtr {.Field = &param_decl.type_node}
454		});
455		continue;
456		},
457
458		State.ParamDeclComma => {
459		const token = self.getNextToken();
460		switch (token.id) {
461		Token.Id.RParen => {
462		_ = stack.pop(); // pop off the ParamDecl
463		continue;
464		},
465		Token.Id.Comma => continue,
466		else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
467		}
468		},
469
470		State.FnDef => \|fn_proto\| {
471		const token = self.getNextToken();
472		switch(token.id) {
473		Token.Id.LBrace => {
474		const block = try self.createBlock(token);
475		fn_proto.body_node = &block.base;
476		stack.append(State { .Block = block }) catch unreachable;
477		continue;
478		},
479		Token.Id.Semicolon => continue,
480		else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
481		}
482		},
483
484		State.Block => \|block\| {
485		const token = self.getNextToken();
486		switch (token.id) {
487		Token.Id.RBrace => {
488		block.end_token = token;
489		continue;
490		},
491		else => {
492		self.putBackToken(token);
493		stack.append(State { .Block = block }) catch unreachable;
494		try stack.append(State { .Statement = block });
495		continue;
496		},
497		}
498		},
499
500		State.Statement => \|block\| {
501		{
502		// Look for comptime var, comptime const
503		const comptime_token = self.getNextToken();
504		if (comptime_token.id == Token.Id.Keyword_comptime) {
505		const mut_token = self.getNextToken();
506		if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
507		// TODO shouldn't need these casts
508		const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
509		mut_token, (?Token)(comptime_token), (?Token)(null));
510		try stack.append(State { .VarDecl = var_decl });
511		continue;
512		}
513		self.putBackToken(mut_token);
514		}
515		self.putBackToken(comptime_token);
516		}
517		{
518		// Look for const, var
519		const mut_token = self.getNextToken();
520		if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
521		// TODO shouldn't need these casts
522		const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
523		mut_token, (?Token)(null), (?Token)(null));
524		try stack.append(State { .VarDecl = var_decl });
525		continue;
526		}
527		self.putBackToken(mut_token);
528		}
529
530		stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
531		try stack.append(State { .Expression = DestPtr{.List = &block.statements} });
532		continue;
533		},
534
535		// These are data, not control flow.
536		State.InfixOp => unreachable,
537		State.PrefixOp => unreachable,
538		State.Operand => unreachable,
539		}
540		@import("std").debug.panic("{}", @tagName(state));
541		//unreachable;
542		}
543		}
544
545		fn createRoot(self: &Parser) !&ast.NodeRoot {
546		const node = try self.allocator.create(ast.NodeRoot);
547
548		*node = ast.NodeRoot {
549		.base = ast.Node {.id = ast.Node.Id.Root},
550		.decls = ArrayList(&ast.Node).init(self.allocator),
551		};
552		return node;
553		}
554
555		fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
556		extern_token: &const ?Token) !&ast.NodeVarDecl
557		{
558		const node = try self.allocator.create(ast.NodeVarDecl);
559
560		*node = ast.NodeVarDecl {
561		.base = ast.Node {.id = ast.Node.Id.VarDecl},
562		.visib_token = *visib_token,
563		.mut_token = *mut_token,
564		.comptime_token = *comptime_token,
565		.extern_token = *extern_token,
566		.type_node = null,
567		.align_node = null,
568		.init_node = null,
569		.lib_name = null,
570		// initialized later
571		.name_token = undefined,
572		.eq_token = undefined,
573		};
574		return node;
575		}
576
577		fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
578		cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto
579		{
580		const node = try self.allocator.create(ast.NodeFnProto);
581
582		*node = ast.NodeFnProto {
583		.base = ast.Node {.id = ast.Node.Id.FnProto},
584		.visib_token = *visib_token,
585		.name_token = null,
586		.fn_token = *fn_token,
587		.params = ArrayList(&ast.Node).init(self.allocator),
588		.return_type = undefined,
589		.var_args_token = null,
590		.extern_token = *extern_token,
591		.inline_token = *inline_token,
592		.cc_token = *cc_token,
593		.body_node = null,
594		.lib_name = null,
595		.align_expr = null,
596		};
597		return node;
598		}
599
600		fn createParamDecl(self: &Parser) !&ast.NodeParamDecl {
601		const node = try self.allocator.create(ast.NodeParamDecl);
602
603		*node = ast.NodeParamDecl {
604		.base = ast.Node {.id = ast.Node.Id.ParamDecl},
605		.comptime_token = null,
606		.noalias_token = null,
607		.name_token = null,
608		.type_node = undefined,
609		.var_args_token = null,
610		};
611		return node;
612		}
613
614		fn createBlock(self: &Parser, begin_token: &const Token) !&ast.NodeBlock {
615		const node = try self.allocator.create(ast.NodeBlock);
616
617		*node = ast.NodeBlock {
618		.base = ast.Node {.id = ast.Node.Id.Block},
619		.begin_token = *begin_token,
620		.end_token = undefined,
621		.statements = ArrayList(&ast.Node).init(self.allocator),
622		};
623		return node;
624		}
625
626		fn createInfixOp(self: &Parser, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp {
627		const node = try self.allocator.create(ast.NodeInfixOp);
628
629		*node = ast.NodeInfixOp {
630		.base = ast.Node {.id = ast.Node.Id.InfixOp},
631		.op_token = *op_token,
632		.lhs = undefined,
633		.op = *op,
634		.rhs = undefined,
635		};
636		return node;
637		}
638
639		fn createPrefixOp(self: &Parser, op_token: &const Token, op: &const ast.NodePrefixOp.PrefixOp) !&ast.NodePrefixOp {
640		const node = try self.allocator.create(ast.NodePrefixOp);
641
642		*node = ast.NodePrefixOp {
643		.base = ast.Node {.id = ast.Node.Id.PrefixOp},
644		.op_token = *op_token,
645		.op = *op,
646		.rhs = undefined,
647		};
648		return node;
649		}
650
651		fn createIdentifier(self: &Parser, name_token: &const Token) !&ast.NodeIdentifier {
652		const node = try self.allocator.create(ast.NodeIdentifier);
653
654		*node = ast.NodeIdentifier {
655		.base = ast.Node {.id = ast.Node.Id.Identifier},
656		.name_token = *name_token,
657		};
658		return node;
659		}
660
661		fn createIntegerLiteral(self: &Parser, token: &const Token) !&ast.NodeIntegerLiteral {
662		const node = try self.allocator.create(ast.NodeIntegerLiteral);
663
664		*node = ast.NodeIntegerLiteral {
665		.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
666		.token = *token,
667		};
668		return node;
669		}
670
671		fn createFloatLiteral(self: &Parser, token: &const Token) !&ast.NodeFloatLiteral {
672		const node = try self.allocator.create(ast.NodeFloatLiteral);
673
674		*node = ast.NodeFloatLiteral {
675		.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
676		.token = *token,
677		};
678		return node;
679		}
680
681		fn createAttachIdentifier(self: &Parser, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {
682		const node = try self.createIdentifier(name_token);
683		try dest_ptr.store(&node.base);
684		return node;
685		}
686
687		fn createAttachParamDecl(self: &Parser, list: &ArrayList(&ast.Node)) !&ast.NodeParamDecl {
688		const node = try self.createParamDecl();
689		try list.append(&node.base);
690		return node;
691		}
692
693		fn createAttachFnProto(self: &Parser, list: &ArrayList(&ast.Node), fn_token: &const Token,
694		extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
695		inline_token: &const ?Token) !&ast.NodeFnProto
696		{
697		const node = try self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
698		try list.append(&node.base);
699		return node;
700		}
701
702		fn createAttachVarDecl(self: &Parser, list: &ArrayList(&ast.Node), visib_token: &const ?Token,
703		mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl
704		{
705		const node = try self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
706		try list.append(&node.base);
707		return node;
708		}
709
710		fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) error {
711		const loc = self.tokenizer.getTokenLocation(token);
712		warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
713		warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
714		{
715		var i: usize = 0;
716		while (i < loc.column) : (i += 1) {
717		warn(" ");
718		}
719		}
720		{
721		const caret_count = token.end - token.start;
722		var i: usize = 0;
723		while (i < caret_count) : (i += 1) {
724		warn("~");
725		}
726		}
727		warn("\n");
728		return error.ParseError;
729		}
730
731		fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void {
732		if (token.id != id) {
733		return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
734		}
735		}
736
737		fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token {
738		const token = self.getNextToken();
739		try self.expectToken(token, id);
740		return token;
741		}
742
743		fn putBackToken(self: &Parser, token: &const Token) void {
744		self.put_back_tokens[self.put_back_count] = *token;
745		self.put_back_count += 1;
746		}
747
748		fn getNextToken(self: &Parser) Token {
749		if (self.put_back_count != 0) {
750		const put_back_index = self.put_back_count - 1;
751		const put_back_token = self.put_back_tokens[put_back_index];
752		self.put_back_count = put_back_index;
753		return put_back_token;
754		} else {
755		return self.tokenizer.next();
756		}
757		}
758
759		const RenderAstFrame = struct {
760		node: &ast.Node,
761		indent: usize,
762		};
763
764		pub fn renderAst(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
765		var stack = self.initUtilityArrayList(RenderAstFrame);
766		defer self.deinitUtilityArrayList(stack);
767
768		try stack.append(RenderAstFrame {
769		.node = &root_node.base,
770		.indent = 0,
771		});
772
773		while (stack.popOrNull()) \|frame\| {
774		{
775		var i: usize = 0;
776		while (i < frame.indent) : (i += 1) {
777		try stream.print(" ");
778		}
779		}
780		try stream.print("{}\n", @tagName(frame.node.id));
781		var child_i: usize = 0;
782		while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
783		try stack.append(RenderAstFrame {
784		.node = child,
785		.indent = frame.indent + 2,
786		});
787		}
788		}
789		}
790
791		const RenderState = union(enum) {
792		TopLevelDecl: &ast.Node,
793		FnProtoRParen: &ast.NodeFnProto,
794		ParamDecl: &ast.Node,
795		Text: []const u8,
796		Expression: &ast.Node,
797		VarDecl: &ast.NodeVarDecl,
798		Statement: &ast.Node,
799		PrintIndent,
800		Indent: usize,
801		};
802
803		pub fn renderSource(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
804		var stack = self.initUtilityArrayList(RenderState);
805		defer self.deinitUtilityArrayList(stack);
806
807		{
808		var i = root_node.decls.len;
809		while (i != 0) {
810		i -= 1;
811		const decl = root_node.decls.items[i];
812		try stack.append(RenderState {.TopLevelDecl = decl});
813		}
814		}
815
816		const indent_delta = 4;
817		var indent: usize = 0;
818		while (stack.popOrNull()) \|state\| {
819		switch (state) {
820		RenderState.TopLevelDecl => \|decl\| {
821		switch (decl.id) {
822		ast.Node.Id.FnProto => {
823		const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl);
824		if (fn_proto.visib_token) \|visib_token\| {
825		switch (visib_token.id) {
826		Token.Id.Keyword_pub => try stream.print("pub "),
827		Token.Id.Keyword_export => try stream.print("export "),
828		else => unreachable,
829		}
830		}
831		if (fn_proto.extern_token) \|extern_token\| {
832		try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
833		}
834		try stream.print("fn");
835
836		if (fn_proto.name_token) \|name_token\| {
837		try stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
838		}
839
840		try stream.print("(");
841
842		try stack.append(RenderState { .Text = "\n" });
843		if (fn_proto.body_node == null) {
844		try stack.append(RenderState { .Text = ";" });
845		}
846
847		try stack.append(RenderState { .FnProtoRParen = fn_proto});
848		var i = fn_proto.params.len;
849		while (i != 0) {
850		i -= 1;
851		const param_decl_node = fn_proto.params.items[i];
852		try stack.append(RenderState { .ParamDecl = param_decl_node});
853		if (i != 0) {
854		try stack.append(RenderState { .Text = ", " });
855		}
856		}
857		},
858		ast.Node.Id.VarDecl => {
859		const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
860		try stack.append(RenderState { .Text = "\n"});
861		try stack.append(RenderState { .VarDecl = var_decl});
862
863		},
864		else => unreachable,
865		}
866		},
867
868		RenderState.VarDecl => \|var_decl\| {
869		if (var_decl.visib_token) \|visib_token\| {
870		try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
871		}
872		if (var_decl.extern_token) \|extern_token\| {
873		try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
874		if (var_decl.lib_name != null) {
875		@panic("TODO");
876		}
877		}
878		if (var_decl.comptime_token) \|comptime_token\| {
879		try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
880		}
881		try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
882		try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
883
884		try stack.append(RenderState { .Text = ";" });
885		if (var_decl.init_node) \|init_node\| {
886		try stack.append(RenderState { .Expression = init_node });
887		try stack.append(RenderState { .Text = " = " });
888		}
889		if (var_decl.align_node) \|align_node\| {
890		try stack.append(RenderState { .Text = ")" });
891		try stack.append(RenderState { .Expression = align_node });
892		try stack.append(RenderState { .Text = " align(" });
893		}
894		if (var_decl.type_node) \|type_node\| {
895		try stream.print(": ");
896		try stack.append(RenderState { .Expression = type_node });
897		}
898		},
899
900		RenderState.ParamDecl => \|base\| {
901		const param_decl = @fieldParentPtr(ast.NodeParamDecl, "base", base);
902		if (param_decl.comptime_token) \|comptime_token\| {
903		try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
904		}
905		if (param_decl.noalias_token) \|noalias_token\| {
906		try stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
907		}
908		if (param_decl.name_token) \|name_token\| {
909		try stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
910		}
911		if (param_decl.var_args_token) \|var_args_token\| {
912		try stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
913		} else {
914		try stack.append(RenderState { .Expression = param_decl.type_node});
915		}
916		},
917		RenderState.Text => \|bytes\| {
918		try stream.write(bytes);
919		},
920		RenderState.Expression => \|base\| switch (base.id) {
921		ast.Node.Id.Identifier => {
922		const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base);
923		try stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
924		},
925		ast.Node.Id.Block => {
926		const block = @fieldParentPtr(ast.NodeBlock, "base", base);
927		try stream.write("{");
928		try stack.append(RenderState { .Text = "}"});
929		try stack.append(RenderState.PrintIndent);
930		try stack.append(RenderState { .Indent = indent});
931		try stack.append(RenderState { .Text = "\n"});
932		var i = block.statements.len;
933		while (i != 0) {
934		i -= 1;
935		const statement_node = block.statements.items[i];
936		try stack.append(RenderState { .Statement = statement_node});
937		try stack.append(RenderState.PrintIndent);
938		try stack.append(RenderState { .Indent = indent + indent_delta});
939		try stack.append(RenderState { .Text = "\n" });
940		}
941		},
942		ast.Node.Id.InfixOp => {
943		const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);
944		try stack.append(RenderState { .Expression = prefix_op_node.rhs });
945		switch (prefix_op_node.op) {
946		ast.NodeInfixOp.InfixOp.EqualEqual => {
947		try stack.append(RenderState { .Text = " == "});
948		},
949		ast.NodeInfixOp.InfixOp.BangEqual => {
950		try stack.append(RenderState { .Text = " != "});
951		},
952		else => unreachable,
953		}
954		try stack.append(RenderState { .Expression = prefix_op_node.lhs });
955		},
956		ast.Node.Id.PrefixOp => {
957		const prefix_op_node = @fieldParentPtr(ast.NodePrefixOp, "base", base);
958		try stack.append(RenderState { .Expression = prefix_op_node.rhs });
959		switch (prefix_op_node.op) {
960		ast.NodePrefixOp.PrefixOp.Return => {
961		try stream.write("return ");
962		},
963		ast.NodePrefixOp.PrefixOp.AddrOf => \|addr_of_info\| {
964		try stream.write("&");
965		if (addr_of_info.volatile_token != null) {
966		try stack.append(RenderState { .Text = "volatile "});
967		}
968		if (addr_of_info.const_token != null) {
969		try stack.append(RenderState { .Text = "const "});
970		}
971		if (addr_of_info.align_expr) \|align_expr\| {
972		try stream.print("align(");
973		try stack.append(RenderState { .Text = ") "});
974		try stack.append(RenderState { .Expression = align_expr});
975		}
976		},
977		else => unreachable,
978		}
979		},
980		ast.Node.Id.IntegerLiteral => {
981		const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
982		try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));
983		},
984		ast.Node.Id.FloatLiteral => {
985		const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base);
986		try stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token));
987		},
988		else => unreachable,
989		},
990		RenderState.FnProtoRParen => \|fn_proto\| {
991		try stream.print(")");
992		if (fn_proto.align_expr != null) {
993		@panic("TODO");
994		}
995		try stream.print(" ");
996		if (fn_proto.body_node) \|body_node\| {
997		try stack.append(RenderState { .Expression = body_node});
998		try stack.append(RenderState { .Text = " "});
999		}
1000		try stack.append(RenderState { .Expression = fn_proto.return_type});
1001		},
1002		RenderState.Statement => \|base\| {
1003		switch (base.id) {
1004		ast.Node.Id.VarDecl => {
1005		const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
1006		try stack.append(RenderState { .VarDecl = var_decl});
1007		},
1008		else => {
1009		try stack.append(RenderState { .Text = ";"});
1010		try stack.append(RenderState { .Expression = base});
1011		},
1012		}
1013		},
1014		RenderState.Indent => \|new_indent\| indent = new_indent,
1015		RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent),
1016		}
1017		}
1018		}
1019
1020		fn initUtilityArrayList(self: &Parser, comptime T: type) ArrayList(T) {
1021		const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
1022		self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
1023		const typed_slice = ([]T)(self.utility_bytes);
1024		return ArrayList(T) {
1025		.allocator = self.allocator,
1026		.items = typed_slice,
1027		.len = 0,
1028		};
1029		}
1030
1031		fn deinitUtilityArrayList(self: &Parser, list: var) void {
1032		self.utility_bytes = ([]align(utility_bytes_align) u8)(list.items);
1033		}
1034
1035		};
1036
1037		var fixed_buffer_mem: [100 * 1024]u8 = undefined;
1038
1039		fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
1040		var padded_source: [0x100]u8 = undefined;
1041		std.mem.copy(u8, padded_source[0..source.len], source);
1042		padded_source[source.len + 0] = '\n';
1043		padded_source[source.len + 1] = '\n';
1044		padded_source[source.len + 2] = '\n';
1045
1046		var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
1047		var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
1048		defer parser.deinit();
1049
1050		const tree = try parser.parse();
1051		defer tree.deinit();
1052
1053		var buffer = try std.Buffer.initSize(allocator, 0);
1054		var buffer_out_stream = io.BufferOutStream.init(&buffer);
1055		try parser.renderSource(&buffer_out_stream.stream, tree.root_node);
1056		return buffer.toOwnedSlice();
1057		}
1058
1059		// TODO test for memory leaks
1060		// TODO test for valid frees
1061		fn testCanonical(source: []const u8) !void {
1062		const needed_alloc_count = x: {
1063		// Try it once with unlimited memory, make sure it works
1064		var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
1065		var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize));
1066		const result_source = try testParse(source, &failing_allocator.allocator);
1067		if (!mem.eql(u8, result_source, source)) {
1068		warn("\n====== expected this output: =========\n");
1069		warn("{}", source);
1070		warn("\n======== instead found this: =========\n");
1071		warn("{}", result_source);
1072		warn("\n======================================\n");
1073		return error.TestFailed;
1074		}
1075		failing_allocator.allocator.free(result_source);
1076		break :x failing_allocator.index;
1077		};
1078
1079		var fail_index: usize = 0;
1080		while (fail_index < needed_alloc_count) : (fail_index += 1) {
1081		var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
1082		var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index);
1083		if (testParse(source, &failing_allocator.allocator)) \|_\| {
1084		return error.NondeterministicMemoryUsage;
1085		} else \|err\| {
1086		assert(err == error.OutOfMemory);
1087		// TODO make this pass
1088		//if (failing_allocator.allocated_bytes != failing_allocator.freed_bytes) {
1089		// warn("\nfail_index: {}/{}\nallocated bytes: {}\nfreed bytes: {}\nallocations: {}\ndeallocations: {}\n",
1090		// fail_index, needed_alloc_count,
1091		// failing_allocator.allocated_bytes, failing_allocator.freed_bytes,
1092		// failing_allocator.index, failing_allocator.deallocations);
1093		// return error.MemoryLeakDetected;
1094		//}
1095		}
1096		}
1097		}
1098
1099		test "zig fmt" {
1100		try testCanonical(
1101		\\extern fn puts(s: &const u8) c_int;
1102		\\
1103		);
1104
1105		try testCanonical(
1106		\\const a = b;
1107		\\pub const a = b;
1108		\\var a = b;
1109		\\pub var a = b;
1110		\\const a: i32 = b;
1111		\\pub const a: i32 = b;
1112		\\var a: i32 = b;
1113		\\pub var a: i32 = b;
1114		\\
1115		);
1116
1117		try testCanonical(
1118		\\extern var foo: c_int;
1119		\\
1120		);
1121
1122		try testCanonical(
1123		\\var foo: c_int align(1);
1124		\\
1125		);
1126
1127		try testCanonical(
1128		\\fn main(argc: c_int, argv: &&u8) c_int {
1129		\\ const a = b;
1130		\\}
1131		\\
1132		);
1133
1134		try testCanonical(
1135		\\fn foo(argc: c_int, argv: &&u8) c_int {
1136		\\ return 0;
1137		\\}
1138		\\
1139		);
1140
1141		try testCanonical(
1142		\\extern fn f1(s: &align(&u8) u8) c_int;
1143		\\
1144		);
1145
1146		try testCanonical(
1147		\\extern fn f1(s: &&align(1) &const &volatile u8) c_int;
1148		\\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int;
1149		\\extern fn f3(s: &align(1) const volatile u8) c_int;
1150		\\
1151		);
1152
1153		try testCanonical(
1154		\\fn f1(a: bool, b: bool) bool {
1155		\\ a != b;
1156		\\ return a == b;
1157		\\}
1158		\\
1159		);
1160		}

src-self-hosted/tokenizer.zig deleted-659

...	...	@@ -1,659 +0,0 @@
1		const std = @import("std");
2		const mem = std.mem;
3
4		pub const Token = struct {
5		id: Id,
6		start: usize,
7		end: usize,
8
9		const KeywordId = struct {
10		bytes: []const u8,
11		id: Id,
12		};
13
14		const keywords = []KeywordId {
15		KeywordId{.bytes="align", .id = Id.Keyword_align},
16		KeywordId{.bytes="and", .id = Id.Keyword_and},
17		KeywordId{.bytes="asm", .id = Id.Keyword_asm},
18		KeywordId{.bytes="break", .id = Id.Keyword_break},
19		KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
20		KeywordId{.bytes="const", .id = Id.Keyword_const},
21		KeywordId{.bytes="continue", .id = Id.Keyword_continue},
22		KeywordId{.bytes="defer", .id = Id.Keyword_defer},
23		KeywordId{.bytes="else", .id = Id.Keyword_else},
24		KeywordId{.bytes="enum", .id = Id.Keyword_enum},
25		KeywordId{.bytes="error", .id = Id.Keyword_error},
26		KeywordId{.bytes="export", .id = Id.Keyword_export},
27		KeywordId{.bytes="extern", .id = Id.Keyword_extern},
28		KeywordId{.bytes="false", .id = Id.Keyword_false},
29		KeywordId{.bytes="fn", .id = Id.Keyword_fn},
30		KeywordId{.bytes="for", .id = Id.Keyword_for},
31		KeywordId{.bytes="goto", .id = Id.Keyword_goto},
32		KeywordId{.bytes="if", .id = Id.Keyword_if},
33		KeywordId{.bytes="inline", .id = Id.Keyword_inline},
34		KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
35		KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
36		KeywordId{.bytes="null", .id = Id.Keyword_null},
37		KeywordId{.bytes="or", .id = Id.Keyword_or},
38		KeywordId{.bytes="packed", .id = Id.Keyword_packed},
39		KeywordId{.bytes="pub", .id = Id.Keyword_pub},
40		KeywordId{.bytes="return", .id = Id.Keyword_return},
41		KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
42		KeywordId{.bytes="struct", .id = Id.Keyword_struct},
43		KeywordId{.bytes="switch", .id = Id.Keyword_switch},
44		KeywordId{.bytes="test", .id = Id.Keyword_test},
45		KeywordId{.bytes="this", .id = Id.Keyword_this},
46		KeywordId{.bytes="true", .id = Id.Keyword_true},
47		KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
48		KeywordId{.bytes="union", .id = Id.Keyword_union},
49		KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
50		KeywordId{.bytes="use", .id = Id.Keyword_use},
51		KeywordId{.bytes="var", .id = Id.Keyword_var},
52		KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
53		KeywordId{.bytes="while", .id = Id.Keyword_while},
54		};
55
56		fn getKeyword(bytes: []const u8) ?Id {
57		for (keywords) \|kw\| {
58		if (mem.eql(u8, kw.bytes, bytes)) {
59		return kw.id;
60		}
61		}
62		return null;
63		}
64
65		const StrLitKind = enum {Normal, C};
66
67		pub const Id = union(enum) {
68		Invalid,
69		Identifier,
70		StringLiteral: StrLitKind,
71		Eof,
72		Builtin,
73		Bang,
74		Equal,
75		EqualEqual,
76		BangEqual,
77		LParen,
78		RParen,
79		Semicolon,
80		Percent,
81		LBrace,
82		RBrace,
83		Period,
84		Ellipsis2,
85		Ellipsis3,
86		Minus,
87		Arrow,
88		Colon,
89		Slash,
90		Comma,
91		Ampersand,
92		AmpersandEqual,
93		IntegerLiteral,
94		FloatLiteral,
95		Keyword_align,
96		Keyword_and,
97		Keyword_asm,
98		Keyword_break,
99		Keyword_comptime,
100		Keyword_const,
101		Keyword_continue,
102		Keyword_defer,
103		Keyword_else,
104		Keyword_enum,
105		Keyword_error,
106		Keyword_export,
107		Keyword_extern,
108		Keyword_false,
109		Keyword_fn,
110		Keyword_for,
111		Keyword_goto,
112		Keyword_if,
113		Keyword_inline,
114		Keyword_nakedcc,
115		Keyword_noalias,
116		Keyword_null,
117		Keyword_or,
118		Keyword_packed,
119		Keyword_pub,
120		Keyword_return,
121		Keyword_stdcallcc,
122		Keyword_struct,
123		Keyword_switch,
124		Keyword_test,
125		Keyword_this,
126		Keyword_true,
127		Keyword_undefined,
128		Keyword_union,
129		Keyword_unreachable,
130		Keyword_use,
131		Keyword_var,
132		Keyword_volatile,
133		Keyword_while,
134		};
135		};
136
137		pub const Tokenizer = struct {
138		buffer: []const u8,
139		index: usize,
140		pending_invalid_token: ?Token,
141
142		pub const Location = struct {
143		line: usize,
144		column: usize,
145		line_start: usize,
146		line_end: usize,
147		};
148
149		pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {
150		var loc = Location {
151		.line = 0,
152		.column = 0,
153		.line_start = 0,
154		.line_end = 0,
155		};
156		for (self.buffer) \|c, i\| {
157		if (i == token.start) {
158		loc.line_end = i;
159		while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
160		return loc;
161		}
162		if (c == '\n') {
163		loc.line += 1;
164		loc.column = 0;
165		loc.line_start = i + 1;
166		} else {
167		loc.column += 1;
168		}
169		}
170		return loc;
171		}
172
173		/// For debugging purposes
174		pub fn dump(self: &Tokenizer, token: &const Token) void {
175		std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
176		}
177
178		/// buffer must end with "\n\n\n". This is so that attempting to decode
179		/// a the 3 trailing bytes of a 4-byte utf8 sequence is never a buffer overflow.
180		pub fn init(buffer: []const u8) Tokenizer {
181		std.debug.assert(buffer[buffer.len - 1] == '\n');
182		std.debug.assert(buffer[buffer.len - 2] == '\n');
183		std.debug.assert(buffer[buffer.len - 3] == '\n');
184		return Tokenizer {
185		.buffer = buffer,
186		.index = 0,
187		.pending_invalid_token = null,
188		};
189		}
190
191		const State = enum {
192		Start,
193		Identifier,
194		Builtin,
195		C,
196		StringLiteral,
197		StringLiteralBackslash,
198		Equal,
199		Bang,
200		Minus,
201		Slash,
202		LineComment,
203		Zero,
204		IntegerLiteral,
205		IntegerLiteralWithRadix,
206		NumberDot,
207		FloatFraction,
208		FloatExponentUnsigned,
209		FloatExponentNumber,
210		Ampersand,
211		Period,
212		Period2,
213		};
214
215		pub fn next(self: &Tokenizer) Token {
216		if (self.pending_invalid_token) \|token\| {
217		self.pending_invalid_token = null;
218		return token;
219		}
220		var state = State.Start;
221		var result = Token {
222		.id = Token.Id.Eof,
223		.start = self.index,
224		.end = undefined,
225		};
226		while (self.index < self.buffer.len) : (self.index += 1) {
227		const c = self.buffer[self.index];
228		switch (state) {
229		State.Start => switch (c) {
230		' ', '\n' => {
231		result.start = self.index + 1;
232		},
233		'c' => {
234		state = State.C;
235		result.id = Token.Id.Identifier;
236		},
237		'"' => {
238		state = State.StringLiteral;
239		result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
240		},
241		'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
242		state = State.Identifier;
243		result.id = Token.Id.Identifier;
244		},
245		'@' => {
246		state = State.Builtin;
247		result.id = Token.Id.Builtin;
248		},
249		'=' => {
250		state = State.Equal;
251		},
252		'!' => {
253		state = State.Bang;
254		},
255		'(' => {
256		result.id = Token.Id.LParen;
257		self.index += 1;
258		break;
259		},
260		')' => {
261		result.id = Token.Id.RParen;
262		self.index += 1;
263		break;
264		},
265		';' => {
266		result.id = Token.Id.Semicolon;
267		self.index += 1;
268		break;
269		},
270		',' => {
271		result.id = Token.Id.Comma;
272		self.index += 1;
273		break;
274		},
275		':' => {
276		result.id = Token.Id.Colon;
277		self.index += 1;
278		break;
279		},
280		'%' => {
281		result.id = Token.Id.Percent;
282		self.index += 1;
283		break;
284		},
285		'{' => {
286		result.id = Token.Id.LBrace;
287		self.index += 1;
288		break;
289		},
290		'}' => {
291		result.id = Token.Id.RBrace;
292		self.index += 1;
293		break;
294		},
295		'.' => {
296		state = State.Period;
297		},
298		'-' => {
299		state = State.Minus;
300		},
301		'/' => {
302		state = State.Slash;
303		},
304		'&' => {
305		state = State.Ampersand;
306		},
307		'0' => {
308		state = State.Zero;
309		result.id = Token.Id.IntegerLiteral;
310		},
311		'1'...'9' => {
312		state = State.IntegerLiteral;
313		result.id = Token.Id.IntegerLiteral;
314		},
315		else => {
316		result.id = Token.Id.Invalid;
317		self.index += 1;
318		break;
319		},
320		},
321		State.Ampersand => switch (c) {
322		'=' => {
323		result.id = Token.Id.AmpersandEqual;
324		self.index += 1;
325		break;
326		},
327		else => {
328		result.id = Token.Id.Ampersand;
329		break;
330		},
331		},
332		State.Identifier => switch (c) {
333		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
334		else => {
335		if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
336		result.id = id;
337		}
338		break;
339		},
340		},
341		State.Builtin => switch (c) {
342		'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
343		else => break,
344		},
345		State.C => switch (c) {
346		'\\' => @panic("TODO"),
347		'"' => {
348		state = State.StringLiteral;
349		result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
350		},
351		'a'...'z', 'A'...'Z', '_', '0'...'9' => {
352		state = State.Identifier;
353		},
354		else => break,
355		},
356		State.StringLiteral => switch (c) {
357		'\\' => {
358		state = State.StringLiteralBackslash;
359		},
360		'"' => {
361		self.index += 1;
362		break;
363		},
364		'\n' => break, // Look for this error later.
365		else => self.checkLiteralCharacter(),
366		},
367
368		State.StringLiteralBackslash => switch (c) {
369		'\n' => break, // Look for this error later.
370		else => {
371		state = State.StringLiteral;
372		},
373		},
374
375		State.Bang => switch (c) {
376		'=' => {
377		result.id = Token.Id.BangEqual;
378		self.index += 1;
379		break;
380		},
381		else => {
382		result.id = Token.Id.Bang;
383		break;
384		},
385		},
386
387		State.Equal => switch (c) {
388		'=' => {
389		result.id = Token.Id.EqualEqual;
390		self.index += 1;
391		break;
392		},
393		else => {
394		result.id = Token.Id.Equal;
395		break;
396		},
397		},
398
399		State.Minus => switch (c) {
400		'>' => {
401		result.id = Token.Id.Arrow;
402		self.index += 1;
403		break;
404		},
405		else => {
406		result.id = Token.Id.Minus;
407		break;
408		},
409		},
410
411		State.Period => switch (c) {
412		'.' => {
413		state = State.Period2;
414		},
415		else => {
416		result.id = Token.Id.Period;
417		break;
418		},
419		},
420
421		State.Period2 => switch (c) {
422		'.' => {
423		result.id = Token.Id.Ellipsis3;
424		self.index += 1;
425		break;
426		},
427		else => {
428		result.id = Token.Id.Ellipsis2;
429		break;
430		},
431		},
432
433		State.Slash => switch (c) {
434		'/' => {
435		result.id = undefined;
436		state = State.LineComment;
437		},
438		else => {
439		result.id = Token.Id.Slash;
440		break;
441		},
442		},
443		State.LineComment => switch (c) {
444		'\n' => {
445		state = State.Start;
446		result = Token {
447		.id = Token.Id.Eof,
448		.start = self.index + 1,
449		.end = undefined,
450		};
451		},
452		else => self.checkLiteralCharacter(),
453		},
454		State.Zero => switch (c) {
455		'b', 'o', 'x' => {
456		state = State.IntegerLiteralWithRadix;
457		},
458		else => {
459		// reinterpret as a normal number
460		self.index -= 1;
461		state = State.IntegerLiteral;
462		},
463		},
464		State.IntegerLiteral => switch (c) {
465		'.' => {
466		state = State.NumberDot;
467		},
468		'p', 'P', 'e', 'E' => {
469		state = State.FloatExponentUnsigned;
470		},
471		'0'...'9' => {},
472		else => break,
473		},
474		State.IntegerLiteralWithRadix => switch (c) {
475		'.' => {
476		state = State.NumberDot;
477		},
478		'p', 'P' => {
479		state = State.FloatExponentUnsigned;
480		},
481		'0'...'9', 'a'...'f', 'A'...'F' => {},
482		else => break,
483		},
484		State.NumberDot => switch (c) {
485		'.' => {
486		self.index -= 1;
487		state = State.Start;
488		break;
489		},
490		else => {
491		self.index -= 1;
492		result.id = Token.Id.FloatLiteral;
493		state = State.FloatFraction;
494		},
495		},
496		State.FloatFraction => switch (c) {
497		'p', 'P' => {
498		state = State.FloatExponentUnsigned;
499		},
500		'0'...'9', 'a'...'f', 'A'...'F' => {},
501		else => break,
502		},
503		State.FloatExponentUnsigned => switch (c) {
504		'+', '-' => {
505		state = State.FloatExponentNumber;
506		},
507		else => {
508		// reinterpret as a normal exponent number
509		self.index -= 1;
510		state = State.FloatExponentNumber;
511		}
512		},
513		State.FloatExponentNumber => switch (c) {
514		'0'...'9', 'a'...'f', 'A'...'F' => {},
515		else => break,
516		},
517		}
518		}
519		result.end = self.index;
520
521		if (result.id == Token.Id.Eof) {
522		if (self.pending_invalid_token) \|token\| {
523		self.pending_invalid_token = null;
524		return token;
525		}
526		}
527
528		return result;
529		}
530
531		pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) []const u8 {
532		return self.buffer[token.start..token.end];
533		}
534
535		fn checkLiteralCharacter(self: &Tokenizer) void {
536		if (self.pending_invalid_token != null) return;
537		const invalid_length = self.getInvalidCharacterLength();
538		if (invalid_length == 0) return;
539		self.pending_invalid_token = Token {
540		.id = Token.Id.Invalid,
541		.start = self.index,
542		.end = self.index + invalid_length,
543		};
544		}
545
546		fn getInvalidCharacterLength(self: &Tokenizer) u3 {
547		const c0 = self.buffer[self.index];
548		if (c0 < 0x80) {
549		if (c0 < 0x20 or c0 == 0x7f) {
550		// ascii control codes are never allowed
551		// (note that \n was checked before we got here)
552		return 1;
553		}
554		// looks fine to me.
555		return 0;
556		} else {
557		// check utf8-encoded character.
558		const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
559		// the last 3 bytes in the buffer are guaranteed to be '\n',
560		// which means we don't need to do any bounds checking here.
561		const bytes = self.buffer[self.index..self.index + length];
562		switch (length) {
563		2 => {
564		const value = std.unicode.utf8Decode2(bytes) catch return length;
565		if (value == 0x85) return length; // U+0085 (NEL)
566		},
567		3 => {
568		const value = std.unicode.utf8Decode3(bytes) catch return length;
569		if (value == 0x2028) return length; // U+2028 (LS)
570		if (value == 0x2029) return length; // U+2029 (PS)
571		},
572		4 => {
573		_ = std.unicode.utf8Decode4(bytes) catch return length;
574		},
575		else => unreachable,
576		}
577		self.index += length - 1;
578		return 0;
579		}
580		}
581		};
582
583
584
585		test "tokenizer" {
586		testTokenize("test", []Token.Id {
587		Token.Id.Keyword_test,
588		});
589		}
590
591		test "tokenizer - invalid token characters" {
592		testTokenize("#", []Token.Id{Token.Id.Invalid});
593		testTokenize("`", []Token.Id{Token.Id.Invalid});
594		}
595
596		test "tokenizer - invalid literal/comment characters" {
597		testTokenize("\"\x00\"", []Token.Id {
598		Token.Id { .StringLiteral = Token.StrLitKind.Normal },
599		Token.Id.Invalid,
600		});
601		testTokenize("//\x00", []Token.Id {
602		Token.Id.Invalid,
603		});
604		testTokenize("//\x1f", []Token.Id {
605		Token.Id.Invalid,
606		});
607		testTokenize("//\x7f", []Token.Id {
608		Token.Id.Invalid,
609		});
610		}
611
612		test "tokenizer - utf8" {
613		testTokenize("//\xc2\x80", []Token.Id{});
614		testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});
615		}
616
617		test "tokenizer - invalid utf8" {
618		testTokenize("//\x80", []Token.Id{Token.Id.Invalid});
619		testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});
620		testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});
621		testTokenize("//\xff", []Token.Id{Token.Id.Invalid});
622		testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});
623		testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});
624		testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});
625		testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});
626		}
627
628		test "tokenizer - illegal unicode codepoints" {
629		// unicode newline characters.U+0085, U+2028, U+2029
630		testTokenize("//\xc2\x84", []Token.Id{});
631		testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});
632		testTokenize("//\xc2\x86", []Token.Id{});
633		testTokenize("//\xe2\x80\xa7", []Token.Id{});
634		testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});
635		testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});
636		testTokenize("//\xe2\x80\xaa", []Token.Id{});
637		}
638
639		fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
640		// (test authors, just make this bigger if you need it)
641		var padded_source: [0x100]u8 = undefined;
642		std.mem.copy(u8, padded_source[0..source.len], source);
643		padded_source[source.len + 0] = '\n';
644		padded_source[source.len + 1] = '\n';
645		padded_source[source.len + 2] = '\n';
646
647		var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
648		for (expected_tokens) \|expected_token_id\| {
649		const token = tokenizer.next();
650		std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
651		switch (expected_token_id) {
652		Token.Id.StringLiteral => \|expected_kind\| {
653		std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => \|kind\| kind, else => unreachable });
654		},
655		else => {},
656		}
657		}
658		std.debug.assert(tokenizer.next().id == Token.Id.Eof);
659		}

std/index.zig+2

...	...	@@ -28,6 +28,7 @@ pub const os = @import("os/index.zig");
28	28	pub const rand = @import("rand.zig");
29	29	pub const sort = @import("sort.zig");
30	30	pub const unicode = @import("unicode.zig");
	31	pub const zig = @import("zig/index.zig");
31	32
32	33	test "std" {
33	34	// run tests from these
...	...	@@ -58,4 +59,5 @@ test "std" {
58	59	_ = @import("rand.zig");
59	60	_ = @import("sort.zig");
60	61	_ = @import("unicode.zig");
	62	_ = @import("zig/index.zig");
61	63	}

std/zig/ast.zig created+271

...	...	@@ -0,0 +1,271 @@
	1	const std = @import("../index.zig");
	2	const assert = std.debug.assert;
	3	const ArrayList = std.ArrayList;
	4	const Token = std.zig.Token;
	5	const mem = std.mem;
	6
	7	pub const Node = struct {
	8	id: Id,
	9
	10	pub const Id = enum {
	11	Root,
	12	VarDecl,
	13	Identifier,
	14	FnProto,
	15	ParamDecl,
	16	Block,
	17	InfixOp,
	18	PrefixOp,
	19	IntegerLiteral,
	20	FloatLiteral,
	21	};
	22
	23	pub fn iterate(base: &Node, index: usize) ?&Node {
	24	return switch (base.id) {
	25	Id.Root => @fieldParentPtr(NodeRoot, "base", base).iterate(index),
	26	Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).iterate(index),
	27	Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).iterate(index),
	28	Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).iterate(index),
	29	Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).iterate(index),
	30	Id.Block => @fieldParentPtr(NodeBlock, "base", base).iterate(index),
	31	Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).iterate(index),
	32	Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).iterate(index),
	33	Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).iterate(index),
	34	Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).iterate(index),
	35	};
	36	}
	37
	38	pub fn destroy(base: &Node, allocator: &mem.Allocator) void {
	39	return switch (base.id) {
	40	Id.Root => allocator.destroy(@fieldParentPtr(NodeRoot, "base", base)),
	41	Id.VarDecl => allocator.destroy(@fieldParentPtr(NodeVarDecl, "base", base)),
	42	Id.Identifier => allocator.destroy(@fieldParentPtr(NodeIdentifier, "base", base)),
	43	Id.FnProto => allocator.destroy(@fieldParentPtr(NodeFnProto, "base", base)),
	44	Id.ParamDecl => allocator.destroy(@fieldParentPtr(NodeParamDecl, "base", base)),
	45	Id.Block => allocator.destroy(@fieldParentPtr(NodeBlock, "base", base)),
	46	Id.InfixOp => allocator.destroy(@fieldParentPtr(NodeInfixOp, "base", base)),
	47	Id.PrefixOp => allocator.destroy(@fieldParentPtr(NodePrefixOp, "base", base)),
	48	Id.IntegerLiteral => allocator.destroy(@fieldParentPtr(NodeIntegerLiteral, "base", base)),
	49	Id.FloatLiteral => allocator.destroy(@fieldParentPtr(NodeFloatLiteral, "base", base)),
	50	};
	51	}
	52	};
	53
	54	pub const NodeRoot = struct {
	55	base: Node,
	56	decls: ArrayList(&Node),
	57
	58	pub fn iterate(self: &NodeRoot, index: usize) ?&Node {
	59	if (index < self.decls.len) {
	60	return self.decls.items[self.decls.len - index - 1];
	61	}
	62	return null;
	63	}
	64	};
	65
	66	pub const NodeVarDecl = struct {
	67	base: Node,
	68	visib_token: ?Token,
	69	name_token: Token,
	70	eq_token: Token,
	71	mut_token: Token,
	72	comptime_token: ?Token,
	73	extern_token: ?Token,
	74	lib_name: ?&Node,
	75	type_node: ?&Node,
	76	align_node: ?&Node,
	77	init_node: ?&Node,
	78
	79	pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node {
	80	var i = index;
	81
	82	if (self.type_node) \|type_node\| {
	83	if (i < 1) return type_node;
	84	i -= 1;
	85	}
	86
	87	if (self.align_node) \|align_node\| {
	88	if (i < 1) return align_node;
	89	i -= 1;
	90	}
	91
	92	if (self.init_node) \|init_node\| {
	93	if (i < 1) return init_node;
	94	i -= 1;
	95	}
	96
	97	return null;
	98	}
	99	};
	100
	101	pub const NodeIdentifier = struct {
	102	base: Node,
	103	name_token: Token,
	104
	105	pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node {
	106	return null;
	107	}
	108	};
	109
	110	pub const NodeFnProto = struct {
	111	base: Node,
	112	visib_token: ?Token,
	113	fn_token: Token,
	114	name_token: ?Token,
	115	params: ArrayList(&Node),
	116	return_type: &Node,
	117	var_args_token: ?Token,
	118	extern_token: ?Token,
	119	inline_token: ?Token,
	120	cc_token: ?Token,
	121	body_node: ?&Node,
	122	lib_name: ?&Node, // populated if this is an extern declaration
	123	align_expr: ?&Node, // populated if align(A) is present
	124
	125	pub fn iterate(self: &NodeFnProto, index: usize) ?&Node {
	126	var i = index;
	127
	128	if (self.body_node) \|body_node\| {
	129	if (i < 1) return body_node;
	130	i -= 1;
	131	}
	132
	133	if (i < 1) return self.return_type;
	134	i -= 1;
	135
	136	if (self.align_expr) \|align_expr\| {
	137	if (i < 1) return align_expr;
	138	i -= 1;
	139	}
	140
	141	if (i < self.params.len) return self.params.items[self.params.len - i - 1];
	142	i -= self.params.len;
	143
	144	if (self.lib_name) \|lib_name\| {
	145	if (i < 1) return lib_name;
	146	i -= 1;
	147	}
	148
	149	return null;
	150	}
	151	};
	152
	153	pub const NodeParamDecl = struct {
	154	base: Node,
	155	comptime_token: ?Token,
	156	noalias_token: ?Token,
	157	name_token: ?Token,
	158	type_node: &Node,
	159	var_args_token: ?Token,
	160
	161	pub fn iterate(self: &NodeParamDecl, index: usize) ?&Node {
	162	var i = index;
	163
	164	if (i < 1) return self.type_node;
	165	i -= 1;
	166
	167	return null;
	168	}
	169	};
	170
	171	pub const NodeBlock = struct {
	172	base: Node,
	173	begin_token: Token,
	174	end_token: Token,
	175	statements: ArrayList(&Node),
	176
	177	pub fn iterate(self: &NodeBlock, index: usize) ?&Node {
	178	var i = index;
	179
	180	if (i < self.statements.len) return self.statements.items[i];
	181	i -= self.statements.len;
	182
	183	return null;
	184	}
	185	};
	186
	187	pub const NodeInfixOp = struct {
	188	base: Node,
	189	op_token: Token,
	190	lhs: &Node,
	191	op: InfixOp,
	192	rhs: &Node,
	193
	194	const InfixOp = enum {
	195	EqualEqual,
	196	BangEqual,
	197	};
	198
	199	pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node {
	200	var i = index;
	201
	202	if (i < 1) return self.lhs;
	203	i -= 1;
	204
	205	switch (self.op) {
	206	InfixOp.EqualEqual => {},
	207	InfixOp.BangEqual => {},
	208	}
	209
	210	if (i < 1) return self.rhs;
	211	i -= 1;
	212
	213	return null;
	214	}
	215	};
	216
	217	pub const NodePrefixOp = struct {
	218	base: Node,
	219	op_token: Token,
	220	op: PrefixOp,
	221	rhs: &Node,
	222
	223	const PrefixOp = union(enum) {
	224	Return,
	225	AddrOf: AddrOfInfo,
	226	};
	227	const AddrOfInfo = struct {
	228	align_expr: ?&Node,
	229	bit_offset_start_token: ?Token,
	230	bit_offset_end_token: ?Token,
	231	const_token: ?Token,
	232	volatile_token: ?Token,
	233	};
	234
	235	pub fn iterate(self: &NodePrefixOp, index: usize) ?&Node {
	236	var i = index;
	237
	238	switch (self.op) {
	239	PrefixOp.Return => {},
	240	PrefixOp.AddrOf => \|addr_of_info\| {
	241	if (addr_of_info.align_expr) \|align_expr\| {
	242	if (i < 1) return align_expr;
	243	i -= 1;
	244	}
	245	},
	246	}
	247
	248	if (i < 1) return self.rhs;
	249	i -= 1;
	250
	251	return null;
	252	}
	253	};
	254
	255	pub const NodeIntegerLiteral = struct {
	256	base: Node,
	257	token: Token,
	258
	259	pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node {
	260	return null;
	261	}
	262	};
	263
	264	pub const NodeFloatLiteral = struct {
	265	base: Node,
	266	token: Token,
	267
	268	pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node {
	269	return null;
	270	}
	271	};

std/zig/index.zig created+11

...	...	@@ -0,0 +1,11 @@
	1	const tokenizer = @import("tokenizer.zig");
	2	pub const Token = tokenizer.Token;
	3	pub const Tokenizer = tokenizer.Tokenizer;
	4	pub const Parser = @import("parser.zig").Parser;
	5	pub const ast = @import("ast.zig");
	6
	7	test "std.zig tests" {
	8	_ = @import("tokenizer.zig");
	9	_ = @import("parser.zig");
	10	_ = @import("ast.zig");
	11	}

std/zig/parser.zig created+1160

...	...	@@ -0,0 +1,1160 @@
	1	const std = @import("../index.zig");
	2	const assert = std.debug.assert;
	3	const ArrayList = std.ArrayList;
	4	const mem = std.mem;
	5	const ast = std.zig.ast;
	6	const Tokenizer = std.zig.Tokenizer;
	7	const Token = std.zig.Token;
	8	const builtin = @import("builtin");
	9	const io = std.io;
	10
	11	// TODO when we make parse errors into error types instead of printing directly,
	12	// get rid of this
	13	const warn = std.debug.warn;
	14
	15	pub const Parser = struct {
	16	allocator: &mem.Allocator,
	17	tokenizer: &Tokenizer,
	18	put_back_tokens: [2]Token,
	19	put_back_count: usize,
	20	source_file_name: []const u8,
	21
	22	pub const Tree = struct {
	23	root_node: &ast.NodeRoot,
	24
	25	pub fn deinit(self: &const Tree) void {
	26	// TODO free the whole arena
	27	}
	28	};
	29
	30	// This memory contents are used only during a function call. It's used to repurpose memory;
	31	// we reuse the same bytes for the stack data structure used by parsing, tree rendering, and
	32	// source rendering.
	33	const utility_bytes_align = @alignOf( union { a: RenderAstFrame, b: State, c: RenderState } );
	34	utility_bytes: []align(utility_bytes_align) u8,
	35
	36	/// `allocator` should be an arena allocator. Parser never calls free on anything. After you're
	37	/// done with a Parser, free the arena. After the arena is freed, no member functions of Parser
	38	/// may be called.
	39	pub fn init(tokenizer: &Tokenizer, allocator: &mem.Allocator, source_file_name: []const u8) Parser {
	40	return Parser {
	41	.allocator = allocator,
	42	.tokenizer = tokenizer,
	43	.put_back_tokens = undefined,
	44	.put_back_count = 0,
	45	.source_file_name = source_file_name,
	46	.utility_bytes = []align(utility_bytes_align) u8{},
	47	};
	48	}
	49
	50	pub fn deinit(self: &Parser) void {
	51	self.allocator.free(self.utility_bytes);
	52	}
	53
	54	const TopLevelDeclCtx = struct {
	55	visib_token: ?Token,
	56	extern_token: ?Token,
	57	};
	58
	59	const DestPtr = union(enum) {
	60	Field: &&ast.Node,
	61	NullableField: &?&ast.Node,
	62	List: &ArrayList(&ast.Node),
	63
	64	pub fn store(self: &const DestPtr, value: &ast.Node) !void {
	65	switch (*self) {
	66	DestPtr.Field => \|ptr\| *ptr = value,
	67	DestPtr.NullableField => \|ptr\| *ptr = value,
	68	DestPtr.List => \|list\| try list.append(value),
	69	}
	70	}
	71	};
	72
	73	const State = union(enum) {
	74	TopLevel,
	75	TopLevelExtern: ?Token,
	76	TopLevelDecl: TopLevelDeclCtx,
	77	Expression: DestPtr,
	78	ExpectOperand,
	79	Operand: &ast.Node,
	80	AfterOperand,
	81	InfixOp: &ast.NodeInfixOp,
	82	PrefixOp: &ast.NodePrefixOp,
	83	AddrOfModifiers: &ast.NodePrefixOp.AddrOfInfo,
	84	TypeExpr: DestPtr,
	85	VarDecl: &ast.NodeVarDecl,
	86	VarDeclAlign: &ast.NodeVarDecl,
	87	VarDeclEq: &ast.NodeVarDecl,
	88	ExpectToken: @TagType(Token.Id),
	89	FnProto: &ast.NodeFnProto,
	90	FnProtoAlign: &ast.NodeFnProto,
	91	ParamDecl: &ast.NodeFnProto,
	92	ParamDeclComma,
	93	FnDef: &ast.NodeFnProto,
	94	Block: &ast.NodeBlock,
	95	Statement: &ast.NodeBlock,
	96	};
	97
	98	/// Returns an AST tree, allocated with the parser's allocator.
	99	/// Result should be freed with `freeAst` when done.
	100	pub fn parse(self: &Parser) !Tree {
	101	var stack = self.initUtilityArrayList(State);
	102	defer self.deinitUtilityArrayList(stack);
	103
	104	const root_node = try self.createRoot();
	105	// TODO errdefer arena free root node
	106
	107	try stack.append(State.TopLevel);
	108
	109	while (true) {
	110	//{
	111	// const token = self.getNextToken();
	112	// warn("{} ", @tagName(token.id));
	113	// self.putBackToken(token);
	114	// var i: usize = stack.len;
	115	// while (i != 0) {
	116	// i -= 1;
	117	// warn("{} ", @tagName(stack.items[i]));
	118	// }
	119	// warn("\n");
	120	//}
	121
	122	// This gives us 1 free append that can't fail
	123	const state = stack.pop();
	124
	125	switch (state) {
	126	State.TopLevel => {
	127	const token = self.getNextToken();
	128	switch (token.id) {
	129	Token.Id.Keyword_pub, Token.Id.Keyword_export => {
	130	stack.append(State { .TopLevelExtern = token }) catch unreachable;
	131	continue;
	132	},
	133	Token.Id.Eof => return Tree {.root_node = root_node},
	134	else => {
	135	self.putBackToken(token);
	136	// TODO shouldn't need this cast
	137	stack.append(State { .TopLevelExtern = null }) catch unreachable;
	138	continue;
	139	},
	140	}
	141	},
	142	State.TopLevelExtern => \|visib_token\| {
	143	const token = self.getNextToken();
	144	if (token.id == Token.Id.Keyword_extern) {
	145	stack.append(State {
	146	.TopLevelDecl = TopLevelDeclCtx {
	147	.visib_token = visib_token,
	148	.extern_token = token,
	149	},
	150	}) catch unreachable;
	151	continue;
	152	}
	153	self.putBackToken(token);
	154	stack.append(State {
	155	.TopLevelDecl = TopLevelDeclCtx {
	156	.visib_token = visib_token,
	157	.extern_token = null,
	158	},
	159	}) catch unreachable;
	160	continue;
	161	},
	162	State.TopLevelDecl => \|ctx\| {
	163	const token = self.getNextToken();
	164	switch (token.id) {
	165	Token.Id.Keyword_var, Token.Id.Keyword_const => {
	166	stack.append(State.TopLevel) catch unreachable;
	167	// TODO shouldn't need these casts
	168	const var_decl_node = try self.createAttachVarDecl(&root_node.decls, ctx.visib_token,
	169	token, (?Token)(null), ctx.extern_token);
	170	try stack.append(State { .VarDecl = var_decl_node });
	171	continue;
	172	},
	173	Token.Id.Keyword_fn => {
	174	stack.append(State.TopLevel) catch unreachable;
	175	// TODO shouldn't need these casts
	176	const fn_proto = try self.createAttachFnProto(&root_node.decls, token,
	177	ctx.extern_token, (?Token)(null), (?Token)(null), (?Token)(null));
	178	try stack.append(State { .FnDef = fn_proto });
	179	try stack.append(State { .FnProto = fn_proto });
	180	continue;
	181	},
	182	Token.Id.StringLiteral => {
	183	@panic("TODO extern with string literal");
	184	},
	185	Token.Id.Keyword_nakedcc, Token.Id.Keyword_stdcallcc => {
	186	stack.append(State.TopLevel) catch unreachable;
	187	const fn_token = try self.eatToken(Token.Id.Keyword_fn);
	188	// TODO shouldn't need this cast
	189	const fn_proto = try self.createAttachFnProto(&root_node.decls, fn_token,
	190	ctx.extern_token, (?Token)(token), (?Token)(null), (?Token)(null));
	191	try stack.append(State { .FnDef = fn_proto });
	192	try stack.append(State { .FnProto = fn_proto });
	193	continue;
	194	},
	195	else => return self.parseError(token, "expected variable declaration or function, found {}", @tagName(token.id)),
	196	}
	197	},
	198	State.VarDecl => \|var_decl\| {
	199	var_decl.name_token = try self.eatToken(Token.Id.Identifier);
	200	stack.append(State { .VarDeclAlign = var_decl }) catch unreachable;
	201
	202	const next_token = self.getNextToken();
	203	if (next_token.id == Token.Id.Colon) {
	204	try stack.append(State { .TypeExpr = DestPtr {.NullableField = &var_decl.type_node} });
	205	continue;
	206	}
	207
	208	self.putBackToken(next_token);
	209	continue;
	210	},
	211	State.VarDeclAlign => \|var_decl\| {
	212	stack.append(State { .VarDeclEq = var_decl }) catch unreachable;
	213
	214	const next_token = self.getNextToken();
	215	if (next_token.id == Token.Id.Keyword_align) {
	216	_ = try self.eatToken(Token.Id.LParen);
	217	try stack.append(State { .ExpectToken = Token.Id.RParen });
	218	try stack.append(State { .Expression = DestPtr{.NullableField = &var_decl.align_node} });
	219	continue;
	220	}
	221
	222	self.putBackToken(next_token);
	223	continue;
	224	},
	225	State.VarDeclEq => \|var_decl\| {
	226	const token = self.getNextToken();
	227	if (token.id == Token.Id.Equal) {
	228	var_decl.eq_token = token;
	229	stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
	230	try stack.append(State {
	231	.Expression = DestPtr {.NullableField = &var_decl.init_node},
	232	});
	233	continue;
	234	}
	235	if (token.id == Token.Id.Semicolon) {
	236	continue;
	237	}
	238	return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id));
	239	},
	240	State.ExpectToken => \|token_id\| {
	241	_ = try self.eatToken(token_id);
	242	continue;
	243	},
	244
	245	State.Expression => \|dest_ptr\| {
	246	// save the dest_ptr for later
	247	stack.append(state) catch unreachable;
	248	try stack.append(State.ExpectOperand);
	249	continue;
	250	},
	251	State.ExpectOperand => {
	252	// we'll either get an operand (like 1 or x),
	253	// or a prefix operator (like ~ or return).
	254	const token = self.getNextToken();
	255	switch (token.id) {
	256	Token.Id.Keyword_return => {
	257	try stack.append(State { .PrefixOp = try self.createPrefixOp(token,
	258	ast.NodePrefixOp.PrefixOp.Return) });
	259	try stack.append(State.ExpectOperand);
	260	continue;
	261	},
	262	Token.Id.Ampersand => {
	263	const prefix_op = try self.createPrefixOp(token, ast.NodePrefixOp.PrefixOp{
	264	.AddrOf = ast.NodePrefixOp.AddrOfInfo {
	265	.align_expr = null,
	266	.bit_offset_start_token = null,
	267	.bit_offset_end_token = null,
	268	.const_token = null,
	269	.volatile_token = null,
	270	}
	271	});
	272	try stack.append(State { .PrefixOp = prefix_op });
	273	try stack.append(State.ExpectOperand);
	274	try stack.append(State { .AddrOfModifiers = &prefix_op.op.AddrOf });
	275	continue;
	276	},
	277	Token.Id.Identifier => {
	278	try stack.append(State {
	279	.Operand = &(try self.createIdentifier(token)).base
	280	});
	281	try stack.append(State.AfterOperand);
	282	continue;
	283	},
	284	Token.Id.IntegerLiteral => {
	285	try stack.append(State {
	286	.Operand = &(try self.createIntegerLiteral(token)).base
	287	});
	288	try stack.append(State.AfterOperand);
	289	continue;
	290	},
	291	Token.Id.FloatLiteral => {
	292	try stack.append(State {
	293	.Operand = &(try self.createFloatLiteral(token)).base
	294	});
	295	try stack.append(State.AfterOperand);
	296	continue;
	297	},
	298	else => return self.parseError(token, "expected primary expression, found {}", @tagName(token.id)),
	299	}
	300	},
	301
	302	State.AfterOperand => {
	303	// we'll either get an infix operator (like != or ^),
	304	// or a postfix operator (like () or {}),
	305	// otherwise this expression is done (like on a ; or else).
	306	var token = self.getNextToken();
	307	switch (token.id) {
	308	Token.Id.EqualEqual => {
	309	try stack.append(State {
	310	.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.EqualEqual)
	311	});
	312	try stack.append(State.ExpectOperand);
	313	continue;
	314	},
	315	Token.Id.BangEqual => {
	316	try stack.append(State {
	317	.InfixOp = try self.createInfixOp(token, ast.NodeInfixOp.InfixOp.BangEqual)
	318	});
	319	try stack.append(State.ExpectOperand);
	320	continue;
	321	},
	322	else => {
	323	// no postfix/infix operator after this operand.
	324	self.putBackToken(token);
	325	// reduce the stack
	326	var expression: &ast.Node = stack.pop().Operand;
	327	while (true) {
	328	switch (stack.pop()) {
	329	State.Expression => \|dest_ptr\| {
	330	// we're done
	331	try dest_ptr.store(expression);
	332	break;
	333	},
	334	State.InfixOp => \|infix_op\| {
	335	infix_op.rhs = expression;
	336	infix_op.lhs = stack.pop().Operand;
	337	expression = &infix_op.base;
	338	continue;
	339	},
	340	State.PrefixOp => \|prefix_op\| {
	341	prefix_op.rhs = expression;
	342	expression = &prefix_op.base;
	343	continue;
	344	},
	345	else => unreachable,
	346	}
	347	}
	348	continue;
	349	},
	350	}
	351	},
	352
	353	State.AddrOfModifiers => \|addr_of_info\| {
	354	var token = self.getNextToken();
	355	switch (token.id) {
	356	Token.Id.Keyword_align => {
	357	stack.append(state) catch unreachable;
	358	if (addr_of_info.align_expr != null) return self.parseError(token, "multiple align qualifiers");
	359	_ = try self.eatToken(Token.Id.LParen);
	360	try stack.append(State { .ExpectToken = Token.Id.RParen });
	361	try stack.append(State { .Expression = DestPtr{.NullableField = &addr_of_info.align_expr} });
	362	continue;
	363	},
	364	Token.Id.Keyword_const => {
	365	stack.append(state) catch unreachable;
	366	if (addr_of_info.const_token != null) return self.parseError(token, "duplicate qualifier: const");
	367	addr_of_info.const_token = token;
	368	continue;
	369	},
	370	Token.Id.Keyword_volatile => {
	371	stack.append(state) catch unreachable;
	372	if (addr_of_info.volatile_token != null) return self.parseError(token, "duplicate qualifier: volatile");
	373	addr_of_info.volatile_token = token;
	374	continue;
	375	},
	376	else => {
	377	self.putBackToken(token);
	378	continue;
	379	},
	380	}
	381	},
	382
	383	State.TypeExpr => \|dest_ptr\| {
	384	const token = self.getNextToken();
	385	if (token.id == Token.Id.Keyword_var) {
	386	@panic("TODO param with type var");
	387	}
	388	self.putBackToken(token);
	389
	390	stack.append(State { .Expression = dest_ptr }) catch unreachable;
	391	continue;
	392	},
	393
	394	State.FnProto => \|fn_proto\| {
	395	stack.append(State { .FnProtoAlign = fn_proto }) catch unreachable;
	396	try stack.append(State { .ParamDecl = fn_proto });
	397	try stack.append(State { .ExpectToken = Token.Id.LParen });
	398
	399	const next_token = self.getNextToken();
	400	if (next_token.id == Token.Id.Identifier) {
	401	fn_proto.name_token = next_token;
	402	continue;
	403	}
	404	self.putBackToken(next_token);
	405	continue;
	406	},
	407
	408	State.FnProtoAlign => \|fn_proto\| {
	409	const token = self.getNextToken();
	410	if (token.id == Token.Id.Keyword_align) {
	411	@panic("TODO fn proto align");
	412	}
	413	self.putBackToken(token);
	414	stack.append(State {
	415	.TypeExpr = DestPtr {.Field = &fn_proto.return_type},
	416	}) catch unreachable;
	417	continue;
	418	},
	419
	420	State.ParamDecl => \|fn_proto\| {
	421	var token = self.getNextToken();
	422	if (token.id == Token.Id.RParen) {
	423	continue;
	424	}
	425	const param_decl = try self.createAttachParamDecl(&fn_proto.params);
	426	if (token.id == Token.Id.Keyword_comptime) {
	427	param_decl.comptime_token = token;
	428	token = self.getNextToken();
	429	} else if (token.id == Token.Id.Keyword_noalias) {
	430	param_decl.noalias_token = token;
	431	token = self.getNextToken();
	432	}
	433	if (token.id == Token.Id.Identifier) {
	434	const next_token = self.getNextToken();
	435	if (next_token.id == Token.Id.Colon) {
	436	param_decl.name_token = token;
	437	token = self.getNextToken();
	438	} else {
	439	self.putBackToken(next_token);
	440	}
	441	}
	442	if (token.id == Token.Id.Ellipsis3) {
	443	param_decl.var_args_token = token;
	444	stack.append(State { .ExpectToken = Token.Id.RParen }) catch unreachable;
	445	continue;
	446	} else {
	447	self.putBackToken(token);
	448	}
	449
	450	stack.append(State { .ParamDecl = fn_proto }) catch unreachable;
	451	try stack.append(State.ParamDeclComma);
	452	try stack.append(State {
	453	.TypeExpr = DestPtr {.Field = &param_decl.type_node}
	454	});
	455	continue;
	456	},
	457
	458	State.ParamDeclComma => {
	459	const token = self.getNextToken();
	460	switch (token.id) {
	461	Token.Id.RParen => {
	462	_ = stack.pop(); // pop off the ParamDecl
	463	continue;
	464	},
	465	Token.Id.Comma => continue,
	466	else => return self.parseError(token, "expected ',' or ')', found {}", @tagName(token.id)),
	467	}
	468	},
	469
	470	State.FnDef => \|fn_proto\| {
	471	const token = self.getNextToken();
	472	switch(token.id) {
	473	Token.Id.LBrace => {
	474	const block = try self.createBlock(token);
	475	fn_proto.body_node = &block.base;
	476	stack.append(State { .Block = block }) catch unreachable;
	477	continue;
	478	},
	479	Token.Id.Semicolon => continue,
	480	else => return self.parseError(token, "expected ';' or '{{', found {}", @tagName(token.id)),
	481	}
	482	},
	483
	484	State.Block => \|block\| {
	485	const token = self.getNextToken();
	486	switch (token.id) {
	487	Token.Id.RBrace => {
	488	block.end_token = token;
	489	continue;
	490	},
	491	else => {
	492	self.putBackToken(token);
	493	stack.append(State { .Block = block }) catch unreachable;
	494	try stack.append(State { .Statement = block });
	495	continue;
	496	},
	497	}
	498	},
	499
	500	State.Statement => \|block\| {
	501	{
	502	// Look for comptime var, comptime const
	503	const comptime_token = self.getNextToken();
	504	if (comptime_token.id == Token.Id.Keyword_comptime) {
	505	const mut_token = self.getNextToken();
	506	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
	507	// TODO shouldn't need these casts
	508	const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
	509	mut_token, (?Token)(comptime_token), (?Token)(null));
	510	try stack.append(State { .VarDecl = var_decl });
	511	continue;
	512	}
	513	self.putBackToken(mut_token);
	514	}
	515	self.putBackToken(comptime_token);
	516	}
	517	{
	518	// Look for const, var
	519	const mut_token = self.getNextToken();
	520	if (mut_token.id == Token.Id.Keyword_var or mut_token.id == Token.Id.Keyword_const) {
	521	// TODO shouldn't need these casts
	522	const var_decl = try self.createAttachVarDecl(&block.statements, (?Token)(null),
	523	mut_token, (?Token)(null), (?Token)(null));
	524	try stack.append(State { .VarDecl = var_decl });
	525	continue;
	526	}
	527	self.putBackToken(mut_token);
	528	}
	529
	530	stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable;
	531	try stack.append(State { .Expression = DestPtr{.List = &block.statements} });
	532	continue;
	533	},
	534
	535	// These are data, not control flow.
	536	State.InfixOp => unreachable,
	537	State.PrefixOp => unreachable,
	538	State.Operand => unreachable,
	539	}
	540	@import("std").debug.panic("{}", @tagName(state));
	541	//unreachable;
	542	}
	543	}
	544
	545	fn createRoot(self: &Parser) !&ast.NodeRoot {
	546	const node = try self.allocator.create(ast.NodeRoot);
	547
	548	*node = ast.NodeRoot {
	549	.base = ast.Node {.id = ast.Node.Id.Root},
	550	.decls = ArrayList(&ast.Node).init(self.allocator),
	551	};
	552	return node;
	553	}
	554
	555	fn createVarDecl(self: &Parser, visib_token: &const ?Token, mut_token: &const Token, comptime_token: &const ?Token,
	556	extern_token: &const ?Token) !&ast.NodeVarDecl
	557	{
	558	const node = try self.allocator.create(ast.NodeVarDecl);
	559
	560	*node = ast.NodeVarDecl {
	561	.base = ast.Node {.id = ast.Node.Id.VarDecl},
	562	.visib_token = *visib_token,
	563	.mut_token = *mut_token,
	564	.comptime_token = *comptime_token,
	565	.extern_token = *extern_token,
	566	.type_node = null,
	567	.align_node = null,
	568	.init_node = null,
	569	.lib_name = null,
	570	// initialized later
	571	.name_token = undefined,
	572	.eq_token = undefined,
	573	};
	574	return node;
	575	}
	576
	577	fn createFnProto(self: &Parser, fn_token: &const Token, extern_token: &const ?Token,
	578	cc_token: &const ?Token, visib_token: &const ?Token, inline_token: &const ?Token) !&ast.NodeFnProto
	579	{
	580	const node = try self.allocator.create(ast.NodeFnProto);
	581
	582	*node = ast.NodeFnProto {
	583	.base = ast.Node {.id = ast.Node.Id.FnProto},
	584	.visib_token = *visib_token,
	585	.name_token = null,
	586	.fn_token = *fn_token,
	587	.params = ArrayList(&ast.Node).init(self.allocator),
	588	.return_type = undefined,
	589	.var_args_token = null,
	590	.extern_token = *extern_token,
	591	.inline_token = *inline_token,
	592	.cc_token = *cc_token,
	593	.body_node = null,
	594	.lib_name = null,
	595	.align_expr = null,
	596	};
	597	return node;
	598	}
	599
	600	fn createParamDecl(self: &Parser) !&ast.NodeParamDecl {
	601	const node = try self.allocator.create(ast.NodeParamDecl);
	602
	603	*node = ast.NodeParamDecl {
	604	.base = ast.Node {.id = ast.Node.Id.ParamDecl},
	605	.comptime_token = null,
	606	.noalias_token = null,
	607	.name_token = null,
	608	.type_node = undefined,
	609	.var_args_token = null,
	610	};
	611	return node;
	612	}
	613
	614	fn createBlock(self: &Parser, begin_token: &const Token) !&ast.NodeBlock {
	615	const node = try self.allocator.create(ast.NodeBlock);
	616
	617	*node = ast.NodeBlock {
	618	.base = ast.Node {.id = ast.Node.Id.Block},
	619	.begin_token = *begin_token,
	620	.end_token = undefined,
	621	.statements = ArrayList(&ast.Node).init(self.allocator),
	622	};
	623	return node;
	624	}
	625
	626	fn createInfixOp(self: &Parser, op_token: &const Token, op: &const ast.NodeInfixOp.InfixOp) !&ast.NodeInfixOp {
	627	const node = try self.allocator.create(ast.NodeInfixOp);
	628
	629	*node = ast.NodeInfixOp {
	630	.base = ast.Node {.id = ast.Node.Id.InfixOp},
	631	.op_token = *op_token,
	632	.lhs = undefined,
	633	.op = *op,
	634	.rhs = undefined,
	635	};
	636	return node;
	637	}
	638
	639	fn createPrefixOp(self: &Parser, op_token: &const Token, op: &const ast.NodePrefixOp.PrefixOp) !&ast.NodePrefixOp {
	640	const node = try self.allocator.create(ast.NodePrefixOp);
	641
	642	*node = ast.NodePrefixOp {
	643	.base = ast.Node {.id = ast.Node.Id.PrefixOp},
	644	.op_token = *op_token,
	645	.op = *op,
	646	.rhs = undefined,
	647	};
	648	return node;
	649	}
	650
	651	fn createIdentifier(self: &Parser, name_token: &const Token) !&ast.NodeIdentifier {
	652	const node = try self.allocator.create(ast.NodeIdentifier);
	653
	654	*node = ast.NodeIdentifier {
	655	.base = ast.Node {.id = ast.Node.Id.Identifier},
	656	.name_token = *name_token,
	657	};
	658	return node;
	659	}
	660
	661	fn createIntegerLiteral(self: &Parser, token: &const Token) !&ast.NodeIntegerLiteral {
	662	const node = try self.allocator.create(ast.NodeIntegerLiteral);
	663
	664	*node = ast.NodeIntegerLiteral {
	665	.base = ast.Node {.id = ast.Node.Id.IntegerLiteral},
	666	.token = *token,
	667	};
	668	return node;
	669	}
	670
	671	fn createFloatLiteral(self: &Parser, token: &const Token) !&ast.NodeFloatLiteral {
	672	const node = try self.allocator.create(ast.NodeFloatLiteral);
	673
	674	*node = ast.NodeFloatLiteral {
	675	.base = ast.Node {.id = ast.Node.Id.FloatLiteral},
	676	.token = *token,
	677	};
	678	return node;
	679	}
	680
	681	fn createAttachIdentifier(self: &Parser, dest_ptr: &const DestPtr, name_token: &const Token) !&ast.NodeIdentifier {
	682	const node = try self.createIdentifier(name_token);
	683	try dest_ptr.store(&node.base);
	684	return node;
	685	}
	686
	687	fn createAttachParamDecl(self: &Parser, list: &ArrayList(&ast.Node)) !&ast.NodeParamDecl {
	688	const node = try self.createParamDecl();
	689	try list.append(&node.base);
	690	return node;
	691	}
	692
	693	fn createAttachFnProto(self: &Parser, list: &ArrayList(&ast.Node), fn_token: &const Token,
	694	extern_token: &const ?Token, cc_token: &const ?Token, visib_token: &const ?Token,
	695	inline_token: &const ?Token) !&ast.NodeFnProto
	696	{
	697	const node = try self.createFnProto(fn_token, extern_token, cc_token, visib_token, inline_token);
	698	try list.append(&node.base);
	699	return node;
	700	}
	701
	702	fn createAttachVarDecl(self: &Parser, list: &ArrayList(&ast.Node), visib_token: &const ?Token,
	703	mut_token: &const Token, comptime_token: &const ?Token, extern_token: &const ?Token) !&ast.NodeVarDecl
	704	{
	705	const node = try self.createVarDecl(visib_token, mut_token, comptime_token, extern_token);
	706	try list.append(&node.base);
	707	return node;
	708	}
	709
	710	fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) error {
	711	const loc = self.tokenizer.getTokenLocation(token);
	712	warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args);
	713	warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]);
	714	{
	715	var i: usize = 0;
	716	while (i < loc.column) : (i += 1) {
	717	warn(" ");
	718	}
	719	}
	720	{
	721	const caret_count = token.end - token.start;
	722	var i: usize = 0;
	723	while (i < caret_count) : (i += 1) {
	724	warn("~");
	725	}
	726	}
	727	warn("\n");
	728	return error.ParseError;
	729	}
	730
	731	fn expectToken(self: &Parser, token: &const Token, id: @TagType(Token.Id)) !void {
	732	if (token.id != id) {
	733	return self.parseError(token, "expected {}, found {}", @tagName(id), @tagName(token.id));
	734	}
	735	}
	736
	737	fn eatToken(self: &Parser, id: @TagType(Token.Id)) !Token {
	738	const token = self.getNextToken();
	739	try self.expectToken(token, id);
	740	return token;
	741	}
	742
	743	fn putBackToken(self: &Parser, token: &const Token) void {
	744	self.put_back_tokens[self.put_back_count] = *token;
	745	self.put_back_count += 1;
	746	}
	747
	748	fn getNextToken(self: &Parser) Token {
	749	if (self.put_back_count != 0) {
	750	const put_back_index = self.put_back_count - 1;
	751	const put_back_token = self.put_back_tokens[put_back_index];
	752	self.put_back_count = put_back_index;
	753	return put_back_token;
	754	} else {
	755	return self.tokenizer.next();
	756	}
	757	}
	758
	759	const RenderAstFrame = struct {
	760	node: &ast.Node,
	761	indent: usize,
	762	};
	763
	764	pub fn renderAst(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
	765	var stack = self.initUtilityArrayList(RenderAstFrame);
	766	defer self.deinitUtilityArrayList(stack);
	767
	768	try stack.append(RenderAstFrame {
	769	.node = &root_node.base,
	770	.indent = 0,
	771	});
	772
	773	while (stack.popOrNull()) \|frame\| {
	774	{
	775	var i: usize = 0;
	776	while (i < frame.indent) : (i += 1) {
	777	try stream.print(" ");
	778	}
	779	}
	780	try stream.print("{}\n", @tagName(frame.node.id));
	781	var child_i: usize = 0;
	782	while (frame.node.iterate(child_i)) \|child\| : (child_i += 1) {
	783	try stack.append(RenderAstFrame {
	784	.node = child,
	785	.indent = frame.indent + 2,
	786	});
	787	}
	788	}
	789	}
	790
	791	const RenderState = union(enum) {
	792	TopLevelDecl: &ast.Node,
	793	FnProtoRParen: &ast.NodeFnProto,
	794	ParamDecl: &ast.Node,
	795	Text: []const u8,
	796	Expression: &ast.Node,
	797	VarDecl: &ast.NodeVarDecl,
	798	Statement: &ast.Node,
	799	PrintIndent,
	800	Indent: usize,
	801	};
	802
	803	pub fn renderSource(self: &Parser, stream: var, root_node: &ast.NodeRoot) !void {
	804	var stack = self.initUtilityArrayList(RenderState);
	805	defer self.deinitUtilityArrayList(stack);
	806
	807	{
	808	var i = root_node.decls.len;
	809	while (i != 0) {
	810	i -= 1;
	811	const decl = root_node.decls.items[i];
	812	try stack.append(RenderState {.TopLevelDecl = decl});
	813	}
	814	}
	815
	816	const indent_delta = 4;
	817	var indent: usize = 0;
	818	while (stack.popOrNull()) \|state\| {
	819	switch (state) {
	820	RenderState.TopLevelDecl => \|decl\| {
	821	switch (decl.id) {
	822	ast.Node.Id.FnProto => {
	823	const fn_proto = @fieldParentPtr(ast.NodeFnProto, "base", decl);
	824	if (fn_proto.visib_token) \|visib_token\| {
	825	switch (visib_token.id) {
	826	Token.Id.Keyword_pub => try stream.print("pub "),
	827	Token.Id.Keyword_export => try stream.print("export "),
	828	else => unreachable,
	829	}
	830	}
	831	if (fn_proto.extern_token) \|extern_token\| {
	832	try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
	833	}
	834	try stream.print("fn");
	835
	836	if (fn_proto.name_token) \|name_token\| {
	837	try stream.print(" {}", self.tokenizer.getTokenSlice(name_token));
	838	}
	839
	840	try stream.print("(");
	841
	842	try stack.append(RenderState { .Text = "\n" });
	843	if (fn_proto.body_node == null) {
	844	try stack.append(RenderState { .Text = ";" });
	845	}
	846
	847	try stack.append(RenderState { .FnProtoRParen = fn_proto});
	848	var i = fn_proto.params.len;
	849	while (i != 0) {
	850	i -= 1;
	851	const param_decl_node = fn_proto.params.items[i];
	852	try stack.append(RenderState { .ParamDecl = param_decl_node});
	853	if (i != 0) {
	854	try stack.append(RenderState { .Text = ", " });
	855	}
	856	}
	857	},
	858	ast.Node.Id.VarDecl => {
	859	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl);
	860	try stack.append(RenderState { .Text = "\n"});
	861	try stack.append(RenderState { .VarDecl = var_decl});
	862
	863	},
	864	else => unreachable,
	865	}
	866	},
	867
	868	RenderState.VarDecl => \|var_decl\| {
	869	if (var_decl.visib_token) \|visib_token\| {
	870	try stream.print("{} ", self.tokenizer.getTokenSlice(visib_token));
	871	}
	872	if (var_decl.extern_token) \|extern_token\| {
	873	try stream.print("{} ", self.tokenizer.getTokenSlice(extern_token));
	874	if (var_decl.lib_name != null) {
	875	@panic("TODO");
	876	}
	877	}
	878	if (var_decl.comptime_token) \|comptime_token\| {
	879	try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
	880	}
	881	try stream.print("{} ", self.tokenizer.getTokenSlice(var_decl.mut_token));
	882	try stream.print("{}", self.tokenizer.getTokenSlice(var_decl.name_token));
	883
	884	try stack.append(RenderState { .Text = ";" });
	885	if (var_decl.init_node) \|init_node\| {
	886	try stack.append(RenderState { .Expression = init_node });
	887	try stack.append(RenderState { .Text = " = " });
	888	}
	889	if (var_decl.align_node) \|align_node\| {
	890	try stack.append(RenderState { .Text = ")" });
	891	try stack.append(RenderState { .Expression = align_node });
	892	try stack.append(RenderState { .Text = " align(" });
	893	}
	894	if (var_decl.type_node) \|type_node\| {
	895	try stream.print(": ");
	896	try stack.append(RenderState { .Expression = type_node });
	897	}
	898	},
	899
	900	RenderState.ParamDecl => \|base\| {
	901	const param_decl = @fieldParentPtr(ast.NodeParamDecl, "base", base);
	902	if (param_decl.comptime_token) \|comptime_token\| {
	903	try stream.print("{} ", self.tokenizer.getTokenSlice(comptime_token));
	904	}
	905	if (param_decl.noalias_token) \|noalias_token\| {
	906	try stream.print("{} ", self.tokenizer.getTokenSlice(noalias_token));
	907	}
	908	if (param_decl.name_token) \|name_token\| {
	909	try stream.print("{}: ", self.tokenizer.getTokenSlice(name_token));
	910	}
	911	if (param_decl.var_args_token) \|var_args_token\| {
	912	try stream.print("{}", self.tokenizer.getTokenSlice(var_args_token));
	913	} else {
	914	try stack.append(RenderState { .Expression = param_decl.type_node});
	915	}
	916	},
	917	RenderState.Text => \|bytes\| {
	918	try stream.write(bytes);
	919	},
	920	RenderState.Expression => \|base\| switch (base.id) {
	921	ast.Node.Id.Identifier => {
	922	const identifier = @fieldParentPtr(ast.NodeIdentifier, "base", base);
	923	try stream.print("{}", self.tokenizer.getTokenSlice(identifier.name_token));
	924	},
	925	ast.Node.Id.Block => {
	926	const block = @fieldParentPtr(ast.NodeBlock, "base", base);
	927	try stream.write("{");
	928	try stack.append(RenderState { .Text = "}"});
	929	try stack.append(RenderState.PrintIndent);
	930	try stack.append(RenderState { .Indent = indent});
	931	try stack.append(RenderState { .Text = "\n"});
	932	var i = block.statements.len;
	933	while (i != 0) {
	934	i -= 1;
	935	const statement_node = block.statements.items[i];
	936	try stack.append(RenderState { .Statement = statement_node});
	937	try stack.append(RenderState.PrintIndent);
	938	try stack.append(RenderState { .Indent = indent + indent_delta});
	939	try stack.append(RenderState { .Text = "\n" });
	940	}
	941	},
	942	ast.Node.Id.InfixOp => {
	943	const prefix_op_node = @fieldParentPtr(ast.NodeInfixOp, "base", base);
	944	try stack.append(RenderState { .Expression = prefix_op_node.rhs });
	945	switch (prefix_op_node.op) {
	946	ast.NodeInfixOp.InfixOp.EqualEqual => {
	947	try stack.append(RenderState { .Text = " == "});
	948	},
	949	ast.NodeInfixOp.InfixOp.BangEqual => {
	950	try stack.append(RenderState { .Text = " != "});
	951	},
	952	else => unreachable,
	953	}
	954	try stack.append(RenderState { .Expression = prefix_op_node.lhs });
	955	},
	956	ast.Node.Id.PrefixOp => {
	957	const prefix_op_node = @fieldParentPtr(ast.NodePrefixOp, "base", base);
	958	try stack.append(RenderState { .Expression = prefix_op_node.rhs });
	959	switch (prefix_op_node.op) {
	960	ast.NodePrefixOp.PrefixOp.Return => {
	961	try stream.write("return ");
	962	},
	963	ast.NodePrefixOp.PrefixOp.AddrOf => \|addr_of_info\| {
	964	try stream.write("&");
	965	if (addr_of_info.volatile_token != null) {
	966	try stack.append(RenderState { .Text = "volatile "});
	967	}
	968	if (addr_of_info.const_token != null) {
	969	try stack.append(RenderState { .Text = "const "});
	970	}
	971	if (addr_of_info.align_expr) \|align_expr\| {
	972	try stream.print("align(");
	973	try stack.append(RenderState { .Text = ") "});
	974	try stack.append(RenderState { .Expression = align_expr});
	975	}
	976	},
	977	else => unreachable,
	978	}
	979	},
	980	ast.Node.Id.IntegerLiteral => {
	981	const integer_literal = @fieldParentPtr(ast.NodeIntegerLiteral, "base", base);
	982	try stream.print("{}", self.tokenizer.getTokenSlice(integer_literal.token));
	983	},
	984	ast.Node.Id.FloatLiteral => {
	985	const float_literal = @fieldParentPtr(ast.NodeFloatLiteral, "base", base);
	986	try stream.print("{}", self.tokenizer.getTokenSlice(float_literal.token));
	987	},
	988	else => unreachable,
	989	},
	990	RenderState.FnProtoRParen => \|fn_proto\| {
	991	try stream.print(")");
	992	if (fn_proto.align_expr != null) {
	993	@panic("TODO");
	994	}
	995	try stream.print(" ");
	996	if (fn_proto.body_node) \|body_node\| {
	997	try stack.append(RenderState { .Expression = body_node});
	998	try stack.append(RenderState { .Text = " "});
	999	}
	1000	try stack.append(RenderState { .Expression = fn_proto.return_type});
	1001	},
	1002	RenderState.Statement => \|base\| {
	1003	switch (base.id) {
	1004	ast.Node.Id.VarDecl => {
	1005	const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", base);
	1006	try stack.append(RenderState { .VarDecl = var_decl});
	1007	},
	1008	else => {
	1009	try stack.append(RenderState { .Text = ";"});
	1010	try stack.append(RenderState { .Expression = base});
	1011	},
	1012	}
	1013	},
	1014	RenderState.Indent => \|new_indent\| indent = new_indent,
	1015	RenderState.PrintIndent => try stream.writeByteNTimes(' ', indent),
	1016	}
	1017	}
	1018	}
	1019
	1020	fn initUtilityArrayList(self: &Parser, comptime T: type) ArrayList(T) {
	1021	const new_byte_count = self.utility_bytes.len - self.utility_bytes.len % @sizeOf(T);
	1022	self.utility_bytes = self.allocator.alignedShrink(u8, utility_bytes_align, self.utility_bytes, new_byte_count);
	1023	const typed_slice = ([]T)(self.utility_bytes);
	1024	return ArrayList(T) {
	1025	.allocator = self.allocator,
	1026	.items = typed_slice,
	1027	.len = 0,
	1028	};
	1029	}
	1030
	1031	fn deinitUtilityArrayList(self: &Parser, list: var) void {
	1032	self.utility_bytes = ([]align(utility_bytes_align) u8)(list.items);
	1033	}
	1034
	1035	};
	1036
	1037	var fixed_buffer_mem: [100 * 1024]u8 = undefined;
	1038
	1039	fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
	1040	var padded_source: [0x100]u8 = undefined;
	1041	std.mem.copy(u8, padded_source[0..source.len], source);
	1042	padded_source[source.len + 0] = '\n';
	1043	padded_source[source.len + 1] = '\n';
	1044	padded_source[source.len + 2] = '\n';
	1045
	1046	var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
	1047	var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
	1048	defer parser.deinit();
	1049
	1050	const tree = try parser.parse();
	1051	defer tree.deinit();
	1052
	1053	var buffer = try std.Buffer.initSize(allocator, 0);
	1054	var buffer_out_stream = io.BufferOutStream.init(&buffer);
	1055	try parser.renderSource(&buffer_out_stream.stream, tree.root_node);
	1056	return buffer.toOwnedSlice();
	1057	}
	1058
	1059	// TODO test for memory leaks
	1060	// TODO test for valid frees
	1061	fn testCanonical(source: []const u8) !void {
	1062	const needed_alloc_count = x: {
	1063	// Try it once with unlimited memory, make sure it works
	1064	var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
	1065	var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, @maxValue(usize));
	1066	const result_source = try testParse(source, &failing_allocator.allocator);
	1067	if (!mem.eql(u8, result_source, source)) {
	1068	warn("\n====== expected this output: =========\n");
	1069	warn("{}", source);
	1070	warn("\n======== instead found this: =========\n");
	1071	warn("{}", result_source);
	1072	warn("\n======================================\n");
	1073	return error.TestFailed;
	1074	}
	1075	failing_allocator.allocator.free(result_source);
	1076	break :x failing_allocator.index;
	1077	};
	1078
	1079	var fail_index: usize = 0;
	1080	while (fail_index < needed_alloc_count) : (fail_index += 1) {
	1081	var fixed_allocator = mem.FixedBufferAllocator.init(fixed_buffer_mem[0..]);
	1082	var failing_allocator = std.debug.FailingAllocator.init(&fixed_allocator.allocator, fail_index);
	1083	if (testParse(source, &failing_allocator.allocator)) \|_\| {
	1084	return error.NondeterministicMemoryUsage;
	1085	} else \|err\| {
	1086	assert(err == error.OutOfMemory);
	1087	// TODO make this pass
	1088	//if (failing_allocator.allocated_bytes != failing_allocator.freed_bytes) {
	1089	// warn("\nfail_index: {}/{}\nallocated bytes: {}\nfreed bytes: {}\nallocations: {}\ndeallocations: {}\n",
	1090	// fail_index, needed_alloc_count,
	1091	// failing_allocator.allocated_bytes, failing_allocator.freed_bytes,
	1092	// failing_allocator.index, failing_allocator.deallocations);
	1093	// return error.MemoryLeakDetected;
	1094	//}
	1095	}
	1096	}
	1097	}
	1098
	1099	test "zig fmt" {
	1100	try testCanonical(
	1101	\\extern fn puts(s: &const u8) c_int;
	1102	\\
	1103	);
	1104
	1105	try testCanonical(
	1106	\\const a = b;
	1107	\\pub const a = b;
	1108	\\var a = b;
	1109	\\pub var a = b;
	1110	\\const a: i32 = b;
	1111	\\pub const a: i32 = b;
	1112	\\var a: i32 = b;
	1113	\\pub var a: i32 = b;
	1114	\\
	1115	);
	1116
	1117	try testCanonical(
	1118	\\extern var foo: c_int;
	1119	\\
	1120	);
	1121
	1122	try testCanonical(
	1123	\\var foo: c_int align(1);
	1124	\\
	1125	);
	1126
	1127	try testCanonical(
	1128	\\fn main(argc: c_int, argv: &&u8) c_int {
	1129	\\ const a = b;
	1130	\\}
	1131	\\
	1132	);
	1133
	1134	try testCanonical(
	1135	\\fn foo(argc: c_int, argv: &&u8) c_int {
	1136	\\ return 0;
	1137	\\}
	1138	\\
	1139	);
	1140
	1141	try testCanonical(
	1142	\\extern fn f1(s: &align(&u8) u8) c_int;
	1143	\\
	1144	);
	1145
	1146	try testCanonical(
	1147	\\extern fn f1(s: &&align(1) &const &volatile u8) c_int;
	1148	\\extern fn f2(s: &align(1) const &align(1) volatile &const volatile u8) c_int;
	1149	\\extern fn f3(s: &align(1) const volatile u8) c_int;
	1150	\\
	1151	);
	1152
	1153	try testCanonical(
	1154	\\fn f1(a: bool, b: bool) bool {
	1155	\\ a != b;
	1156	\\ return a == b;
	1157	\\}
	1158	\\
	1159	);
	1160	}

std/zig/tokenizer.zig created+659

...	...	@@ -0,0 +1,659 @@
	1	const std = @import("../index.zig");
	2	const mem = std.mem;
	3
	4	pub const Token = struct {
	5	id: Id,
	6	start: usize,
	7	end: usize,
	8
	9	const KeywordId = struct {
	10	bytes: []const u8,
	11	id: Id,
	12	};
	13
	14	const keywords = []KeywordId {
	15	KeywordId{.bytes="align", .id = Id.Keyword_align},
	16	KeywordId{.bytes="and", .id = Id.Keyword_and},
	17	KeywordId{.bytes="asm", .id = Id.Keyword_asm},
	18	KeywordId{.bytes="break", .id = Id.Keyword_break},
	19	KeywordId{.bytes="comptime", .id = Id.Keyword_comptime},
	20	KeywordId{.bytes="const", .id = Id.Keyword_const},
	21	KeywordId{.bytes="continue", .id = Id.Keyword_continue},
	22	KeywordId{.bytes="defer", .id = Id.Keyword_defer},
	23	KeywordId{.bytes="else", .id = Id.Keyword_else},
	24	KeywordId{.bytes="enum", .id = Id.Keyword_enum},
	25	KeywordId{.bytes="error", .id = Id.Keyword_error},
	26	KeywordId{.bytes="export", .id = Id.Keyword_export},
	27	KeywordId{.bytes="extern", .id = Id.Keyword_extern},
	28	KeywordId{.bytes="false", .id = Id.Keyword_false},
	29	KeywordId{.bytes="fn", .id = Id.Keyword_fn},
	30	KeywordId{.bytes="for", .id = Id.Keyword_for},
	31	KeywordId{.bytes="goto", .id = Id.Keyword_goto},
	32	KeywordId{.bytes="if", .id = Id.Keyword_if},
	33	KeywordId{.bytes="inline", .id = Id.Keyword_inline},
	34	KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc},
	35	KeywordId{.bytes="noalias", .id = Id.Keyword_noalias},
	36	KeywordId{.bytes="null", .id = Id.Keyword_null},
	37	KeywordId{.bytes="or", .id = Id.Keyword_or},
	38	KeywordId{.bytes="packed", .id = Id.Keyword_packed},
	39	KeywordId{.bytes="pub", .id = Id.Keyword_pub},
	40	KeywordId{.bytes="return", .id = Id.Keyword_return},
	41	KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc},
	42	KeywordId{.bytes="struct", .id = Id.Keyword_struct},
	43	KeywordId{.bytes="switch", .id = Id.Keyword_switch},
	44	KeywordId{.bytes="test", .id = Id.Keyword_test},
	45	KeywordId{.bytes="this", .id = Id.Keyword_this},
	46	KeywordId{.bytes="true", .id = Id.Keyword_true},
	47	KeywordId{.bytes="undefined", .id = Id.Keyword_undefined},
	48	KeywordId{.bytes="union", .id = Id.Keyword_union},
	49	KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable},
	50	KeywordId{.bytes="use", .id = Id.Keyword_use},
	51	KeywordId{.bytes="var", .id = Id.Keyword_var},
	52	KeywordId{.bytes="volatile", .id = Id.Keyword_volatile},
	53	KeywordId{.bytes="while", .id = Id.Keyword_while},
	54	};
	55
	56	fn getKeyword(bytes: []const u8) ?Id {
	57	for (keywords) \|kw\| {
	58	if (mem.eql(u8, kw.bytes, bytes)) {
	59	return kw.id;
	60	}
	61	}
	62	return null;
	63	}
	64
	65	const StrLitKind = enum {Normal, C};
	66
	67	pub const Id = union(enum) {
	68	Invalid,
	69	Identifier,
	70	StringLiteral: StrLitKind,
	71	Eof,
	72	Builtin,
	73	Bang,
	74	Equal,
	75	EqualEqual,
	76	BangEqual,
	77	LParen,
	78	RParen,
	79	Semicolon,
	80	Percent,
	81	LBrace,
	82	RBrace,
	83	Period,
	84	Ellipsis2,
	85	Ellipsis3,
	86	Minus,
	87	Arrow,
	88	Colon,
	89	Slash,
	90	Comma,
	91	Ampersand,
	92	AmpersandEqual,
	93	IntegerLiteral,
	94	FloatLiteral,
	95	Keyword_align,
	96	Keyword_and,
	97	Keyword_asm,
	98	Keyword_break,
	99	Keyword_comptime,
	100	Keyword_const,
	101	Keyword_continue,
	102	Keyword_defer,
	103	Keyword_else,
	104	Keyword_enum,
	105	Keyword_error,
	106	Keyword_export,
	107	Keyword_extern,
	108	Keyword_false,
	109	Keyword_fn,
	110	Keyword_for,
	111	Keyword_goto,
	112	Keyword_if,
	113	Keyword_inline,
	114	Keyword_nakedcc,
	115	Keyword_noalias,
	116	Keyword_null,
	117	Keyword_or,
	118	Keyword_packed,
	119	Keyword_pub,
	120	Keyword_return,
	121	Keyword_stdcallcc,
	122	Keyword_struct,
	123	Keyword_switch,
	124	Keyword_test,
	125	Keyword_this,
	126	Keyword_true,
	127	Keyword_undefined,
	128	Keyword_union,
	129	Keyword_unreachable,
	130	Keyword_use,
	131	Keyword_var,
	132	Keyword_volatile,
	133	Keyword_while,
	134	};
	135	};
	136
	137	pub const Tokenizer = struct {
	138	buffer: []const u8,
	139	index: usize,
	140	pending_invalid_token: ?Token,
	141
	142	pub const Location = struct {
	143	line: usize,
	144	column: usize,
	145	line_start: usize,
	146	line_end: usize,
	147	};
	148
	149	pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location {
	150	var loc = Location {
	151	.line = 0,
	152	.column = 0,
	153	.line_start = 0,
	154	.line_end = 0,
	155	};
	156	for (self.buffer) \|c, i\| {
	157	if (i == token.start) {
	158	loc.line_end = i;
	159	while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
	160	return loc;
	161	}
	162	if (c == '\n') {
	163	loc.line += 1;
	164	loc.column = 0;
	165	loc.line_start = i + 1;
	166	} else {
	167	loc.column += 1;
	168	}
	169	}
	170	return loc;
	171	}
	172
	173	/// For debugging purposes
	174	pub fn dump(self: &Tokenizer, token: &const Token) void {
	175	std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
	176	}
	177
	178	/// buffer must end with "\n\n\n". This is so that attempting to decode
	179	/// a the 3 trailing bytes of a 4-byte utf8 sequence is never a buffer overflow.
	180	pub fn init(buffer: []const u8) Tokenizer {
	181	std.debug.assert(buffer[buffer.len - 1] == '\n');
	182	std.debug.assert(buffer[buffer.len - 2] == '\n');
	183	std.debug.assert(buffer[buffer.len - 3] == '\n');
	184	return Tokenizer {
	185	.buffer = buffer,
	186	.index = 0,
	187	.pending_invalid_token = null,
	188	};
	189	}
	190
	191	const State = enum {
	192	Start,
	193	Identifier,
	194	Builtin,
	195	C,
	196	StringLiteral,
	197	StringLiteralBackslash,
	198	Equal,
	199	Bang,
	200	Minus,
	201	Slash,
	202	LineComment,
	203	Zero,
	204	IntegerLiteral,
	205	IntegerLiteralWithRadix,
	206	NumberDot,
	207	FloatFraction,
	208	FloatExponentUnsigned,
	209	FloatExponentNumber,
	210	Ampersand,
	211	Period,
	212	Period2,
	213	};
	214
	215	pub fn next(self: &Tokenizer) Token {
	216	if (self.pending_invalid_token) \|token\| {
	217	self.pending_invalid_token = null;
	218	return token;
	219	}
	220	var state = State.Start;
	221	var result = Token {
	222	.id = Token.Id.Eof,
	223	.start = self.index,
	224	.end = undefined,
	225	};
	226	while (self.index < self.buffer.len) : (self.index += 1) {
	227	const c = self.buffer[self.index];
	228	switch (state) {
	229	State.Start => switch (c) {
	230	' ', '\n' => {
	231	result.start = self.index + 1;
	232	},
	233	'c' => {
	234	state = State.C;
	235	result.id = Token.Id.Identifier;
	236	},
	237	'"' => {
	238	state = State.StringLiteral;
	239	result.id = Token.Id { .StringLiteral = Token.StrLitKind.Normal };
	240	},
	241	'a'...'b', 'd'...'z', 'A'...'Z', '_' => {
	242	state = State.Identifier;
	243	result.id = Token.Id.Identifier;
	244	},
	245	'@' => {
	246	state = State.Builtin;
	247	result.id = Token.Id.Builtin;
	248	},
	249	'=' => {
	250	state = State.Equal;
	251	},
	252	'!' => {
	253	state = State.Bang;
	254	},
	255	'(' => {
	256	result.id = Token.Id.LParen;
	257	self.index += 1;
	258	break;
	259	},
	260	')' => {
	261	result.id = Token.Id.RParen;
	262	self.index += 1;
	263	break;
	264	},
	265	';' => {
	266	result.id = Token.Id.Semicolon;
	267	self.index += 1;
	268	break;
	269	},
	270	',' => {
	271	result.id = Token.Id.Comma;
	272	self.index += 1;
	273	break;
	274	},
	275	':' => {
	276	result.id = Token.Id.Colon;
	277	self.index += 1;
	278	break;
	279	},
	280	'%' => {
	281	result.id = Token.Id.Percent;
	282	self.index += 1;
	283	break;
	284	},
	285	'{' => {
	286	result.id = Token.Id.LBrace;
	287	self.index += 1;
	288	break;
	289	},
	290	'}' => {
	291	result.id = Token.Id.RBrace;
	292	self.index += 1;
	293	break;
	294	},
	295	'.' => {
	296	state = State.Period;
	297	},
	298	'-' => {
	299	state = State.Minus;
	300	},
	301	'/' => {
	302	state = State.Slash;
	303	},
	304	'&' => {
	305	state = State.Ampersand;
	306	},
	307	'0' => {
	308	state = State.Zero;
	309	result.id = Token.Id.IntegerLiteral;
	310	},
	311	'1'...'9' => {
	312	state = State.IntegerLiteral;
	313	result.id = Token.Id.IntegerLiteral;
	314	},
	315	else => {
	316	result.id = Token.Id.Invalid;
	317	self.index += 1;
	318	break;
	319	},
	320	},
	321	State.Ampersand => switch (c) {
	322	'=' => {
	323	result.id = Token.Id.AmpersandEqual;
	324	self.index += 1;
	325	break;
	326	},
	327	else => {
	328	result.id = Token.Id.Ampersand;
	329	break;
	330	},
	331	},
	332	State.Identifier => switch (c) {
	333	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	334	else => {
	335	if (Token.getKeyword(self.buffer[result.start..self.index])) \|id\| {
	336	result.id = id;
	337	}
	338	break;
	339	},
	340	},
	341	State.Builtin => switch (c) {
	342	'a'...'z', 'A'...'Z', '_', '0'...'9' => {},
	343	else => break,
	344	},
	345	State.C => switch (c) {
	346	'\\' => @panic("TODO"),
	347	'"' => {
	348	state = State.StringLiteral;
	349	result.id = Token.Id { .StringLiteral = Token.StrLitKind.C };
	350	},
	351	'a'...'z', 'A'...'Z', '_', '0'...'9' => {
	352	state = State.Identifier;
	353	},
	354	else => break,
	355	},
	356	State.StringLiteral => switch (c) {
	357	'\\' => {
	358	state = State.StringLiteralBackslash;
	359	},
	360	'"' => {
	361	self.index += 1;
	362	break;
	363	},
	364	'\n' => break, // Look for this error later.
	365	else => self.checkLiteralCharacter(),
	366	},
	367
	368	State.StringLiteralBackslash => switch (c) {
	369	'\n' => break, // Look for this error later.
	370	else => {
	371	state = State.StringLiteral;
	372	},
	373	},
	374
	375	State.Bang => switch (c) {
	376	'=' => {
	377	result.id = Token.Id.BangEqual;
	378	self.index += 1;
	379	break;
	380	},
	381	else => {
	382	result.id = Token.Id.Bang;
	383	break;
	384	},
	385	},
	386
	387	State.Equal => switch (c) {
	388	'=' => {
	389	result.id = Token.Id.EqualEqual;
	390	self.index += 1;
	391	break;
	392	},
	393	else => {
	394	result.id = Token.Id.Equal;
	395	break;
	396	},
	397	},
	398
	399	State.Minus => switch (c) {
	400	'>' => {
	401	result.id = Token.Id.Arrow;
	402	self.index += 1;
	403	break;
	404	},
	405	else => {
	406	result.id = Token.Id.Minus;
	407	break;
	408	},
	409	},
	410
	411	State.Period => switch (c) {
	412	'.' => {
	413	state = State.Period2;
	414	},
	415	else => {
	416	result.id = Token.Id.Period;
	417	break;
	418	},
	419	},
	420
	421	State.Period2 => switch (c) {
	422	'.' => {
	423	result.id = Token.Id.Ellipsis3;
	424	self.index += 1;
	425	break;
	426	},
	427	else => {
	428	result.id = Token.Id.Ellipsis2;
	429	break;
	430	},
	431	},
	432
	433	State.Slash => switch (c) {
	434	'/' => {
	435	result.id = undefined;
	436	state = State.LineComment;
	437	},
	438	else => {
	439	result.id = Token.Id.Slash;
	440	break;
	441	},
	442	},
	443	State.LineComment => switch (c) {
	444	'\n' => {
	445	state = State.Start;
	446	result = Token {
	447	.id = Token.Id.Eof,
	448	.start = self.index + 1,
	449	.end = undefined,
	450	};
	451	},
	452	else => self.checkLiteralCharacter(),
	453	},
	454	State.Zero => switch (c) {
	455	'b', 'o', 'x' => {
	456	state = State.IntegerLiteralWithRadix;
	457	},
	458	else => {
	459	// reinterpret as a normal number
	460	self.index -= 1;
	461	state = State.IntegerLiteral;
	462	},
	463	},
	464	State.IntegerLiteral => switch (c) {
	465	'.' => {
	466	state = State.NumberDot;
	467	},
	468	'p', 'P', 'e', 'E' => {
	469	state = State.FloatExponentUnsigned;
	470	},
	471	'0'...'9' => {},
	472	else => break,
	473	},
	474	State.IntegerLiteralWithRadix => switch (c) {
	475	'.' => {
	476	state = State.NumberDot;
	477	},
	478	'p', 'P' => {
	479	state = State.FloatExponentUnsigned;
	480	},
	481	'0'...'9', 'a'...'f', 'A'...'F' => {},
	482	else => break,
	483	},
	484	State.NumberDot => switch (c) {
	485	'.' => {
	486	self.index -= 1;
	487	state = State.Start;
	488	break;
	489	},
	490	else => {
	491	self.index -= 1;
	492	result.id = Token.Id.FloatLiteral;
	493	state = State.FloatFraction;
	494	},
	495	},
	496	State.FloatFraction => switch (c) {
	497	'p', 'P' => {
	498	state = State.FloatExponentUnsigned;
	499	},
	500	'0'...'9', 'a'...'f', 'A'...'F' => {},
	501	else => break,
	502	},
	503	State.FloatExponentUnsigned => switch (c) {
	504	'+', '-' => {
	505	state = State.FloatExponentNumber;
	506	},
	507	else => {
	508	// reinterpret as a normal exponent number
	509	self.index -= 1;
	510	state = State.FloatExponentNumber;
	511	}
	512	},
	513	State.FloatExponentNumber => switch (c) {
	514	'0'...'9', 'a'...'f', 'A'...'F' => {},
	515	else => break,
	516	},
	517	}
	518	}
	519	result.end = self.index;
	520
	521	if (result.id == Token.Id.Eof) {
	522	if (self.pending_invalid_token) \|token\| {
	523	self.pending_invalid_token = null;
	524	return token;
	525	}
	526	}
	527
	528	return result;
	529	}
	530
	531	pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) []const u8 {
	532	return self.buffer[token.start..token.end];
	533	}
	534
	535	fn checkLiteralCharacter(self: &Tokenizer) void {
	536	if (self.pending_invalid_token != null) return;
	537	const invalid_length = self.getInvalidCharacterLength();
	538	if (invalid_length == 0) return;
	539	self.pending_invalid_token = Token {
	540	.id = Token.Id.Invalid,
	541	.start = self.index,
	542	.end = self.index + invalid_length,
	543	};
	544	}
	545
	546	fn getInvalidCharacterLength(self: &Tokenizer) u3 {
	547	const c0 = self.buffer[self.index];
	548	if (c0 < 0x80) {
	549	if (c0 < 0x20 or c0 == 0x7f) {
	550	// ascii control codes are never allowed
	551	// (note that \n was checked before we got here)
	552	return 1;
	553	}
	554	// looks fine to me.
	555	return 0;
	556	} else {
	557	// check utf8-encoded character.
	558	const length = std.unicode.utf8ByteSequenceLength(c0) catch return 1;
	559	// the last 3 bytes in the buffer are guaranteed to be '\n',
	560	// which means we don't need to do any bounds checking here.
	561	const bytes = self.buffer[self.index..self.index + length];
	562	switch (length) {
	563	2 => {
	564	const value = std.unicode.utf8Decode2(bytes) catch return length;
	565	if (value == 0x85) return length; // U+0085 (NEL)
	566	},
	567	3 => {
	568	const value = std.unicode.utf8Decode3(bytes) catch return length;
	569	if (value == 0x2028) return length; // U+2028 (LS)
	570	if (value == 0x2029) return length; // U+2029 (PS)
	571	},
	572	4 => {
	573	_ = std.unicode.utf8Decode4(bytes) catch return length;
	574	},
	575	else => unreachable,
	576	}
	577	self.index += length - 1;
	578	return 0;
	579	}
	580	}
	581	};
	582
	583
	584
	585	test "tokenizer" {
	586	testTokenize("test", []Token.Id {
	587	Token.Id.Keyword_test,
	588	});
	589	}
	590
	591	test "tokenizer - invalid token characters" {
	592	testTokenize("#", []Token.Id{Token.Id.Invalid});
	593	testTokenize("`", []Token.Id{Token.Id.Invalid});
	594	}
	595
	596	test "tokenizer - invalid literal/comment characters" {
	597	testTokenize("\"\x00\"", []Token.Id {
	598	Token.Id { .StringLiteral = Token.StrLitKind.Normal },
	599	Token.Id.Invalid,
	600	});
	601	testTokenize("//\x00", []Token.Id {
	602	Token.Id.Invalid,
	603	});
	604	testTokenize("//\x1f", []Token.Id {
	605	Token.Id.Invalid,
	606	});
	607	testTokenize("//\x7f", []Token.Id {
	608	Token.Id.Invalid,
	609	});
	610	}
	611
	612	test "tokenizer - utf8" {
	613	testTokenize("//\xc2\x80", []Token.Id{});
	614	testTokenize("//\xf4\x8f\xbf\xbf", []Token.Id{});
	615	}
	616
	617	test "tokenizer - invalid utf8" {
	618	testTokenize("//\x80", []Token.Id{Token.Id.Invalid});
	619	testTokenize("//\xbf", []Token.Id{Token.Id.Invalid});
	620	testTokenize("//\xf8", []Token.Id{Token.Id.Invalid});
	621	testTokenize("//\xff", []Token.Id{Token.Id.Invalid});
	622	testTokenize("//\xc2\xc0", []Token.Id{Token.Id.Invalid});
	623	testTokenize("//\xe0", []Token.Id{Token.Id.Invalid});
	624	testTokenize("//\xf0", []Token.Id{Token.Id.Invalid});
	625	testTokenize("//\xf0\x90\x80\xc0", []Token.Id{Token.Id.Invalid});
	626	}
	627
	628	test "tokenizer - illegal unicode codepoints" {
	629	// unicode newline characters.U+0085, U+2028, U+2029
	630	testTokenize("//\xc2\x84", []Token.Id{});
	631	testTokenize("//\xc2\x85", []Token.Id{Token.Id.Invalid});
	632	testTokenize("//\xc2\x86", []Token.Id{});
	633	testTokenize("//\xe2\x80\xa7", []Token.Id{});
	634	testTokenize("//\xe2\x80\xa8", []Token.Id{Token.Id.Invalid});
	635	testTokenize("//\xe2\x80\xa9", []Token.Id{Token.Id.Invalid});
	636	testTokenize("//\xe2\x80\xaa", []Token.Id{});
	637	}
	638
	639	fn testTokenize(source: []const u8, expected_tokens: []const Token.Id) void {
	640	// (test authors, just make this bigger if you need it)
	641	var padded_source: [0x100]u8 = undefined;
	642	std.mem.copy(u8, padded_source[0..source.len], source);
	643	padded_source[source.len + 0] = '\n';
	644	padded_source[source.len + 1] = '\n';
	645	padded_source[source.len + 2] = '\n';
	646
	647	var tokenizer = Tokenizer.init(padded_source[0..source.len + 3]);
	648	for (expected_tokens) \|expected_token_id\| {
	649	const token = tokenizer.next();
	650	std.debug.assert(@TagType(Token.Id)(token.id) == @TagType(Token.Id)(expected_token_id));
	651	switch (expected_token_id) {
	652	Token.Id.StringLiteral => \|expected_kind\| {
	653	std.debug.assert(expected_kind == switch (token.id) { Token.Id.StringLiteral => \|kind\| kind, else => unreachable });
	654	},
	655	else => {},
	656	}
	657	}
	658	std.debug.assert(tokenizer.next().id == Token.Id.Eof);
	659	}