Commit 64c149ca19cc299485693acbda325eee51d37f28 | zig

author	Andrew Kelley <andrew@ziglang.org> 2020-05-20 16:13:02-04:00
committer	Andrew Kelley <andrew@ziglang.org> 2020-05-20 16:13:02-04:00
log	64c149ca19cc299485693acbda325eee51d37f28
tree	4722fb13d235122b1e69d78bc19a868c7c52609a
parent	b1bcdc96ca4451be95caa00fe71aafb578611e34

fields and decls: ArrayList appending, memcpy to ast arena

This makes fields and decl ast nodes part of the Root and ContainerDecl
AST nodes.

Surprisingly, it's a performance regression from using a singly-linked
list for these nodes:

throughput: 76.5 MiB/s => 69.4 MiB/s

However it has much better memory usage:

maxrss: 392 KB => 77 KB

It's also better API for consumers of the parser, since it is a flat
list in memory.

3 files changed, 137 insertions(+), 86 deletions(-)

lib
- std
  - zig
    ast.zig
    parse.zig
    render.zig

lib/std/zig/ast.zig+75-31

...	...	@@ -6,6 +6,7 @@ const mem = std.mem;
6	6	const Token = std.zig.Token;
7	7
8	8	pub const TokenIndex = usize;
	9	pub const NodeIndex = usize;
9	10
10	11	pub const Tree = struct {
11	12	/// Reference to externally-owned data.
...	...	@@ -616,40 +617,62 @@ pub const Node = struct {
616	617	}
617	618	}
618	619
	620	/// The decls data follows this struct in memory as an array of Node pointers.
619	621	pub const Root = struct {
620	622	base: Node = Node{ .id = .Root },
621		decls: DeclList,
622	623	eof_token: TokenIndex,
	624	decls_len: NodeIndex,
	625
	626	/// After this the caller must initialize the decls list.
	627	pub fn create(allocator: mem.Allocator, decls_len: NodeIndex, eof_token: TokenIndex) !Root {
	628	const bytes = try allocator.alignedAlloc(u8, @alignOf(Root), sizeInBytes(decls_len));
	629	const self = @ptrCast(*Root, bytes.ptr);
	630	self.* = .{
	631	.eof_token = eof_token,
	632	.decls_len = decls_len,
	633	};
	634	return self;
	635	}
623	636
624		pub const DeclList = LinkedList(*Node);
	637	pub fn destroy(self: Decl, allocator: mem.Allocator) void {
	638	const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.decls_len)];
	639	allocator.free(bytes);
	640	}
625	641
626	642	pub fn iterate(self: *const Root) Node.Iterator {
627		return .{ .parent_node = &self.base, .index = 0, .node = self.decls.first };
	643	return .{ .parent_node = &self.base, .index = 0, .node = null };
628	644	}
629	645
630	646	pub fn iterateNext(self: const Root, it: Node.Iterator) ?*Node {
631		const decl = it.node orelse return null;
632		it.node = decl.next;
633		return decl.data;
	647	var i = it.index;
	648	it.index += 1;
	649
	650	if (i < self.decls_len) return self.declsConst()[i];
	651	return null;
	652	}
	653
	654	pub fn decls(self: Root) []Node {
	655	const decls_start = @ptrCast([*]u8, self) + @sizeOf(Root);
	656	return @ptrCast([]Node, decls_start)[0..self.decls_len];
	657	}
	658
	659	pub fn declsConst(self: const Root) []const Node {
	660	const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Root);
	661	return @ptrCast([]const Node, decls_start)[0..self.decls_len];
634	662	}
635	663
636	664	pub fn firstToken(self: *const Root) TokenIndex {
637		if (self.decls.first) \|first\| {
638		return first.data.firstToken();
639		} else {
640		return self.eof_token;
641		}
	665	if (self.decls_len == 0) return self.eof_token;
	666	return self.declsConst()[0].firstToken();
642	667	}
643	668
644	669	pub fn lastToken(self: *const Root) TokenIndex {
645		if (self.decls.first) \|first\| {
646		var node = first;
647		while (true) {
648		node = node.next orelse return node.data.lastToken();
649		}
650		} else {
651		return self.eof_token;
652		}
	670	if (self.decls_len == 0) return self.eof_token;
	671	return self.declsConst()[self.decls_len - 1].lastToken();
	672	}
	673
	674	fn sizeInBytes(decls_len: NodeIndex) usize {
	675	return @sizeOf(Root) + @sizeOf(Node) @as(usize, decls_len);
653	676	}
654	677	};
655	678
...	...	@@ -777,14 +800,12 @@ pub const Node = struct {
777	800
778	801	pub const ContainerDecl = struct {
779	802	base: Node = Node{ .id = .ContainerDecl },
780		layout_token: ?TokenIndex,
781	803	kind_token: TokenIndex,
782		init_arg_expr: InitArg,
783		fields_and_decls: DeclList,
	804	layout_token: ?TokenIndex,
784	805	lbrace_token: TokenIndex,
785	806	rbrace_token: TokenIndex,
786
787		pub const DeclList = Root.DeclList;
	807	fields_and_decls_len: NodeIndex,
	808	init_arg_expr: InitArg,
788	809
789	810	pub const InitArg = union(enum) {
790	811	None,
...	...	@@ -792,8 +813,19 @@ pub const Node = struct {
792	813	Type: *Node,
793	814	};
794	815
	816	/// After this the caller must initialize the fields_and_decls list.
	817	pub fn alloc(allocator: mem.Allocator, fields_and_decls_len: NodeIndex) !ContainerDecl {
	818	const bytes = try allocator.alignedAlloc(u8, @alignOf(ContainerDecl), sizeInBytes(fields_and_decls_len));
	819	return @ptrCast(*ContainerDecl, bytes.ptr);
	820	}
	821
	822	pub fn free(self: Decl, allocator: mem.Allocator) void {
	823	const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.fields_and_decls_len)];
	824	allocator.free(bytes);
	825	}
	826
795	827	pub fn iterate(self: *const ContainerDecl) Node.Iterator {
796		return .{ .parent_node = &self.base, .index = 0, .node = self.fields_and_decls.first };
	828	return .{ .parent_node = &self.base, .index = 0, .node = null };
797	829	}
798	830
799	831	pub fn iterateNext(self: const ContainerDecl, it: Node.Iterator) ?*Node {
...	...	@@ -808,10 +840,8 @@ pub const Node = struct {
808	840	.None, .Enum => {},
809	841	}
810	842
811		if (it.node) \|child\| {
812		it.node = child.next;
813		return child.data;
814		}
	843	if (i < self.fields_and_decls_len) return self.fieldsAndDeclsConst()[i];
	844	i -= self.fields_and_decls_len;
815	845
816	846	return null;
817	847	}
...	...	@@ -826,6 +856,20 @@ pub const Node = struct {
826	856	pub fn lastToken(self: *const ContainerDecl) TokenIndex {
827	857	return self.rbrace_token;
828	858	}
	859
	860	pub fn fieldsAndDecls(self: ContainerDecl) []Node {
	861	const decls_start = @ptrCast([*]u8, self) + @sizeOf(ContainerDecl);
	862	return @ptrCast([]Node, decls_start)[0..self.fields_and_decls_len];
	863	}
	864
	865	pub fn fieldsAndDeclsConst(self: const ContainerDecl) []const Node {
	866	const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ContainerDecl);
	867	return @ptrCast([]const Node, decls_start)[0..self.fields_and_decls_len];
	868	}
	869
	870	fn sizeInBytes(fields_and_decls_len: NodeIndex) usize {
	871	return @sizeOf(ContainerDecl) + @sizeOf(Node) @as(usize, fields_and_decls_len);
	872	}
829	873	};
830	874
831	875	pub const ContainerField = struct {
...	...	@@ -1116,7 +1160,7 @@ pub const Node = struct {
1116	1160	statements: StatementList,
1117	1161	rbrace: TokenIndex,
1118	1162
1119		pub const StatementList = Root.DeclList;
	1163	pub const StatementList = LinkedList(*Node);
1120	1164
1121	1165	pub fn iterate(self: *const Block) Node.Iterator {
1122	1166	return .{ .parent_node = &self.base, .index = 0, .node = self.statements.first };
...	...	@@ -2615,7 +2659,7 @@ pub const Node = struct {
2615	2659	test "iterate" {
2616	2660	var root = Node.Root{
2617	2661	.base = Node{ .id = Node.Id.Root },
2618		.decls = Node.Root.DeclList.init(std.testing.allocator),
	2662	.decls_len = 0,
2619	2663	.eof_token = 0,
2620	2664	};
2621	2665	var base = &root.base;

lib/std/zig/parse.zig+41-31

...	...	@@ -63,16 +63,20 @@ const Parser = struct {
63	63
64	64	/// Root <- skip ContainerMembers eof
65	65	fn parseRoot(p: Parser) Allocator.Error!Node.Root {
66		const node = try p.arena.allocator.create(Node.Root);
67		node.* = .{
68		.decls = try parseContainerMembers(p, true),
69		// parseContainerMembers will try to skip as much
70		// invalid tokens as it can so this can only be the EOF
71		.eof_token = p.eatToken(.Eof).?,
72		};
	66	const decls = try parseContainerMembers(p, true);
	67	defer p.gpa.free(decls);
	68
	69	// parseContainerMembers will try to skip as much
	70	// invalid tokens as it can so this can only be the EOF
	71	const eof_token = p.eatToken(.Eof).?;
	72
	73	const node = try Node.Root.create(&p.arena.allocator, decls.len, eof_token);
	74	std.mem.copy(*ast.Node, node.decls(), decls);
	75
73	76	return node;
74	77	}
75	78
	79	/// Helper function for appending elements to a singly linked list.
76	80	fn llpush(
77	81	p: *Parser,
78	82	comptime T: type,
...	...	@@ -92,9 +96,9 @@ const Parser = struct {
92	96	/// / ContainerField COMMA ContainerMembers
93	97	/// / ContainerField
94	98	/// /
95		fn parseContainerMembers(p: *Parser, top_level: bool) !Node.Root.DeclList {
96		var list = Node.Root.DeclList{};
97		var list_it = &list.first;
	99	fn parseContainerMembers(p: Parser, top_level: bool) ![]ast.Node {
	100	var list = std.ArrayList(*ast.Node).init(p.gpa);
	101	defer list.deinit();
98	102
99	103	var field_state: union(enum) {
100	104	/// no fields have been seen
...	...	@@ -110,7 +114,7 @@ const Parser = struct {
110	114
111	115	while (true) {
112	116	if (try p.parseContainerDocComments()) \|node\| {
113		list_it = try p.llpush(*Node, list_it, node);
	117	try list.append(node);
114	118	continue;
115	119	}
116	120
...	...	@@ -127,7 +131,7 @@ const Parser = struct {
127	131	field_state = .{ .end = node.firstToken() };
128	132	}
129	133	node.cast(Node.TestDecl).?.doc_comments = doc_comments;
130		list_it = try p.llpush(*Node, list_it, node);
	134	try list.append(node);
131	135	continue;
132	136	}
133	137
...	...	@@ -142,7 +146,7 @@ const Parser = struct {
142	146	field_state = .{ .end = node.firstToken() };
143	147	}
144	148	node.cast(Node.Comptime).?.doc_comments = doc_comments;
145		list_it = try p.llpush(*Node, list_it, node);
	149	try list.append(node);
146	150	continue;
147	151	}
148	152
...	...	@@ -173,7 +177,7 @@ const Parser = struct {
173	177	},
174	178	else => unreachable,
175	179	}
176		list_it = try p.llpush(*Node, list_it, node);
	180	try list.append(node);
177	181	if (try p.parseAppendedDocComment(node.lastToken())) \|appended_comment\| {
178	182	switch (node.id) {
179	183	.FnProto => {},
...	...	@@ -215,7 +219,7 @@ const Parser = struct {
215	219
216	220	const field = node.cast(Node.ContainerField).?;
217	221	field.doc_comments = doc_comments;
218		list_it = try p.llpush(*Node, list_it, node);
	222	try list.append(node);
219	223	const comma = p.eatToken(.Comma) orelse {
220	224	// try to continue parsing
221	225	const index = p.tok_i;
...	...	@@ -275,7 +279,7 @@ const Parser = struct {
275	279	}
276	280	}
277	281
278		return list;
	282	return list.toOwnedSlice();
279	283	}
280	284
281	285	/// Attempts to find next container member by searching for certain tokens
...	...	@@ -2778,24 +2782,36 @@ const Parser = struct {
2778	2782
2779	2783	/// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE
2780	2784	fn parseContainerDeclAuto(p: Parser) !?Node {
2781		const node = (try p.parseContainerDeclType()) orelse return null;
	2785	const container_decl_type = (try p.parseContainerDeclType()) orelse return null;
2782	2786	const lbrace = try p.expectToken(.LBrace);
2783	2787	const members = try p.parseContainerMembers(false);
	2788	defer p.gpa.free(members);
2784	2789	const rbrace = try p.expectToken(.RBrace);
2785	2790
2786		const decl_type = node.cast(Node.ContainerDecl).?;
2787		decl_type.fields_and_decls = members;
2788		decl_type.lbrace_token = lbrace;
2789		decl_type.rbrace_token = rbrace;
2790
2791		return node;
	2791	const node = try Node.ContainerDecl.alloc(&p.arena.allocator, members.len);
	2792	node.* = .{
	2793	.layout_token = null,
	2794	.kind_token = container_decl_type.kind_token,
	2795	.init_arg_expr = container_decl_type.init_arg_expr,
	2796	.fields_and_decls_len = members.len,
	2797	.lbrace_token = lbrace,
	2798	.rbrace_token = rbrace,
	2799	};
	2800	std.mem.copy(*ast.Node, node.fieldsAndDecls(), members);
	2801	return &node.base;
2792	2802	}
2793	2803
	2804	/// Holds temporary data until we are ready to construct the full ContainerDecl AST node.
	2805	const ContainerDeclType = struct {
	2806	kind_token: TokenIndex,
	2807	init_arg_expr: ast.Node.ContainerDecl.InitArg,
	2808	};
	2809
2794	2810	/// ContainerDeclType
2795	2811	/// <- KEYWORD_struct
2796	2812	/// / KEYWORD_enum (LPAREN Expr RPAREN)?
2797	2813	/// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)?
2798		fn parseContainerDeclType(p: Parser) !?Node {
	2814	fn parseContainerDeclType(p: *Parser) !?ContainerDeclType {
2799	2815	const kind_token = p.nextToken();
2800	2816
2801	2817	const init_arg_expr = switch (kind_token.ptr.id) {
...	...	@@ -2838,16 +2854,10 @@ const Parser = struct {
2838	2854	},
2839	2855	};
2840	2856
2841		const node = try p.arena.allocator.create(Node.ContainerDecl);
2842		node.* = .{
2843		.layout_token = null,
	2857	return ContainerDeclType{
2844	2858	.kind_token = kind_token.index,
2845	2859	.init_arg_expr = init_arg_expr,
2846		.fields_and_decls = undefined, // set by caller
2847		.lbrace_token = undefined, // set by caller
2848		.rbrace_token = undefined, // set by caller
2849	2860	};
2850		return &node.base;
2851	2861	}
2852	2862
2853	2863	/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN

lib/std/zig/render.zig+21-24

...	...	@@ -79,9 +79,10 @@ fn renderRoot(
79	79	}
80	80
81	81	var start_col: usize = 0;
82		var it = tree.root_node.decls.first orelse return;
	82	var decl_i: ast.NodeIndex = 0;
	83	const root_decls = tree.root_node.decls();
83	84	while (true) {
84		var decl = it.data;
	85	var decl = root_decls[decl_i];
85	86
86	87	// This loop does the following:
87	88	//
...	...	@@ -130,14 +131,15 @@ fn renderRoot(
130	131	token_index = decl.firstToken();
131	132
132	133	while (!fmt_active) {
133		it = it.next orelse {
	134	decl_i += 1;
	135	if (decl_i >= root_decls.len) {
134	136	// If there's no next reformatted `decl`, just copy the
135	137	// remaining input tokens and bail out.
136	138	const start = tree.tokens[copy_start_token_index].start;
137	139	try copyFixingWhitespace(stream, tree.source[start..]);
138	140	return;
139		};
140		decl = it.data;
	141	}
	142	decl = root_decls[decl_i];
141	143	var decl_first_token_index = decl.firstToken();
142	144
143	145	while (token_index < decl_first_token_index) : (token_index += 1) {
...	...	@@ -178,8 +180,9 @@ fn renderRoot(
178	180	}
179	181
180	182	try renderTopLevelDecl(allocator, stream, tree, 0, &start_col, decl);
181		it = it.next orelse return;
182		try renderExtraNewline(tree, stream, &start_col, it.data);
	183	decl_i += 1;
	184	if (decl_i >= root_decls.len) return;
	185	try renderExtraNewline(tree, stream, &start_col, root_decls[decl_i]);
183	186	}
184	187	}
185	188
...	...	@@ -1189,7 +1192,7 @@ fn renderExpression(
1189	1192	},
1190	1193	}
1191	1194
1192		if (container_decl.fields_and_decls.first == null) {
	1195	if (container_decl.fields_and_decls_len == 0) {
1193	1196	try renderToken(tree, stream, container_decl.lbrace_token, indent + indent_delta, start_col, Space.None); // {
1194	1197	return renderToken(tree, stream, container_decl.rbrace_token, indent, start_col, space); // }
1195	1198	}
...	...	@@ -1203,18 +1206,18 @@ fn renderExpression(
1203	1206	break :blk tree.tokens[maybe_comma].id == .Comma;
1204	1207	};
1205	1208
	1209	const fields_and_decls = container_decl.fieldsAndDecls();
	1210
1206	1211	// Check if the first declaration and the { are on the same line
1207	1212	const src_has_newline = !tree.tokensOnSameLine(
1208	1213	container_decl.lbrace_token,
1209		container_decl.fields_and_decls.first.?.data.firstToken(),
	1214	fields_and_decls[0].firstToken(),
1210	1215	);
1211	1216
1212	1217	// We can only print all the elements in-line if all the
1213	1218	// declarations inside are fields
1214	1219	const src_has_only_fields = blk: {
1215		var it = container_decl.fields_and_decls.first;
1216		while (it) \|decl_node\| : (it = decl_node.next) {
1217		const decl = decl_node.data;
	1220	for (fields_and_decls) \|decl\| {
1218	1221	if (decl.id != .ContainerField) break :blk false;
1219	1222	}
1220	1223	break :blk true;
...	...	@@ -1225,14 +1228,12 @@ fn renderExpression(
1225	1228	const new_indent = indent + indent_delta;
1226	1229	try renderToken(tree, stream, container_decl.lbrace_token, new_indent, start_col, .Newline); // {
1227	1230
1228		var it = container_decl.fields_and_decls.first;
1229		while (it) \|decl_node\| : (it = decl_node.next) {
1230		const decl = decl_node.data;
	1231	for (fields_and_decls) \|decl, i\| {
1231	1232	try stream.writeByteNTimes(' ', new_indent);
1232	1233	try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl, .Newline);
1233	1234
1234		if (decl_node.next) \|next_decl\| {
1235		try renderExtraNewline(tree, stream, start_col, next_decl.data);
	1235	if (i + 1 < fields_and_decls.len) {
	1236	try renderExtraNewline(tree, stream, start_col, fields_and_decls[i + 1]);
1236	1237	}
1237	1238	}
1238	1239
...	...	@@ -1245,10 +1246,8 @@ fn renderExpression(
1245	1246	const new_indent = indent + indent_delta;
1246	1247	try stream.writeByteNTimes(' ', new_indent);
1247	1248
1248		var it = container_decl.fields_and_decls.first;
1249		while (it) \|decl_node\| : (it = decl_node.next) {
1250		const decl = decl_node.data;
1251		const space_after_decl: Space = if (decl_node.next == null) .Newline else .Space;
	1249	for (fields_and_decls) \|decl, i\| {
	1250	const space_after_decl: Space = if (i + 1 >= fields_and_decls.len) .Newline else .Space;
1252	1251	try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl, space_after_decl);
1253	1252	}
1254	1253
...	...	@@ -1257,9 +1256,7 @@ fn renderExpression(
1257	1256	// All the declarations on the same line
1258	1257	try renderToken(tree, stream, container_decl.lbrace_token, indent, start_col, .Space); // {
1259	1258
1260		var it = container_decl.fields_and_decls.first;
1261		while (it) \|decl_node\| : (it = decl_node.next) {
1262		const decl = decl_node.data;
	1259	for (fields_and_decls) \|decl\| {
1263	1260	try renderContainerDecl(allocator, stream, tree, indent, start_col, decl, .Space);
1264	1261	}
1265	1262	}