authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-21 22:52:45-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-21 22:52:45-04:00
log19de25993654f3e089eaf46391612e8a7e406d7e
treec59768f81a84b306ee043457a849da78ef08f4c5
parent9377af934f5be92821f36b5d2255f086e5a84cda

stage2 parser: arrays and structs directly in memory after the node

Slightly reduced memory usage. Roughly the same API and perf.

3 files changed, 134 insertions(+), 26 deletions(-)

lib/std/zig/ast.zig+116-12
...@@ -2067,11 +2067,23 @@ pub const Node = struct {...@@ -2067,11 +2067,23 @@ pub const Node = struct {
2067 }2067 }
2068 };2068 };
20692069
2070 /// Elements occur directly in memory after ArrayInitializer.
2070 pub const ArrayInitializer = struct {2071 pub const ArrayInitializer = struct {
2071 base: Node = Node{ .id = .ArrayInitializer },2072 base: Node = Node{ .id = .ArrayInitializer },
2072 rtoken: TokenIndex,2073 rtoken: TokenIndex,
2074 list_len: NodeIndex,
2073 lhs: *Node,2075 lhs: *Node,
2074 list: []*Node,2076
2077 /// After this the caller must initialize the fields_and_decls list.
2078 pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*ArrayInitializer {
2079 const bytes = try allocator.alignedAlloc(u8, @alignOf(ArrayInitializer), sizeInBytes(list_len));
2080 return @ptrCast(*ArrayInitializer, bytes.ptr);
2081 }
2082
2083 pub fn free(self: *ArrayInitializer, allocator: *mem.Allocator) void {
2084 const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
2085 allocator.free(bytes);
2086 }
20752087
2076 pub fn iterate(self: *const ArrayInitializer) Node.Iterator {2088 pub fn iterate(self: *const ArrayInitializer) Node.Iterator {
2077 return .{ .parent_node = &self.base, .index = 0, .node = null };2089 return .{ .parent_node = &self.base, .index = 0, .node = null };
...@@ -2084,8 +2096,8 @@ pub const Node = struct {...@@ -2084,8 +2096,8 @@ pub const Node = struct {
2084 if (i < 1) return self.lhs;2096 if (i < 1) return self.lhs;
2085 i -= 1;2097 i -= 1;
20862098
2087 if (i < self.list.len) return self.list[i];2099 if (i < self.list_len) return self.listConst()[i];
2088 i -= self.list.len;2100 i -= self.list_len;
20892101
2090 return null;2102 return null;
2091 }2103 }
...@@ -2097,13 +2109,39 @@ pub const Node = struct {...@@ -2097,13 +2109,39 @@ pub const Node = struct {
2097 pub fn lastToken(self: *const ArrayInitializer) TokenIndex {2109 pub fn lastToken(self: *const ArrayInitializer) TokenIndex {
2098 return self.rtoken;2110 return self.rtoken;
2099 }2111 }
2112
2113 pub fn list(self: *ArrayInitializer) []*Node {
2114 const decls_start = @ptrCast([*]u8, self) + @sizeOf(ArrayInitializer);
2115 return @ptrCast([*]*Node, decls_start)[0..self.list_len];
2116 }
2117
2118 pub fn listConst(self: *const ArrayInitializer) []const *Node {
2119 const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ArrayInitializer);
2120 return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
2121 }
2122
2123 fn sizeInBytes(list_len: NodeIndex) usize {
2124 return @sizeOf(ArrayInitializer) + @sizeOf(*Node) * @as(usize, list_len);
2125 }
2100 };2126 };
21012127
2128 /// Elements occur directly in memory after ArrayInitializerDot.
2102 pub const ArrayInitializerDot = struct {2129 pub const ArrayInitializerDot = struct {
2103 base: Node = Node{ .id = .ArrayInitializerDot },2130 base: Node = Node{ .id = .ArrayInitializerDot },
2104 dot: TokenIndex,2131 dot: TokenIndex,
2105 rtoken: TokenIndex,2132 rtoken: TokenIndex,
2106 list: []*Node,2133 list_len: NodeIndex,
2134
2135 /// After this the caller must initialize the fields_and_decls list.
2136 pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*ArrayInitializerDot {
2137 const bytes = try allocator.alignedAlloc(u8, @alignOf(ArrayInitializerDot), sizeInBytes(list_len));
2138 return @ptrCast(*ArrayInitializerDot, bytes.ptr);
2139 }
2140
2141 pub fn free(self: *ArrayInitializerDot, allocator: *mem.Allocator) void {
2142 const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
2143 allocator.free(bytes);
2144 }
21072145
2108 pub fn iterate(self: *const ArrayInitializerDot) Node.Iterator {2146 pub fn iterate(self: *const ArrayInitializerDot) Node.Iterator {
2109 return .{ .parent_node = &self.base, .index = 0, .node = null };2147 return .{ .parent_node = &self.base, .index = 0, .node = null };
...@@ -2113,8 +2151,8 @@ pub const Node = struct {...@@ -2113,8 +2151,8 @@ pub const Node = struct {
2113 var i = it.index;2151 var i = it.index;
2114 it.index += 1;2152 it.index += 1;
21152153
2116 if (i < self.list.len) return self.list[i];2154 if (i < self.list_len) return self.listConst()[i];
2117 i -= self.list.len;2155 i -= self.list_len;
21182156
2119 return null;2157 return null;
2120 }2158 }
...@@ -2126,13 +2164,39 @@ pub const Node = struct {...@@ -2126,13 +2164,39 @@ pub const Node = struct {
2126 pub fn lastToken(self: *const ArrayInitializerDot) TokenIndex {2164 pub fn lastToken(self: *const ArrayInitializerDot) TokenIndex {
2127 return self.rtoken;2165 return self.rtoken;
2128 }2166 }
2167
2168 pub fn list(self: *ArrayInitializerDot) []*Node {
2169 const decls_start = @ptrCast([*]u8, self) + @sizeOf(ArrayInitializerDot);
2170 return @ptrCast([*]*Node, decls_start)[0..self.list_len];
2171 }
2172
2173 pub fn listConst(self: *const ArrayInitializerDot) []const *Node {
2174 const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ArrayInitializerDot);
2175 return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
2176 }
2177
2178 fn sizeInBytes(list_len: NodeIndex) usize {
2179 return @sizeOf(ArrayInitializerDot) + @sizeOf(*Node) * @as(usize, list_len);
2180 }
2129 };2181 };
21302182
2183 /// Elements occur directly in memory after StructInitializer.
2131 pub const StructInitializer = struct {2184 pub const StructInitializer = struct {
2132 base: Node = Node{ .id = .StructInitializer },2185 base: Node = Node{ .id = .StructInitializer },
2133 rtoken: TokenIndex,2186 rtoken: TokenIndex,
2187 list_len: NodeIndex,
2134 lhs: *Node,2188 lhs: *Node,
2135 list: []*Node,2189
2190 /// After this the caller must initialize the fields_and_decls list.
2191 pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*StructInitializer {
2192 const bytes = try allocator.alignedAlloc(u8, @alignOf(StructInitializer), sizeInBytes(list_len));
2193 return @ptrCast(*StructInitializer, bytes.ptr);
2194 }
2195
2196 pub fn free(self: *StructInitializer, allocator: *mem.Allocator) void {
2197 const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
2198 allocator.free(bytes);
2199 }
21362200
2137 pub fn iterate(self: *const StructInitializer) Node.Iterator {2201 pub fn iterate(self: *const StructInitializer) Node.Iterator {
2138 return .{ .parent_node = &self.base, .index = 0, .node = null };2202 return .{ .parent_node = &self.base, .index = 0, .node = null };
...@@ -2145,8 +2209,8 @@ pub const Node = struct {...@@ -2145,8 +2209,8 @@ pub const Node = struct {
2145 if (i < 1) return self.lhs;2209 if (i < 1) return self.lhs;
2146 i -= 1;2210 i -= 1;
21472211
2148 if (i < self.list.len) return self.list[i];2212 if (i < self.list_len) return self.listConst()[i];
2149 i -= self.list.len;2213 i -= self.list_len;
21502214
2151 return null;2215 return null;
2152 }2216 }
...@@ -2158,13 +2222,39 @@ pub const Node = struct {...@@ -2158,13 +2222,39 @@ pub const Node = struct {
2158 pub fn lastToken(self: *const StructInitializer) TokenIndex {2222 pub fn lastToken(self: *const StructInitializer) TokenIndex {
2159 return self.rtoken;2223 return self.rtoken;
2160 }2224 }
2225
2226 pub fn list(self: *StructInitializer) []*Node {
2227 const decls_start = @ptrCast([*]u8, self) + @sizeOf(StructInitializer);
2228 return @ptrCast([*]*Node, decls_start)[0..self.list_len];
2229 }
2230
2231 pub fn listConst(self: *const StructInitializer) []const *Node {
2232 const decls_start = @ptrCast([*]const u8, self) + @sizeOf(StructInitializer);
2233 return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
2234 }
2235
2236 fn sizeInBytes(list_len: NodeIndex) usize {
2237 return @sizeOf(StructInitializer) + @sizeOf(*Node) * @as(usize, list_len);
2238 }
2161 };2239 };
21622240
2241 /// Elements occur directly in memory after StructInitializerDot.
2163 pub const StructInitializerDot = struct {2242 pub const StructInitializerDot = struct {
2164 base: Node = Node{ .id = .StructInitializerDot },2243 base: Node = Node{ .id = .StructInitializerDot },
2165 dot: TokenIndex,2244 dot: TokenIndex,
2166 rtoken: TokenIndex,2245 rtoken: TokenIndex,
2167 list: []*Node,2246 list_len: NodeIndex,
2247
2248 /// After this the caller must initialize the fields_and_decls list.
2249 pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*StructInitializerDot {
2250 const bytes = try allocator.alignedAlloc(u8, @alignOf(StructInitializerDot), sizeInBytes(list_len));
2251 return @ptrCast(*StructInitializerDot, bytes.ptr);
2252 }
2253
2254 pub fn free(self: *StructInitializerDot, allocator: *mem.Allocator) void {
2255 const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)];
2256 allocator.free(bytes);
2257 }
21682258
2169 pub fn iterate(self: *const StructInitializerDot) Node.Iterator {2259 pub fn iterate(self: *const StructInitializerDot) Node.Iterator {
2170 return .{ .parent_node = &self.base, .index = 0, .node = null };2260 return .{ .parent_node = &self.base, .index = 0, .node = null };
...@@ -2174,8 +2264,8 @@ pub const Node = struct {...@@ -2174,8 +2264,8 @@ pub const Node = struct {
2174 var i = it.index;2264 var i = it.index;
2175 it.index += 1;2265 it.index += 1;
21762266
2177 if (i < self.list.len) return self.list[i];2267 if (i < self.list_len) return self.listConst()[i];
2178 i -= self.list.len;2268 i -= self.list_len;
21792269
2180 return null;2270 return null;
2181 }2271 }
...@@ -2187,6 +2277,20 @@ pub const Node = struct {...@@ -2187,6 +2277,20 @@ pub const Node = struct {
2187 pub fn lastToken(self: *const StructInitializerDot) TokenIndex {2277 pub fn lastToken(self: *const StructInitializerDot) TokenIndex {
2188 return self.rtoken;2278 return self.rtoken;
2189 }2279 }
2280
2281 pub fn list(self: *StructInitializerDot) []*Node {
2282 const decls_start = @ptrCast([*]u8, self) + @sizeOf(StructInitializerDot);
2283 return @ptrCast([*]*Node, decls_start)[0..self.list_len];
2284 }
2285
2286 pub fn listConst(self: *const StructInitializerDot) []const *Node {
2287 const decls_start = @ptrCast([*]const u8, self) + @sizeOf(StructInitializerDot);
2288 return @ptrCast([*]const *Node, decls_start)[0..self.list_len];
2289 }
2290
2291 fn sizeInBytes(list_len: NodeIndex) usize {
2292 return @sizeOf(StructInitializerDot) + @sizeOf(*Node) * @as(usize, list_len);
2293 }
2190 };2294 };
21912295
2192 /// Parameter nodes directly follow Call in memory.2296 /// Parameter nodes directly follow Call in memory.
lib/std/zig/parse.zig+14-10
...@@ -1318,12 +1318,13 @@ const Parser = struct {...@@ -1318,12 +1318,13 @@ const Parser = struct {
1318 const next = (try p.parseFieldInit()) orelse break;1318 const next = (try p.parseFieldInit()) orelse break;
1319 try init_list.append(next);1319 try init_list.append(next);
1320 }1320 }
1321 const node = try p.arena.allocator.create(Node.StructInitializer);1321 const node = try Node.StructInitializer.alloc(&p.arena.allocator, init_list.items.len);
1322 node.* = .{1322 node.* = .{
1323 .lhs = lhs,1323 .lhs = lhs,
1324 .rtoken = try p.expectToken(.RBrace),1324 .rtoken = try p.expectToken(.RBrace),
1325 .list = try p.arena.allocator.dupe(*Node, init_list.items),1325 .list_len = init_list.items.len,
1326 };1326 };
1327 std.mem.copy(*Node, node.list(), init_list.items);
1327 return &node.base;1328 return &node.base;
1328 }1329 }
13291330
...@@ -1333,12 +1334,13 @@ const Parser = struct {...@@ -1333,12 +1334,13 @@ const Parser = struct {
1333 const next = (try p.parseExpr()) orelse break;1334 const next = (try p.parseExpr()) orelse break;
1334 try init_list.append(next);1335 try init_list.append(next);
1335 }1336 }
1336 const node = try p.arena.allocator.create(Node.ArrayInitializer);1337 const node = try Node.ArrayInitializer.alloc(&p.arena.allocator, init_list.items.len);
1337 node.* = .{1338 node.* = .{
1338 .lhs = lhs,1339 .lhs = lhs,
1339 .rtoken = try p.expectToken(.RBrace),1340 .rtoken = try p.expectToken(.RBrace),
1340 .list = try p.arena.allocator.dupe(*Node, init_list.items),1341 .list_len = init_list.items.len,
1341 };1342 };
1343 std.mem.copy(*Node, node.list(), init_list.items);
1342 return &node.base;1344 return &node.base;
1343 }1345 }
13441346
...@@ -1346,7 +1348,7 @@ const Parser = struct {...@@ -1346,7 +1348,7 @@ const Parser = struct {
1346 node.* = .{1348 node.* = .{
1347 .lhs = lhs,1349 .lhs = lhs,
1348 .rtoken = try p.expectToken(.RBrace),1350 .rtoken = try p.expectToken(.RBrace),
1349 .list = &[0]*Node{},1351 .list_len = 0,
1350 };1352 };
1351 return &node.base;1353 return &node.base;
1352 }1354 }
...@@ -1366,12 +1368,13 @@ const Parser = struct {...@@ -1366,12 +1368,13 @@ const Parser = struct {
1366 const next = (try p.parseFieldInit()) orelse break;1368 const next = (try p.parseFieldInit()) orelse break;
1367 try init_list.append(next);1369 try init_list.append(next);
1368 }1370 }
1369 const node = try p.arena.allocator.create(Node.StructInitializerDot);1371 const node = try Node.StructInitializerDot.alloc(&p.arena.allocator, init_list.items.len);
1370 node.* = .{1372 node.* = .{
1371 .dot = dot,1373 .dot = dot,
1372 .rtoken = try p.expectToken(.RBrace),1374 .rtoken = try p.expectToken(.RBrace),
1373 .list = try p.arena.allocator.dupe(*Node, init_list.items),1375 .list_len = init_list.items.len,
1374 };1376 };
1377 std.mem.copy(*Node, node.list(), init_list.items);
1375 return &node.base;1378 return &node.base;
1376 }1379 }
13771380
...@@ -1381,12 +1384,13 @@ const Parser = struct {...@@ -1381,12 +1384,13 @@ const Parser = struct {
1381 const next = (try p.parseExpr()) orelse break;1384 const next = (try p.parseExpr()) orelse break;
1382 try init_list.append(next);1385 try init_list.append(next);
1383 }1386 }
1384 const node = try p.arena.allocator.create(Node.ArrayInitializerDot);1387 const node = try Node.ArrayInitializerDot.alloc(&p.arena.allocator, init_list.items.len);
1385 node.* = .{1388 node.* = .{
1386 .dot = dot,1389 .dot = dot,
1387 .rtoken = try p.expectToken(.RBrace),1390 .rtoken = try p.expectToken(.RBrace),
1388 .list = try p.arena.allocator.dupe(*Node, init_list.items),1391 .list_len = init_list.items.len,
1389 };1392 };
1393 std.mem.copy(*Node, node.list(), init_list.items);
1390 return &node.base;1394 return &node.base;
1391 }1395 }
13921396
...@@ -1394,7 +1398,7 @@ const Parser = struct {...@@ -1394,7 +1398,7 @@ const Parser = struct {
1394 node.* = .{1398 node.* = .{
1395 .dot = dot,1399 .dot = dot,
1396 .rtoken = try p.expectToken(.RBrace),1400 .rtoken = try p.expectToken(.RBrace),
1397 .list = &[0]*Node{},1401 .list_len = 0,
1398 };1402 };
1399 return &node.base;1403 return &node.base;
1400 }1404 }
lib/std/zig/render.zig+4-4
...@@ -620,13 +620,13 @@ fn renderExpression(...@@ -620,13 +620,13 @@ fn renderExpression(
620 .ArrayInitializerDot => blk: {620 .ArrayInitializerDot => blk: {
621 const casted = @fieldParentPtr(ast.Node.ArrayInitializerDot, "base", base);621 const casted = @fieldParentPtr(ast.Node.ArrayInitializerDot, "base", base);
622 rtoken = casted.rtoken;622 rtoken = casted.rtoken;
623 exprs = casted.list;623 exprs = casted.list();
624 break :blk .{ .dot = casted.dot };624 break :blk .{ .dot = casted.dot };
625 },625 },
626 .ArrayInitializer => blk: {626 .ArrayInitializer => blk: {
627 const casted = @fieldParentPtr(ast.Node.ArrayInitializer, "base", base);627 const casted = @fieldParentPtr(ast.Node.ArrayInitializer, "base", base);
628 rtoken = casted.rtoken;628 rtoken = casted.rtoken;
629 exprs = casted.list;629 exprs = casted.list();
630 break :blk .{ .node = casted.lhs };630 break :blk .{ .node = casted.lhs };
631 },631 },
632 else => unreachable,632 else => unreachable,
...@@ -784,13 +784,13 @@ fn renderExpression(...@@ -784,13 +784,13 @@ fn renderExpression(
784 .StructInitializerDot => blk: {784 .StructInitializerDot => blk: {
785 const casted = @fieldParentPtr(ast.Node.StructInitializerDot, "base", base);785 const casted = @fieldParentPtr(ast.Node.StructInitializerDot, "base", base);
786 rtoken = casted.rtoken;786 rtoken = casted.rtoken;
787 field_inits = casted.list;787 field_inits = casted.list();
788 break :blk .{ .dot = casted.dot };788 break :blk .{ .dot = casted.dot };
789 },789 },
790 .StructInitializer => blk: {790 .StructInitializer => blk: {
791 const casted = @fieldParentPtr(ast.Node.StructInitializer, "base", base);791 const casted = @fieldParentPtr(ast.Node.StructInitializer, "base", base);
792 rtoken = casted.rtoken;792 rtoken = casted.rtoken;
793 field_inits = casted.list;793 field_inits = casted.list();
794 break :blk .{ .node = casted.lhs };794 break :blk .{ .node = casted.lhs };
795 },795 },
796 else => unreachable,796 else => unreachable,