authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-20 22:41:18-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-20 22:41:18-04:00
log84df1d4f3d0312553f5a3857ed67042319c20846
tree78cdf050acb81fa642629bfad9b0a5593865036e
parent44aeb383289bc80644e3c6aef2b7b2494149c80b

stage2 parser: elide memcpy of large initialization lists

throughput: 71.4 MiB/s => 72.9 MiB/s

2 files changed, 56 insertions(+), 4 deletions(-)

lib/std/zig/ast.zig+6
......@@ -18,6 +18,8 @@ pub const Tree = struct {
1818
1919 arena: std.heap.ArenaAllocator.State,
2020 gpa: *mem.Allocator,
21 /// This keeps track of slices of memory that must be freed on deinit.
22 owned_memory: [][]u8,
2123
2224 /// translate-c uses this to avoid having to emit correct newlines
2325 /// TODO get rid of this hack
......@@ -26,6 +28,10 @@ pub const Tree = struct {
2628 pub fn deinit(self: *Tree) void {
2729 self.gpa.free(self.tokens);
2830 self.gpa.free(self.errors);
31 for (self.owned_memory) |list| {
32 self.gpa.free(list);
33 }
34 self.gpa.free(self.owned_memory);
2935 self.arena.promote(self.gpa).deinit();
3036 }
3137
lib/std/zig/parse.zig+50-4
......@@ -10,6 +10,13 @@ const Token = std.zig.Token;
1010
1111pub const Error = error{ParseError} || Allocator.Error;
1212
13/// This is the maximum length of a list that will be copied into the ast.Tree
14/// arena when parsing. If the list is longer than this, the ast.Tree will have
15/// a reference to the memory allocated in the general purpose allocator, and
16/// will free it separately. Simply put, lists longer than this will elide the
17/// memcpy().
18const large_list_len = 512;
19
1320/// Result should be freed with tree.deinit() when there are
1421/// no more references to any of the tokens or nodes.
1522pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
......@@ -32,6 +39,11 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
3239 .tokens = tokens.items,
3340 .errors = .{},
3441 .tok_i = 0,
42 .owned_memory = .{},
43 };
44 defer parser.owned_memory.deinit(gpa);
45 errdefer for (parser.owned_memory.items) |list| {
46 gpa.free(list);
3547 };
3648 defer parser.errors.deinit(gpa);
3749 errdefer parser.arena.deinit();
......@@ -46,6 +58,7 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
4658 .source = source,
4759 .tokens = tokens.toOwnedSlice(),
4860 .errors = parser.errors.toOwnedSlice(gpa),
61 .owned_memory = parser.owned_memory.toOwnedSlice(gpa),
4962 .root_node = root_node,
5063 .arena = parser.arena.state,
5164 };
......@@ -60,6 +73,7 @@ const Parser = struct {
6073 tokens: []const Token,
6174 tok_i: TokenIndex,
6275 errors: std.ArrayListUnmanaged(AstError),
76 owned_memory: std.ArrayListUnmanaged([]u8),
6377
6478 /// Root <- skip ContainerMembers eof
6579 fn parseRoot(p: *Parser) Allocator.Error!*Node.Root {
......@@ -1307,11 +1321,19 @@ const Parser = struct {
13071321 const next = (try p.parseFieldInit()) orelse break;
13081322 try init_list.append(next);
13091323 }
1324
1325 const list = if (init_list.items.len > large_list_len) blk: {
1326 try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1);
1327 const list = init_list.toOwnedSlice();
1328 p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list));
1329 break :blk list;
1330 } else try p.arena.allocator.dupe(*Node, init_list.items);
1331
13101332 const node = try p.arena.allocator.create(Node.StructInitializer);
13111333 node.* = .{
13121334 .lhs = lhs,
13131335 .rtoken = try p.expectToken(.RBrace),
1314 .list = try p.arena.allocator.dupe(*Node, init_list.items),
1336 .list = list,
13151337 };
13161338 return &node.base;
13171339 }
......@@ -1322,11 +1344,19 @@ const Parser = struct {
13221344 const next = (try p.parseExpr()) orelse break;
13231345 try init_list.append(next);
13241346 }
1347
1348 const list = if (init_list.items.len > large_list_len) blk: {
1349 try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1);
1350 const list = init_list.toOwnedSlice();
1351 p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list));
1352 break :blk list;
1353 } else try p.arena.allocator.dupe(*Node, init_list.items);
1354
13251355 const node = try p.arena.allocator.create(Node.ArrayInitializer);
13261356 node.* = .{
13271357 .lhs = lhs,
13281358 .rtoken = try p.expectToken(.RBrace),
1329 .list = try p.arena.allocator.dupe(*Node, init_list.items),
1359 .list = list,
13301360 };
13311361 return &node.base;
13321362 }
......@@ -1355,11 +1385,19 @@ const Parser = struct {
13551385 const next = (try p.parseFieldInit()) orelse break;
13561386 try init_list.append(next);
13571387 }
1388
1389 const list = if (init_list.items.len > large_list_len) blk: {
1390 try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1);
1391 const list = init_list.toOwnedSlice();
1392 p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list));
1393 break :blk list;
1394 } else try p.arena.allocator.dupe(*Node, init_list.items);
1395
13581396 const node = try p.arena.allocator.create(Node.StructInitializerDot);
13591397 node.* = .{
13601398 .dot = dot,
13611399 .rtoken = try p.expectToken(.RBrace),
1362 .list = try p.arena.allocator.dupe(*Node, init_list.items),
1400 .list = list,
13631401 };
13641402 return &node.base;
13651403 }
......@@ -1370,11 +1408,19 @@ const Parser = struct {
13701408 const next = (try p.parseExpr()) orelse break;
13711409 try init_list.append(next);
13721410 }
1411
1412 const list = if (init_list.items.len > large_list_len) blk: {
1413 try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1);
1414 const list = init_list.toOwnedSlice();
1415 p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list));
1416 break :blk list;
1417 } else try p.arena.allocator.dupe(*Node, init_list.items);
1418
13731419 const node = try p.arena.allocator.create(Node.ArrayInitializerDot);
13741420 node.* = .{
13751421 .dot = dot,
13761422 .rtoken = try p.expectToken(.RBrace),
1377 .list = try p.arena.allocator.dupe(*Node, init_list.items),
1423 .list = list,
13781424 };
13791425 return &node.base;
13801426 }