authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 15:02:02-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 15:02:02-04:00
log6d5ec184ab1a1b8c155714801f7d6cb7ea6f5b8f
treefdc134758797b6ddd388ed67ea1e3707719c2393
parent72716ecc3a1f236fc46d754fa7387317c40aade5

stage2 parser: heuristics to pre-allocate token arrays

throughput: 72.2 MiB/s => 75.3 MiB/s

1 files changed, 5 insertions(+), 2 deletions(-)

lib/std/zig/parse.zig+5-2
...@@ -14,13 +14,16 @@ pub const Error = error{ParseError} || Allocator.Error;...@@ -14,13 +14,16 @@ pub const Error = error{ParseError} || Allocator.Error;
14/// Result should be freed with tree.deinit() when there are14/// Result should be freed with tree.deinit() when there are
15/// no more references to any of the tokens or nodes.15/// no more references to any of the tokens or nodes.
16pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {16pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
17 // TODO optimization idea: ensureCapacity on the tokens list and
18 // then appendAssumeCapacity inside the loop.
19 var token_ids = std.ArrayList(Token.Id).init(gpa);17 var token_ids = std.ArrayList(Token.Id).init(gpa);
20 defer token_ids.deinit();18 defer token_ids.deinit();
21 var token_locs = std.ArrayList(Token.Loc).init(gpa);19 var token_locs = std.ArrayList(Token.Loc).init(gpa);
22 defer token_locs.deinit();20 defer token_locs.deinit();
2321
22 // Empirically, the zig std lib has an 8:1 ratio of source bytes to token count.
23 const estimated_token_count = source.len / 8;
24 try token_ids.ensureCapacity(estimated_token_count);
25 try token_locs.ensureCapacity(estimated_token_count);
26
24 var tokenizer = std.zig.Tokenizer.init(source);27 var tokenizer = std.zig.Tokenizer.init(source);
25 while (true) {28 while (true) {
26 const token = tokenizer.next();29 const token = tokenizer.next();