authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 15:12:23-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 15:12:23-04:00
log69ff89fd1207bc95adf7a349319973c6070ce540
treee918fe35d37fb0e35551e13b8d2b47aed5221b86
parent0ecdbdb3cb3d9558b5d2dbd928ead124d98c74ca

stage2 parser: heuristics to pre-allocate token arrays

throughput: 72.2 MiB/s => 75.3 MiB/s I also tried the idea from the deleted comment in this commit and it made the throughput worse.

1 files changed, 5 insertions(+), 2 deletions(-)

lib/std/zig/parse.zig+5-2
......@@ -14,13 +14,16 @@ pub const Error = error{ParseError} || Allocator.Error;
1414/// Result should be freed with tree.deinit() when there are
1515/// no more references to any of the tokens or nodes.
1616pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
17 // TODO optimization idea: ensureCapacity on the tokens list and
18 // then appendAssumeCapacity inside the loop.
1917 var token_ids = std.ArrayList(Token.Id).init(gpa);
2018 defer token_ids.deinit();
2119 var token_locs = std.ArrayList(Token.Loc).init(gpa);
2220 defer token_locs.deinit();
2321
22 // Empirically, the zig std lib has an 8:1 ratio of source bytes to token count.
23 const estimated_token_count = source.len / 8;
24 try token_ids.ensureCapacity(estimated_token_count);
25 try token_locs.ensureCapacity(estimated_token_count);
26
2427 var tokenizer = std.zig.Tokenizer.init(source);
2528 while (true) {
2629 const token = tokenizer.next();