authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 15:12:23-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-25 15:12:23-04:00
log69ff89fd1207bc95adf7a349319973c6070ce540
treee918fe35d37fb0e35551e13b8d2b47aed5221b86
parent0ecdbdb3cb3d9558b5d2dbd928ead124d98c74ca

stage2 parser: heuristics to pre-allocate token arrays

throughput: 72.2 MiB/s => 75.3 MiB/s I also tried the idea from the deleted comment in this commit and it made the throughput worse.

1 files changed, 5 insertions(+), 2 deletions(-)

lib/std/zig/parse.zig+5-2
...@@ -14,13 +14,16 @@ pub const Error = error{ParseError} || Allocator.Error;...@@ -14,13 +14,16 @@ pub const Error = error{ParseError} || Allocator.Error;
14/// Result should be freed with tree.deinit() when there are14/// Result should be freed with tree.deinit() when there are
15/// no more references to any of the tokens or nodes.15/// no more references to any of the tokens or nodes.
16pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {16pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree {
17 // TODO optimization idea: ensureCapacity on the tokens list and
18 // then appendAssumeCapacity inside the loop.
19 var token_ids = std.ArrayList(Token.Id).init(gpa);17 var token_ids = std.ArrayList(Token.Id).init(gpa);
20 defer token_ids.deinit();18 defer token_ids.deinit();
21 var token_locs = std.ArrayList(Token.Loc).init(gpa);19 var token_locs = std.ArrayList(Token.Loc).init(gpa);
22 defer token_locs.deinit();20 defer token_locs.deinit();
2321
22 // Empirically, the zig std lib has an 8:1 ratio of source bytes to token count.
23 const estimated_token_count = source.len / 8;
24 try token_ids.ensureCapacity(estimated_token_count);
25 try token_locs.ensureCapacity(estimated_token_count);
26
24 var tokenizer = std.zig.Tokenizer.init(source);27 var tokenizer = std.zig.Tokenizer.init(source);
25 while (true) {28 while (true) {
26 const token = tokenizer.next();29 const token = tokenizer.next();