authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-26 14:07:47-04:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-26 18:59:49-04:00
logba41a9d5d7b761ad8e3ad0f1b863c7f6457598a6
treea51c4089cb46a2af176302325aa1975f90d1fa3c
parentb6e1670d2ba1e7724878ee2b7176077da5cb7c35

different strategy for tokenizing keywords

throughput: 279 MiB/s => 347 MiB/s

1 files changed, 40 insertions(+), 9 deletions(-)

lib/std/zig/tokenizer.zig+40-9
...@@ -13,14 +13,11 @@ pub const Token = struct {...@@ -13,14 +13,11 @@ pub const Token = struct {
13 pub const Keyword = struct {13 pub const Keyword = struct {
14 bytes: []const u8,14 bytes: []const u8,
15 id: Id,15 id: Id,
16 hash: u32,
1716
18 fn init(bytes: []const u8, id: Id) Keyword {17 fn init(bytes: []const u8, id: Id) Keyword {
19 @setEvalBranchQuota(2000);
20 return .{18 return .{
21 .bytes = bytes,19 .bytes = bytes,
22 .id = id,20 .id = id,
23 .hash = std.hash_map.hashString(bytes),
24 };21 };
25 }22 }
26 };23 };
...@@ -79,15 +76,49 @@ pub const Token = struct {...@@ -79,15 +76,49 @@ pub const Token = struct {
79 Keyword.init("while", .Keyword_while),76 Keyword.init("while", .Keyword_while),
80 };77 };
8178
82 // TODO perfect hash at comptime
83 pub fn getKeyword(bytes: []const u8) ?Id {79 pub fn getKeyword(bytes: []const u8) ?Id {
84 var hash = std.hash_map.hashString(bytes);80 const precomputed = comptime blk: {
85 for (keywords) |kw| {81 @setEvalBranchQuota(2000);
86 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {82 var sorted_keywords = keywords;
87 return kw.id;83 const lenAsc = (struct {
84 fn lenAsc(a: Keyword, b: Keyword) bool {
85 return a.bytes.len < b.bytes.len;
86 }
87 }).lenAsc;
88 std.sort.sort(Keyword, &sorted_keywords, lenAsc);
89 const min_len = sorted_keywords[0].bytes.len;
90 const max_len = sorted_keywords[sorted_keywords.len - 1].bytes.len;
91 var len_indexes: [max_len + 1]usize = undefined;
92 var len: usize = 0;
93 var kw_i: usize = 0;
94 while (len <= max_len) : (len += 1) {
95 // find the first keyword len == len
96 while (len > sorted_keywords[kw_i].bytes.len) {
97 kw_i += 1;
98 }
99 len_indexes[len] = kw_i;
88 }100 }
101 break :blk .{
102 .min_len = min_len,
103 .max_len = max_len,
104 .sorted_keywords = sorted_keywords,
105 .len_indexes = len_indexes,
106 };
107 };
108 if (bytes.len < precomputed.min_len or bytes.len > precomputed.max_len)
109 return null;
110
111 var i = precomputed.len_indexes[bytes.len];
112 while (true) {
113 const kw = precomputed.sorted_keywords[i];
114 if (kw.bytes.len != bytes.len)
115 return null;
116 if (mem.eql(u8, kw.bytes, bytes))
117 return kw.id;
118 i += 1;
119 if (i >= precomputed.sorted_keywords.len)
120 return null;
89 }121 }
90 return null;
91 }122 }
92123
93 pub const Id = enum {124 pub const Id = enum {