authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-05-28 23:06:40-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-05-28 23:06:40-04:00
log1e0de896b8343e82a46a23fc287eab4fce06b423
treeefd7d52dc291600651bbac4e481d3666e50a26f3
parent6e89692d81bd5d50a634c32a1588a7c505860f32
parent3cac0a5614500ad90fbc3913ae4dc6aa56f2d595
signature Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #5452 from squeek502/comptime-string-map

Add std.ComptimeStringMap based on the tokenizer optimization in #5442

5 files changed, 324 insertions(+), 178 deletions(-)

lib/std/c/tokenizer.zig+63-67
......@@ -277,83 +277,79 @@ pub const Token = struct {
277277 };
278278
279279 // TODO extensions
280 pub const keywords = [_]Keyword{
281 Keyword.init("auto", .Keyword_auto),
282 Keyword.init("break", .Keyword_break),
283 Keyword.init("case", .Keyword_case),
284 Keyword.init("char", .Keyword_char),
285 Keyword.init("const", .Keyword_const),
286 Keyword.init("continue", .Keyword_continue),
287 Keyword.init("default", .Keyword_default),
288 Keyword.init("do", .Keyword_do),
289 Keyword.init("double", .Keyword_double),
290 Keyword.init("else", .Keyword_else),
291 Keyword.init("enum", .Keyword_enum),
292 Keyword.init("extern", .Keyword_extern),
293 Keyword.init("float", .Keyword_float),
294 Keyword.init("for", .Keyword_for),
295 Keyword.init("goto", .Keyword_goto),
296 Keyword.init("if", .Keyword_if),
297 Keyword.init("int", .Keyword_int),
298 Keyword.init("long", .Keyword_long),
299 Keyword.init("register", .Keyword_register),
300 Keyword.init("return", .Keyword_return),
301 Keyword.init("short", .Keyword_short),
302 Keyword.init("signed", .Keyword_signed),
303 Keyword.init("sizeof", .Keyword_sizeof),
304 Keyword.init("static", .Keyword_static),
305 Keyword.init("struct", .Keyword_struct),
306 Keyword.init("switch", .Keyword_switch),
307 Keyword.init("typedef", .Keyword_typedef),
308 Keyword.init("union", .Keyword_union),
309 Keyword.init("unsigned", .Keyword_unsigned),
310 Keyword.init("void", .Keyword_void),
311 Keyword.init("volatile", .Keyword_volatile),
312 Keyword.init("while", .Keyword_while),
280 pub const keywords = std.ComptimeStringMap(Id, .{
281 .{"auto", .Keyword_auto},
282 .{"break", .Keyword_break},
283 .{"case", .Keyword_case},
284 .{"char", .Keyword_char},
285 .{"const", .Keyword_const},
286 .{"continue", .Keyword_continue},
287 .{"default", .Keyword_default},
288 .{"do", .Keyword_do},
289 .{"double", .Keyword_double},
290 .{"else", .Keyword_else},
291 .{"enum", .Keyword_enum},
292 .{"extern", .Keyword_extern},
293 .{"float", .Keyword_float},
294 .{"for", .Keyword_for},
295 .{"goto", .Keyword_goto},
296 .{"if", .Keyword_if},
297 .{"int", .Keyword_int},
298 .{"long", .Keyword_long},
299 .{"register", .Keyword_register},
300 .{"return", .Keyword_return},
301 .{"short", .Keyword_short},
302 .{"signed", .Keyword_signed},
303 .{"sizeof", .Keyword_sizeof},
304 .{"static", .Keyword_static},
305 .{"struct", .Keyword_struct},
306 .{"switch", .Keyword_switch},
307 .{"typedef", .Keyword_typedef},
308 .{"union", .Keyword_union},
309 .{"unsigned", .Keyword_unsigned},
310 .{"void", .Keyword_void},
311 .{"volatile", .Keyword_volatile},
312 .{"while", .Keyword_while},
313313
314314 // ISO C99
315 Keyword.init("_Bool", .Keyword_bool),
316 Keyword.init("_Complex", .Keyword_complex),
317 Keyword.init("_Imaginary", .Keyword_imaginary),
318 Keyword.init("inline", .Keyword_inline),
319 Keyword.init("restrict", .Keyword_restrict),
315 .{"_Bool", .Keyword_bool},
316 .{"_Complex", .Keyword_complex},
317 .{"_Imaginary", .Keyword_imaginary},
318 .{"inline", .Keyword_inline},
319 .{"restrict", .Keyword_restrict},
320320
321321 // ISO C11
322 Keyword.init("_Alignas", .Keyword_alignas),
323 Keyword.init("_Alignof", .Keyword_alignof),
324 Keyword.init("_Atomic", .Keyword_atomic),
325 Keyword.init("_Generic", .Keyword_generic),
326 Keyword.init("_Noreturn", .Keyword_noreturn),
327 Keyword.init("_Static_assert", .Keyword_static_assert),
328 Keyword.init("_Thread_local", .Keyword_thread_local),
322 .{"_Alignas", .Keyword_alignas},
323 .{"_Alignof", .Keyword_alignof},
324 .{"_Atomic", .Keyword_atomic},
325 .{"_Generic", .Keyword_generic},
326 .{"_Noreturn", .Keyword_noreturn},
327 .{"_Static_assert", .Keyword_static_assert},
328 .{"_Thread_local", .Keyword_thread_local},
329329
330330 // Preprocessor directives
331 Keyword.init("include", .Keyword_include),
332 Keyword.init("define", .Keyword_define),
333 Keyword.init("ifdef", .Keyword_ifdef),
334 Keyword.init("ifndef", .Keyword_ifndef),
335 Keyword.init("error", .Keyword_error),
336 Keyword.init("pragma", .Keyword_pragma),
337 };
331 .{"include", .Keyword_include},
332 .{"define", .Keyword_define},
333 .{"ifdef", .Keyword_ifdef},
334 .{"ifndef", .Keyword_ifndef},
335 .{"error", .Keyword_error},
336 .{"pragma", .Keyword_pragma},
337 });
338338
339 // TODO perfect hash at comptime
340339 // TODO do this in the preprocessor
341340 pub fn getKeyword(bytes: []const u8, pp_directive: bool) ?Id {
342 var hash = std.hash_map.hashString(bytes);
343 for (keywords) |kw| {
344 if (kw.hash == hash and mem.eql(u8, kw.bytes, bytes)) {
345 switch (kw.id) {
346 .Keyword_include,
347 .Keyword_define,
348 .Keyword_ifdef,
349 .Keyword_ifndef,
350 .Keyword_error,
351 .Keyword_pragma,
352 => if (!pp_directive) return null,
353 else => {},
354 }
355 return kw.id;
341 if (keywords.get(bytes)) |id| {
342 switch (id) {
343 .Keyword_include,
344 .Keyword_define,
345 .Keyword_ifdef,
346 .Keyword_ifndef,
347 .Keyword_error,
348 .Keyword_pragma,
349 => if (!pp_directive) return null,
350 else => {},
356351 }
352 return id;
357353 }
358354 return null;
359355 }
lib/std/comptime_string_map.zig created+177
......@@ -0,0 +1,177 @@
1const std = @import("std.zig");
2const mem = std.mem;
3
4/// Like ComptimeStringHashMap but optimized for small sets of disparate string keys.
5/// Works by separating the keys by length at comptime and only checking strings of
6/// equal length at runtime.
7///
8/// `kvs` expects a list literal containing list literals or an array/slice of structs
9/// where `.@"0"` is the `[]const u8` key and `.@"1"` is the associated value of type `V`.
10/// TODO: https://github.com/ziglang/zig/issues/4335
11pub fn ComptimeStringMap(comptime V: type, comptime kvs: var) type {
12 const precomputed = comptime blk: {
13 @setEvalBranchQuota(2000);
14 const KV = struct {
15 key: []const u8,
16 value: V,
17 };
18 var sorted_kvs: [kvs.len]KV = undefined;
19 const lenAsc = (struct {
20 fn lenAsc(a: KV, b: KV) bool {
21 return a.key.len < b.key.len;
22 }
23 }).lenAsc;
24 for (kvs) |kv, i| {
25 if (V != void) {
26 sorted_kvs[i] = .{.key = kv.@"0", .value = kv.@"1"};
27 } else {
28 sorted_kvs[i] = .{.key = kv.@"0", .value = {}};
29 }
30 }
31 std.sort.sort(KV, &sorted_kvs, lenAsc);
32 const min_len = sorted_kvs[0].key.len;
33 const max_len = sorted_kvs[sorted_kvs.len - 1].key.len;
34 var len_indexes: [max_len + 1]usize = undefined;
35 var len: usize = 0;
36 var i: usize = 0;
37 while (len <= max_len) : (len += 1) {
38 // find the first keyword len == len
39 while (len > sorted_kvs[i].key.len) {
40 i += 1;
41 }
42 len_indexes[len] = i;
43 }
44 break :blk .{
45 .min_len = min_len,
46 .max_len = max_len,
47 .sorted_kvs = sorted_kvs,
48 .len_indexes = len_indexes,
49 };
50 };
51
52 return struct {
53 pub fn has(str: []const u8) bool {
54 return get(str) != null;
55 }
56
57 pub fn get(str: []const u8) ?V {
58 if (str.len < precomputed.min_len or str.len > precomputed.max_len)
59 return null;
60
61 var i = precomputed.len_indexes[str.len];
62 while (true) {
63 const kv = precomputed.sorted_kvs[i];
64 if (kv.key.len != str.len)
65 return null;
66 if (mem.eql(u8, kv.key, str))
67 return kv.value;
68 i += 1;
69 if (i >= precomputed.sorted_kvs.len)
70 return null;
71 }
72 }
73 };
74}
75
76const TestEnum = enum {
77 A,
78 B,
79 C,
80 D,
81 E,
82};
83
84test "ComptimeStringMap list literal of list literals" {
85 const map = ComptimeStringMap(TestEnum, .{
86 .{"these", .D},
87 .{"have", .A},
88 .{"nothing", .B},
89 .{"incommon", .C},
90 .{"samelen", .E},
91 });
92
93 testMap(map);
94}
95
96test "ComptimeStringMap array of structs" {
97 const KV = struct {
98 @"0": []const u8,
99 @"1": TestEnum,
100 };
101 const map = ComptimeStringMap(TestEnum, [_]KV{
102 .{.@"0" = "these", .@"1" = .D},
103 .{.@"0" = "have", .@"1" = .A},
104 .{.@"0" = "nothing", .@"1" = .B},
105 .{.@"0" = "incommon", .@"1" = .C},
106 .{.@"0" = "samelen", .@"1" = .E},
107 });
108
109 testMap(map);
110}
111
112test "ComptimeStringMap slice of structs" {
113 const KV = struct {
114 @"0": []const u8,
115 @"1": TestEnum,
116 };
117 const slice: []const KV = &[_]KV{
118 .{.@"0" = "these", .@"1" = .D},
119 .{.@"0" = "have", .@"1" = .A},
120 .{.@"0" = "nothing", .@"1" = .B},
121 .{.@"0" = "incommon", .@"1" = .C},
122 .{.@"0" = "samelen", .@"1" = .E},
123 };
124 const map = ComptimeStringMap(TestEnum, slice);
125
126 testMap(map);
127}
128
129fn testMap(comptime map: var) void {
130 std.testing.expectEqual(TestEnum.A, map.get("have").?);
131 std.testing.expectEqual(TestEnum.B, map.get("nothing").?);
132 std.testing.expect(null == map.get("missing"));
133 std.testing.expectEqual(TestEnum.D, map.get("these").?);
134 std.testing.expectEqual(TestEnum.E, map.get("samelen").?);
135
136 std.testing.expect(!map.has("missing"));
137 std.testing.expect(map.has("these"));
138}
139
140test "ComptimeStringMap void value type, slice of structs" {
141 const KV = struct {
142 @"0": []const u8,
143 };
144 const slice: []const KV = &[_]KV{
145 .{.@"0" = "these"},
146 .{.@"0" = "have"},
147 .{.@"0" = "nothing"},
148 .{.@"0" = "incommon"},
149 .{.@"0" = "samelen"},
150 };
151 const map = ComptimeStringMap(void, slice);
152
153 testSet(map);
154}
155
156test "ComptimeStringMap void value type, list literal of list literals" {
157 const map = ComptimeStringMap(void, .{
158 .{"these"},
159 .{"have"},
160 .{"nothing"},
161 .{"incommon"},
162 .{"samelen"},
163 });
164
165 testSet(map);
166}
167
168fn testSet(comptime map: var) void {
169 std.testing.expectEqual({}, map.get("have").?);
170 std.testing.expectEqual({}, map.get("nothing").?);
171 std.testing.expect(null == map.get("missing"));
172 std.testing.expectEqual({}, map.get("these").?);
173 std.testing.expectEqual({}, map.get("samelen").?);
174
175 std.testing.expect(!map.has("missing"));
176 std.testing.expect(map.has("these"));
177}
lib/std/meta.zig+29-4
......@@ -53,12 +53,37 @@ test "std.meta.tagName" {
5353}
5454
5555pub fn stringToEnum(comptime T: type, str: []const u8) ?T {
56 inline for (@typeInfo(T).Enum.fields) |enumField| {
57 if (mem.eql(u8, str, enumField.name)) {
58 return @field(T, enumField.name);
56 // Using ComptimeStringMap here is more performant, but it will start to take too
57 // long to compile if the enum is large enough, due to the current limits of comptime
58 // performance when doing things like constructing lookup maps at comptime.
59 // TODO The '100' here is arbitrary and should be increased when possible:
60 // - https://github.com/ziglang/zig/issues/4055
61 // - https://github.com/ziglang/zig/issues/3863
62 if (@typeInfo(T).Enum.fields.len <= 100) {
63 const kvs = comptime build_kvs: {
64 // In order to generate an array of structs that play nice with anonymous
65 // list literals, we need to give them "0" and "1" field names.
66 // TODO https://github.com/ziglang/zig/issues/4335
67 const EnumKV = struct {
68 @"0": []const u8,
69 @"1": T,
70 };
71 var kvs_array: [@typeInfo(T).Enum.fields.len]EnumKV = undefined;
72 inline for (@typeInfo(T).Enum.fields) |enumField, i| {
73 kvs_array[i] = .{ .@"0" = enumField.name, .@"1" = @field(T, enumField.name) };
74 }
75 break :build_kvs kvs_array[0..];
76 };
77 const map = std.ComptimeStringMap(T, kvs);
78 return map.get(str);
79 } else {
80 inline for (@typeInfo(T).Enum.fields) |enumField| {
81 if (mem.eql(u8, str, enumField.name)) {
82 return @field(T, enumField.name);
83 }
5984 }
85 return null;
6086 }
61 return null;
6287}
6388
6489test "std.meta.stringToEnum" {
lib/std/std.zig+1
......@@ -8,6 +8,7 @@ pub const BloomFilter = @import("bloom_filter.zig").BloomFilter;
88pub const BufMap = @import("buf_map.zig").BufMap;
99pub const BufSet = @import("buf_set.zig").BufSet;
1010pub const ChildProcess = @import("child_process.zig").ChildProcess;
11pub const ComptimeStringMap = @import("comptime_string_map.zig").ComptimeStringMap;
1112pub const DynLib = @import("dynamic_library.zig").DynLib;
1213pub const HashMap = @import("hash_map.zig").HashMap;
1314pub const Mutex = @import("mutex.zig").Mutex;
lib/std/zig/tokenizer.zig+54-107
......@@ -10,115 +10,62 @@ pub const Token = struct {
1010 end: usize,
1111 };
1212
13 pub const Keyword = struct {
14 bytes: []const u8,
15 id: Id,
16
17 fn init(bytes: []const u8, id: Id) Keyword {
18 return .{
19 .bytes = bytes,
20 .id = id,
21 };
22 }
23 };
24
25 pub const keywords = [_]Keyword{
26 Keyword.init("align", .Keyword_align),
27 Keyword.init("allowzero", .Keyword_allowzero),
28 Keyword.init("and", .Keyword_and),
29 Keyword.init("anyframe", .Keyword_anyframe),
30 Keyword.init("asm", .Keyword_asm),
31 Keyword.init("async", .Keyword_async),
32 Keyword.init("await", .Keyword_await),
33 Keyword.init("break", .Keyword_break),
34 Keyword.init("callconv", .Keyword_callconv),
35 Keyword.init("catch", .Keyword_catch),
36 Keyword.init("comptime", .Keyword_comptime),
37 Keyword.init("const", .Keyword_const),
38 Keyword.init("continue", .Keyword_continue),
39 Keyword.init("defer", .Keyword_defer),
40 Keyword.init("else", .Keyword_else),
41 Keyword.init("enum", .Keyword_enum),
42 Keyword.init("errdefer", .Keyword_errdefer),
43 Keyword.init("error", .Keyword_error),
44 Keyword.init("export", .Keyword_export),
45 Keyword.init("extern", .Keyword_extern),
46 Keyword.init("false", .Keyword_false),
47 Keyword.init("fn", .Keyword_fn),
48 Keyword.init("for", .Keyword_for),
49 Keyword.init("if", .Keyword_if),
50 Keyword.init("inline", .Keyword_inline),
51 Keyword.init("noalias", .Keyword_noalias),
52 Keyword.init("noasync", .Keyword_nosuspend), // TODO: remove this
53 Keyword.init("noinline", .Keyword_noinline),
54 Keyword.init("nosuspend", .Keyword_nosuspend),
55 Keyword.init("null", .Keyword_null),
56 Keyword.init("or", .Keyword_or),
57 Keyword.init("orelse", .Keyword_orelse),
58 Keyword.init("packed", .Keyword_packed),
59 Keyword.init("pub", .Keyword_pub),
60 Keyword.init("resume", .Keyword_resume),
61 Keyword.init("return", .Keyword_return),
62 Keyword.init("linksection", .Keyword_linksection),
63 Keyword.init("struct", .Keyword_struct),
64 Keyword.init("suspend", .Keyword_suspend),
65 Keyword.init("switch", .Keyword_switch),
66 Keyword.init("test", .Keyword_test),
67 Keyword.init("threadlocal", .Keyword_threadlocal),
68 Keyword.init("true", .Keyword_true),
69 Keyword.init("try", .Keyword_try),
70 Keyword.init("undefined", .Keyword_undefined),
71 Keyword.init("union", .Keyword_union),
72 Keyword.init("unreachable", .Keyword_unreachable),
73 Keyword.init("usingnamespace", .Keyword_usingnamespace),
74 Keyword.init("var", .Keyword_var),
75 Keyword.init("volatile", .Keyword_volatile),
76 Keyword.init("while", .Keyword_while),
77 };
13 pub const keywords = std.ComptimeStringMap(Id, .{
14 .{"align", .Keyword_align},
15 .{"allowzero", .Keyword_allowzero},
16 .{"and", .Keyword_and},
17 .{"anyframe", .Keyword_anyframe},
18 .{"asm", .Keyword_asm},
19 .{"async", .Keyword_async},
20 .{"await", .Keyword_await},
21 .{"break", .Keyword_break},
22 .{"callconv", .Keyword_callconv},
23 .{"catch", .Keyword_catch},
24 .{"comptime", .Keyword_comptime},
25 .{"const", .Keyword_const},
26 .{"continue", .Keyword_continue},
27 .{"defer", .Keyword_defer},
28 .{"else", .Keyword_else},
29 .{"enum", .Keyword_enum},
30 .{"errdefer", .Keyword_errdefer},
31 .{"error", .Keyword_error},
32 .{"export", .Keyword_export},
33 .{"extern", .Keyword_extern},
34 .{"false", .Keyword_false},
35 .{"fn", .Keyword_fn},
36 .{"for", .Keyword_for},
37 .{"if", .Keyword_if},
38 .{"inline", .Keyword_inline},
39 .{"noalias", .Keyword_noalias},
40 .{"noasync", .Keyword_nosuspend}, // TODO: remove this
41 .{"noinline", .Keyword_noinline},
42 .{"nosuspend", .Keyword_nosuspend},
43 .{"null", .Keyword_null},
44 .{"or", .Keyword_or},
45 .{"orelse", .Keyword_orelse},
46 .{"packed", .Keyword_packed},
47 .{"pub", .Keyword_pub},
48 .{"resume", .Keyword_resume},
49 .{"return", .Keyword_return},
50 .{"linksection", .Keyword_linksection},
51 .{"struct", .Keyword_struct},
52 .{"suspend", .Keyword_suspend},
53 .{"switch", .Keyword_switch},
54 .{"test", .Keyword_test},
55 .{"threadlocal", .Keyword_threadlocal},
56 .{"true", .Keyword_true},
57 .{"try", .Keyword_try},
58 .{"undefined", .Keyword_undefined},
59 .{"union", .Keyword_union},
60 .{"unreachable", .Keyword_unreachable},
61 .{"usingnamespace", .Keyword_usingnamespace},
62 .{"var", .Keyword_var},
63 .{"volatile", .Keyword_volatile},
64 .{"while", .Keyword_while},
65 });
7866
7967 pub fn getKeyword(bytes: []const u8) ?Id {
80 const precomputed = comptime blk: {
81 @setEvalBranchQuota(2000);
82 var sorted_keywords = keywords;
83 const lenAsc = (struct {
84 fn lenAsc(a: Keyword, b: Keyword) bool {
85 return a.bytes.len < b.bytes.len;
86 }
87 }).lenAsc;
88 std.sort.sort(Keyword, &sorted_keywords, lenAsc);
89 const min_len = sorted_keywords[0].bytes.len;
90 const max_len = sorted_keywords[sorted_keywords.len - 1].bytes.len;
91 var len_indexes: [max_len + 1]usize = undefined;
92 var len: usize = 0;
93 var kw_i: usize = 0;
94 while (len <= max_len) : (len += 1) {
95 // find the first keyword len == len
96 while (len > sorted_keywords[kw_i].bytes.len) {
97 kw_i += 1;
98 }
99 len_indexes[len] = kw_i;
100 }
101 break :blk .{
102 .min_len = min_len,
103 .max_len = max_len,
104 .sorted_keywords = sorted_keywords,
105 .len_indexes = len_indexes,
106 };
107 };
108 if (bytes.len < precomputed.min_len or bytes.len > precomputed.max_len)
109 return null;
110
111 var i = precomputed.len_indexes[bytes.len];
112 while (true) {
113 const kw = precomputed.sorted_keywords[i];
114 if (kw.bytes.len != bytes.len)
115 return null;
116 if (mem.eql(u8, kw.bytes, bytes))
117 return kw.id;
118 i += 1;
119 if (i >= precomputed.sorted_keywords.len)
120 return null;
121 }
68 return keywords.get(bytes);
12269 }
12370
12471 pub const Id = enum {