authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-07-25 03:47:45+10:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-07-24 13:47:45-04:00
log559150e8440d288adfaeb84c8bc3ec400605287d
treecd786a3360a780067c1f12d9ec84beca49f087b7
parentd82b35901035a325ca7afd38b28ff2386f90ae84
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Xxhash perf (#15947)

Improvements for xxHash performance, both on small keys as well as large slices. * std.hash: make benchmarks consistent for xxhash There is some odd behaviour in the timings for the XXHash benchmarks introduced in 879f0b9, specifically the changes to the allocation in benchmarkHash. The problem is somewhere in the difference between 9628243 and 9362d61 (these are commit that were force-pushed over but can be found in PR #15917). * std.hash: correctly calculate throughput in benchmark * std.hash: add hashes per sec to small key output * std.hash: add exact and small xxhash routines * std.hash: add --small-only flag to benchmark * std.hash.xxhash: extract stateless Accumulator type * std.hash.xxhash: cleanup hash() and improve small key perf * std.hash.xxhash: port xxhash64 changes to xxhash32 * std.hash: change benchmark --small-only flag to --include-array

2 files changed, 564 insertions(+), 157 deletions(-)

lib/std/hash/benchmark.zig+224-19
......@@ -18,6 +18,7 @@ const Hash = struct {
1818 name: []const u8,
1919 has_iterative_api: bool = true,
2020 has_crypto_api: bool = false,
21 has_anytype_api: ?[]const comptime_int = null,
2122 init_u8s: ?[]const u8 = null,
2223 init_u64: ?u64 = null,
2324};
......@@ -27,11 +28,13 @@ const hashes = [_]Hash{
2728 .ty = hash.XxHash64,
2829 .name = "xxhash64",
2930 .init_u64 = 0,
31 .has_anytype_api = @as([]const comptime_int, &[_]comptime_int{ 8, 16, 32, 48, 64, 80, 96, 112, 128 }),
3032 },
3133 Hash{
3234 .ty = hash.XxHash32,
3335 .name = "xxhash32",
3436 .init_u64 = 0,
37 .has_anytype_api = @as([]const comptime_int, &[_]comptime_int{ 8, 16, 32, 48, 64, 80, 96, 112, 128 }),
3538 },
3639 Hash{
3740 .ty = hash.Wyhash,
......@@ -99,14 +102,14 @@ const Result = struct {
99102};
100103
101104const block_size: usize = 8 * 8192;
102const alignment: usize = 64;
103105
104106pub fn benchmarkHash(comptime H: anytype, bytes: usize, allocator: std.mem.Allocator) !Result {
105 const blocks_count = bytes / block_size;
106 var blocks = try allocator.alloc(u8, block_size + alignment * (blocks_count - 1));
107 var blocks = try allocator.alloc(u8, bytes);
107108 defer allocator.free(blocks);
108109 random.bytes(blocks);
109110
111 const block_count = bytes / block_size;
112
110113 var h = blk: {
111114 if (H.init_u8s) |init| {
112115 break :blk H.ty.init(init[0..H.ty.key_length]);
......@@ -118,17 +121,17 @@ pub fn benchmarkHash(comptime H: anytype, bytes: usize, allocator: std.mem.Alloc
118121 };
119122
120123 var timer = try Timer.start();
121 const start = timer.lap();
122 for (0..blocks_count) |i| {
123 h.update(blocks[i * alignment ..][0..block_size]);
124 for (0..block_count) |i| {
125 h.update(blocks[i * block_size ..][0..block_size]);
124126 }
125127 const final = if (H.has_crypto_api) @as(u64, @truncate(h.finalInt())) else h.final();
126128 std.mem.doNotOptimizeAway(final);
127129
128 const end = timer.read();
130 const elapsed_ns = timer.read();
129131
130 const elapsed_s = @as(f64, @floatFromInt(end - start)) / time.ns_per_s;
131 const throughput = @as(u64, @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s));
132 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
133 const size_float: f64 = @floatFromInt(block_size * block_count);
134 const throughput: u64 = @intFromFloat(size_float / elapsed_s);
132135
133136 return Result{
134137 .hash = final,
......@@ -144,7 +147,6 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize
144147 const key_count = bytes / key_size;
145148
146149 var timer = try Timer.start();
147 const start = timer.lap();
148150
149151 var sum: u64 = 0;
150152 for (0..key_count) |i| {
......@@ -164,10 +166,59 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize
164166 };
165167 sum +%= final;
166168 }
167 const end = timer.read();
169 const elapsed_ns = timer.read();
170
171 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
172 const size_float: f64 = @floatFromInt(key_count * key_size);
173 const throughput: u64 = @intFromFloat(size_float / elapsed_s);
174
175 std.mem.doNotOptimizeAway(sum);
176
177 return Result{
178 .hash = sum,
179 .throughput = throughput,
180 };
181}
182
183// the array and array pointer benchmarks for xxhash are very sensitive to in-lining,
184// if you see strange performance changes consider using `.never_inline` or `.always_inline`
185// to ensure the changes are not only due to the optimiser inlining the benchmark differently
186pub fn benchmarkHashSmallKeysArrayPtr(
187 comptime H: anytype,
188 comptime key_size: usize,
189 bytes: usize,
190 allocator: std.mem.Allocator,
191) !Result {
192 var blocks = try allocator.alloc(u8, bytes);
193 defer allocator.free(blocks);
194 random.bytes(blocks);
195
196 const key_count = bytes / key_size;
197
198 var timer = try Timer.start();
199
200 var sum: u64 = 0;
201 for (0..key_count) |i| {
202 const small_key = blocks[i * key_size ..][0..key_size];
203 const final: u64 = blk: {
204 if (H.init_u8s) |init| {
205 if (H.has_crypto_api) {
206 break :blk @truncate(H.ty.toInt(small_key, init[0..H.ty.key_length]));
207 } else {
208 break :blk H.ty.hash(init, small_key);
209 }
210 }
211 if (H.init_u64) |init| {
212 break :blk H.ty.hash(init, small_key);
213 }
214 break :blk H.ty.hash(small_key);
215 };
216 sum +%= final;
217 }
218 const elapsed_ns = timer.read();
168219
169 const elapsed_s = @as(f64, @floatFromInt(end - start)) / time.ns_per_s;
170 const throughput = @as(u64, @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s));
220 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
221 const throughput: u64 = @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s);
171222
172223 std.mem.doNotOptimizeAway(sum);
173224
......@@ -177,6 +228,95 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize
177228 };
178229}
179230
231// the array and array pointer benchmarks for xxhash are very sensitive to in-lining,
232// if you see strange performance changes consider using `.never_inline` or `.always_inline`
233// to ensure the changes are not only due to the optimiser inlining the benchmark differently
234pub fn benchmarkHashSmallKeysArray(
235 comptime H: anytype,
236 comptime key_size: usize,
237 bytes: usize,
238 allocator: std.mem.Allocator,
239) !Result {
240 var blocks = try allocator.alloc(u8, bytes);
241 defer allocator.free(blocks);
242 random.bytes(blocks);
243
244 const key_count = bytes / key_size;
245
246 var i: usize = 0;
247 var timer = try Timer.start();
248
249 var sum: u64 = 0;
250 while (i < key_count) : (i += 1) {
251 const small_key = blocks[i * key_size ..][0..key_size];
252 const final: u64 = blk: {
253 if (H.init_u8s) |init| {
254 if (H.has_crypto_api) {
255 break :blk @truncate(H.ty.toInt(small_key, init[0..H.ty.key_length]));
256 } else {
257 break :blk H.ty.hash(init, small_key.*);
258 }
259 }
260 if (H.init_u64) |init| {
261 break :blk H.ty.hash(init, small_key.*);
262 }
263 break :blk H.ty.hash(small_key.*);
264 };
265 sum +%= final;
266 }
267 const elapsed_ns = timer.read();
268
269 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
270 const throughput: u64 = @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s);
271
272 std.mem.doNotOptimizeAway(sum);
273
274 return Result{
275 .hash = sum,
276 .throughput = throughput,
277 };
278}
279
280pub fn benchmarkHashSmallApi(comptime H: anytype, key_size: usize, bytes: usize, allocator: std.mem.Allocator) !Result {
281 var blocks = try allocator.alloc(u8, bytes);
282 defer allocator.free(blocks);
283 random.bytes(blocks);
284
285 const key_count = bytes / key_size;
286
287 var timer = try Timer.start();
288
289 var sum: u64 = 0;
290 for (0..key_count) |i| {
291 const small_key = blocks[i * key_size ..][0..key_size];
292 const final: u64 = blk: {
293 if (H.init_u8s) |init| {
294 if (H.has_crypto_api) {
295 break :blk @truncate(H.ty.toInt(small_key, init[0..H.ty.key_length]));
296 } else {
297 break :blk H.ty.hashSmall(init, small_key);
298 }
299 }
300 if (H.init_u64) |init| {
301 break :blk H.ty.hashSmall(init, small_key);
302 }
303 break :blk H.ty.hashSmall(small_key);
304 };
305 sum +%= final;
306 }
307 const elapsed_ns = timer.read();
308
309 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
310 const throughput: u64 = @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s);
311
312 std.mem.doNotOptimizeAway(sum);
313
314 return Result{
315 .throughput = throughput,
316 .hash = sum,
317 };
318}
319
180320fn usage() void {
181321 std.debug.print(
182322 \\throughput_test [options]
......@@ -205,9 +345,12 @@ pub fn main() !void {
205345
206346 var filter: ?[]u8 = "";
207347 var count: usize = mode(128 * MiB);
208 var key_size: usize = 32;
348 var key_size: ?usize = null;
209349 var seed: u32 = 0;
210350 var test_iterative_only = false;
351 var test_arrays = false;
352
353 const default_small_key_size = 32;
211354
212355 var i: usize = 1;
213356 while (i < args.len) : (i += 1) {
......@@ -248,12 +391,14 @@ pub fn main() !void {
248391 }
249392
250393 key_size = try std.fmt.parseUnsigned(usize, args[i], 10);
251 if (key_size > block_size) {
394 if (key_size.? > block_size) {
252395 try stdout.print("key_size cannot exceed block size of {}\n", .{block_size});
253396 std.os.exit(1);
254397 }
255398 } else if (std.mem.eql(u8, args[i], "--iterative-only")) {
256399 test_iterative_only = true;
400 } else if (std.mem.eql(u8, args[i], "--include-array")) {
401 test_arrays = true;
257402 } else if (std.mem.eql(u8, args[i], "--help")) {
258403 usage();
259404 return;
......@@ -268,7 +413,7 @@ pub fn main() !void {
268413 const allocator = gpa.allocator();
269414
270415 inline for (hashes) |H| {
271 if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) {
416 if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) hash: {
272417 if (!test_iterative_only or H.has_iterative_api) {
273418 try stdout.print("{s}\n", .{H.name});
274419
......@@ -281,9 +426,69 @@ pub fn main() !void {
281426 }
282427
283428 if (!test_iterative_only) {
284 prng.seed(seed);
285 const result_small = try benchmarkHashSmallKeys(H, key_size, count, allocator);
286 try stdout.print(" small keys: {:5} MiB/s [{x:0<16}]\n", .{ result_small.throughput / (1 * MiB), result_small.hash });
429 if (key_size) |size| {
430 prng.seed(seed);
431 const result_small = try benchmarkHashSmallKeys(H, size, count, allocator);
432 try stdout.print(" small keys: {:3}B {:5} MiB/s {} Hashes/s [{x:0<16}]\n", .{
433 size,
434 result_small.throughput / (1 * MiB),
435 result_small.throughput / size,
436 result_small.hash,
437 });
438
439 if (!test_arrays) break :hash;
440 if (H.has_anytype_api) |sizes| {
441 inline for (sizes) |exact_size| {
442 if (size == exact_size) {
443 prng.seed(seed);
444 const result_array = try benchmarkHashSmallKeysArray(H, exact_size, count, allocator);
445 prng.seed(seed);
446 const result_ptr = try benchmarkHashSmallKeysArrayPtr(H, exact_size, count, allocator);
447 try stdout.print(" array: {:5} MiB/s [{x:0<16}]\n", .{
448 result_array.throughput / (1 * MiB),
449 result_array.hash,
450 });
451 try stdout.print(" array ptr: {:5} MiB/s [{x:0<16}]\n", .{
452 result_ptr.throughput / (1 * MiB),
453 result_ptr.hash,
454 });
455 }
456 }
457 }
458 } else {
459 prng.seed(seed);
460 const result_small = try benchmarkHashSmallKeys(H, default_small_key_size, count, allocator);
461 try stdout.print(" small keys: {:3}B {:5} MiB/s {} Hashes/s [{x:0<16}]\n", .{
462 default_small_key_size,
463 result_small.throughput / (1 * MiB),
464 result_small.throughput / default_small_key_size,
465 result_small.hash,
466 });
467
468 if (!test_arrays) break :hash;
469 if (H.has_anytype_api) |sizes| {
470 try stdout.print(" array:\n", .{});
471 inline for (sizes) |exact_size| {
472 prng.seed(seed);
473 const result = try benchmarkHashSmallKeysArray(H, exact_size, count, allocator);
474 try stdout.print(" {d: >3}B {:5} MiB/s [{x:0<16}]\n", .{
475 exact_size,
476 result.throughput / (1 * MiB),
477 result.hash,
478 });
479 }
480 try stdout.print(" array ptr: \n", .{});
481 inline for (sizes) |exact_size| {
482 prng.seed(seed);
483 const result = try benchmarkHashSmallKeysArrayPtr(H, exact_size, count, allocator);
484 try stdout.print(" {d: >3}B {:5} MiB/s [{x:0<16}]\n", .{
485 exact_size,
486 result.throughput / (1 * MiB),
487 result.hash,
488 });
489 }
490 }
491 }
287492 }
288493 }
289494 }
lib/std/hash/xxhash.zig+340-138
......@@ -5,11 +5,7 @@ const expectEqual = std.testing.expectEqual;
55const rotl = std.math.rotl;
66
77pub const XxHash64 = struct {
8 acc1: u64,
9 acc2: u64,
10 acc3: u64,
11 acc4: u64,
12
8 accumulator: Accumulator,
139 seed: u64,
1410 buf: [32]u8,
1511 buf_len: usize,
......@@ -21,20 +17,174 @@ pub const XxHash64 = struct {
2117 const prime_4 = 0x85EBCA77C2B2AE63; // 0b1000010111101011110010100111011111000010101100101010111001100011
2218 const prime_5 = 0x27D4EB2F165667C5; // 0b0010011111010100111010110010111100010110010101100110011111000101
2319
20 const Accumulator = struct {
21 acc1: u64,
22 acc2: u64,
23 acc3: u64,
24 acc4: u64,
25
26 fn init(seed: u64) Accumulator {
27 return .{
28 .acc1 = seed +% prime_1 +% prime_2,
29 .acc2 = seed +% prime_2,
30 .acc3 = seed,
31 .acc4 = seed -% prime_1,
32 };
33 }
34
35 fn updateEmpty(self: *Accumulator, input: anytype, comptime unroll_count: usize) usize {
36 var i: usize = 0;
37
38 if (unroll_count > 0) {
39 const unrolled_bytes = unroll_count * 32;
40 while (i + unrolled_bytes <= input.len) : (i += unrolled_bytes) {
41 inline for (0..unroll_count) |j| {
42 self.processStripe(input[i + j * 32 ..][0..32]);
43 }
44 }
45 }
46
47 while (i + 32 <= input.len) : (i += 32) {
48 self.processStripe(input[i..][0..32]);
49 }
50
51 return i;
52 }
53
54 fn processStripe(self: *Accumulator, buf: *const [32]u8) void {
55 self.acc1 = round(self.acc1, mem.readIntLittle(u64, buf[0..8]));
56 self.acc2 = round(self.acc2, mem.readIntLittle(u64, buf[8..16]));
57 self.acc3 = round(self.acc3, mem.readIntLittle(u64, buf[16..24]));
58 self.acc4 = round(self.acc4, mem.readIntLittle(u64, buf[24..32]));
59 }
60
61 fn merge(self: Accumulator) u64 {
62 var acc = rotl(u64, self.acc1, 1) +% rotl(u64, self.acc2, 7) +%
63 rotl(u64, self.acc3, 12) +% rotl(u64, self.acc4, 18);
64 acc = mergeAccumulator(acc, self.acc1);
65 acc = mergeAccumulator(acc, self.acc2);
66 acc = mergeAccumulator(acc, self.acc3);
67 acc = mergeAccumulator(acc, self.acc4);
68 return acc;
69 }
70
71 fn mergeAccumulator(acc: u64, other: u64) u64 {
72 const a = acc ^ round(0, other);
73 const b = a *% prime_1;
74 return b +% prime_4;
75 }
76 };
77
78 fn finalize(
79 unfinished: u64,
80 byte_count: usize,
81 partial: anytype,
82 ) u64 {
83 std.debug.assert(partial.len < 32);
84 var acc = unfinished +% @as(u64, byte_count) +% @as(u64, partial.len);
85
86 switch (partial.len) {
87 inline 0, 1, 2, 3 => |count| {
88 inline for (0..count) |i| acc = finalize1(acc, partial[i]);
89 return avalanche(acc);
90 },
91 inline 4, 5, 6, 7 => |count| {
92 acc = finalize4(acc, partial[0..4]);
93 inline for (4..count) |i| acc = finalize1(acc, partial[i]);
94 return avalanche(acc);
95 },
96 inline 8, 9, 10, 11 => |count| {
97 acc = finalize8(acc, partial[0..8]);
98 inline for (8..count) |i| acc = finalize1(acc, partial[i]);
99 return avalanche(acc);
100 },
101 inline 12, 13, 14, 15 => |count| {
102 acc = finalize8(acc, partial[0..8]);
103 acc = finalize4(acc, partial[8..12]);
104 inline for (12..count) |i| acc = finalize1(acc, partial[i]);
105 return avalanche(acc);
106 },
107 inline 16, 17, 18, 19 => |count| {
108 acc = finalize8(acc, partial[0..8]);
109 acc = finalize8(acc, partial[8..16]);
110 inline for (16..count) |i| acc = finalize1(acc, partial[i]);
111 return avalanche(acc);
112 },
113 inline 20, 21, 22, 23 => |count| {
114 acc = finalize8(acc, partial[0..8]);
115 acc = finalize8(acc, partial[8..16]);
116 acc = finalize4(acc, partial[16..20]);
117 inline for (20..count) |i| acc = finalize1(acc, partial[i]);
118 return avalanche(acc);
119 },
120 inline 24, 25, 26, 27 => |count| {
121 acc = finalize8(acc, partial[0..8]);
122 acc = finalize8(acc, partial[8..16]);
123 acc = finalize8(acc, partial[16..24]);
124 inline for (24..count) |i| acc = finalize1(acc, partial[i]);
125 return avalanche(acc);
126 },
127 inline 28, 29, 30, 31 => |count| {
128 acc = finalize8(acc, partial[0..8]);
129 acc = finalize8(acc, partial[8..16]);
130 acc = finalize8(acc, partial[16..24]);
131 acc = finalize4(acc, partial[24..28]);
132 inline for (28..count) |i| acc = finalize1(acc, partial[i]);
133 return avalanche(acc);
134 },
135 else => unreachable,
136 }
137 }
138
139 fn finalize8(v: u64, bytes: *const [8]u8) u64 {
140 var acc = v;
141 const lane = mem.readIntLittle(u64, bytes);
142 acc ^= round(0, lane);
143 acc = rotl(u64, acc, 27) *% prime_1;
144 acc +%= prime_4;
145 return acc;
146 }
147
148 fn finalize4(v: u64, bytes: *const [4]u8) u64 {
149 var acc = v;
150 const lane = @as(u64, mem.readIntLittle(u32, bytes));
151 acc ^= lane *% prime_1;
152 acc = rotl(u64, acc, 23) *% prime_2;
153 acc +%= prime_3;
154 return acc;
155 }
156
157 fn finalize1(v: u64, byte: u8) u64 {
158 var acc = v;
159 const lane = @as(u64, byte);
160 acc ^= lane *% prime_5;
161 acc = rotl(u64, acc, 11) *% prime_1;
162 return acc;
163 }
164
165 fn avalanche(value: u64) u64 {
166 var result = value ^ (value >> 33);
167 result *%= prime_2;
168 result ^= result >> 29;
169 result *%= prime_3;
170 result ^= result >> 32;
171
172 return result;
173 }
174
24175 pub fn init(seed: u64) XxHash64 {
25176 return XxHash64{
177 .accumulator = Accumulator.init(seed),
26178 .seed = seed,
27 .acc1 = seed +% prime_1 +% prime_2,
28 .acc2 = seed +% prime_2,
29 .acc3 = seed,
30 .acc4 = seed -% prime_1,
31179 .buf = undefined,
32180 .buf_len = 0,
33181 .byte_count = 0,
34182 };
35183 }
36184
37 pub fn update(self: *XxHash64, input: []const u8) void {
185 pub fn update(self: *XxHash64, input: anytype) void {
186 validateType(@TypeOf(input));
187
38188 if (input.len < 32 - self.buf_len) {
39189 @memcpy(self.buf[self.buf_len..][0..input.len], input);
40190 self.buf_len += input.len;
......@@ -46,99 +196,54 @@ pub const XxHash64 = struct {
46196 if (self.buf_len > 0) {
47197 i = 32 - self.buf_len;
48198 @memcpy(self.buf[self.buf_len..][0..i], input[0..i]);
49 self.processStripe(&self.buf);
50 self.buf_len = 0;
199 self.accumulator.processStripe(&self.buf);
200 self.byte_count += self.buf_len;
51201 }
52202
53 while (i + 32 <= input.len) : (i += 32) {
54 self.processStripe(input[i..][0..32]);
55 }
203 i += self.accumulator.updateEmpty(input[i..], 32);
204 self.byte_count += i;
56205
57206 const remaining_bytes = input[i..];
58207 @memcpy(self.buf[0..remaining_bytes.len], remaining_bytes);
59208 self.buf_len = remaining_bytes.len;
60209 }
61210
62 inline fn processStripe(self: *XxHash64, buf: *const [32]u8) void {
63 self.acc1 = round(self.acc1, mem.readIntLittle(u64, buf[0..8]));
64 self.acc2 = round(self.acc2, mem.readIntLittle(u64, buf[8..16]));
65 self.acc3 = round(self.acc3, mem.readIntLittle(u64, buf[16..24]));
66 self.acc4 = round(self.acc4, mem.readIntLittle(u64, buf[24..32]));
67 self.byte_count += 32;
68 }
69
70 inline fn round(acc: u64, lane: u64) u64 {
211 fn round(acc: u64, lane: u64) u64 {
71212 const a = acc +% (lane *% prime_2);
72213 const b = rotl(u64, a, 31);
73214 return b *% prime_1;
74215 }
75216
76217 pub fn final(self: *XxHash64) u64 {
77 var acc: u64 = undefined;
78
79 if (self.byte_count < 32) {
80 acc = self.seed +% prime_5;
81 } else {
82 acc = rotl(u64, self.acc1, 1) +% rotl(u64, self.acc2, 7) +%
83 rotl(u64, self.acc3, 12) +% rotl(u64, self.acc4, 18);
84 acc = mergeAccumulator(acc, self.acc1);
85 acc = mergeAccumulator(acc, self.acc2);
86 acc = mergeAccumulator(acc, self.acc3);
87 acc = mergeAccumulator(acc, self.acc4);
88 }
218 const unfinished = if (self.byte_count < 32)
219 self.seed +% prime_5
220 else
221 self.accumulator.merge();
89222
90 acc = acc +% @as(u64, self.byte_count) +% @as(u64, self.buf_len);
223 return finalize(unfinished, self.byte_count, self.buf[0..self.buf_len]);
224 }
91225
92 var pos: usize = 0;
93 while (pos + 8 <= self.buf_len) : (pos += 8) {
94 const lane = mem.readIntLittle(u64, self.buf[pos..][0..8]);
95 acc ^= round(0, lane);
96 acc = rotl(u64, acc, 27) *% prime_1;
97 acc +%= prime_4;
98 }
226 const Size = enum {
227 small,
228 large,
229 unknown,
230 };
99231
100 if (pos + 4 <= self.buf_len) {
101 const lane = @as(u64, mem.readIntLittle(u32, self.buf[pos..][0..4]));
102 acc ^= lane *% prime_1;
103 acc = rotl(u64, acc, 23) *% prime_2;
104 acc +%= prime_3;
105 pos += 4;
106 }
232 pub fn hash(seed: u64, input: anytype) u64 {
233 validateType(@TypeOf(input));
107234
108 while (pos < self.buf_len) : (pos += 1) {
109 const lane = @as(u64, self.buf[pos]);
110 acc ^= lane *% prime_5;
111 acc = rotl(u64, acc, 11) *% prime_1;
235 if (input.len < 32) {
236 return finalize(seed +% prime_5, 0, input);
237 } else {
238 var hasher = Accumulator.init(seed);
239 const i = hasher.updateEmpty(input, 0);
240 return finalize(hasher.merge(), i, input[i..]);
112241 }
113
114 acc ^= acc >> 33;
115 acc *%= prime_2;
116 acc ^= acc >> 29;
117 acc *%= prime_3;
118 acc ^= acc >> 32;
119
120 return acc;
121 }
122
123 inline fn mergeAccumulator(acc: u64, other: u64) u64 {
124 const a = acc ^ round(0, other);
125 const b = a *% prime_1;
126 return b +% prime_4;
127 }
128
129 pub fn hash(seed: u64, input: []const u8) u64 {
130 var hasher = XxHash64.init(seed);
131 hasher.update(input);
132 return hasher.final();
133242 }
134243};
135244
136245pub const XxHash32 = struct {
137 acc1: u32,
138 acc2: u32,
139 acc3: u32,
140 acc4: u32,
141
246 accumulator: Accumulator,
142247 seed: u32,
143248 buf: [16]u8,
144249 buf_len: usize,
......@@ -150,13 +255,57 @@ pub const XxHash32 = struct {
150255 const prime_4 = 0x27D4EB2F; // 0b00100111110101001110101100101111
151256 const prime_5 = 0x165667B1; // 0b00010110010101100110011110110001
152257
258 const Accumulator = struct {
259 acc1: u32,
260 acc2: u32,
261 acc3: u32,
262 acc4: u32,
263
264 fn init(seed: u32) Accumulator {
265 return .{
266 .acc1 = seed +% prime_1 +% prime_2,
267 .acc2 = seed +% prime_2,
268 .acc3 = seed,
269 .acc4 = seed -% prime_1,
270 };
271 }
272
273 fn updateEmpty(self: *Accumulator, input: anytype, comptime unroll_count: usize) usize {
274 var i: usize = 0;
275
276 if (unroll_count > 0) {
277 const unrolled_bytes = unroll_count * 16;
278 while (i + unrolled_bytes <= input.len) : (i += unrolled_bytes) {
279 inline for (0..unroll_count) |j| {
280 self.processStripe(input[i + j * 16 ..][0..16]);
281 }
282 }
283 }
284
285 while (i + 16 <= input.len) : (i += 16) {
286 self.processStripe(input[i..][0..16]);
287 }
288
289 return i;
290 }
291
292 fn processStripe(self: *Accumulator, buf: *const [16]u8) void {
293 self.acc1 = round(self.acc1, mem.readIntLittle(u32, buf[0..4]));
294 self.acc2 = round(self.acc2, mem.readIntLittle(u32, buf[4..8]));
295 self.acc3 = round(self.acc3, mem.readIntLittle(u32, buf[8..12]));
296 self.acc4 = round(self.acc4, mem.readIntLittle(u32, buf[12..16]));
297 }
298
299 fn merge(self: Accumulator) u32 {
300 return rotl(u32, self.acc1, 1) +% rotl(u32, self.acc2, 7) +%
301 rotl(u32, self.acc3, 12) +% rotl(u32, self.acc4, 18);
302 }
303 };
304
153305 pub fn init(seed: u32) XxHash32 {
154306 return XxHash32{
307 .accumulator = Accumulator.init(seed),
155308 .seed = seed,
156 .acc1 = seed +% prime_1 +% prime_2,
157 .acc2 = seed +% prime_2,
158 .acc3 = seed,
159 .acc4 = seed -% prime_1,
160309 .buf = undefined,
161310 .buf_len = 0,
162311 .byte_count = 0,
......@@ -164,6 +313,8 @@ pub const XxHash32 = struct {
164313 }
165314
166315 pub fn update(self: *XxHash32, input: []const u8) void {
316 validateType(@TypeOf(input));
317
167318 if (input.len < 16 - self.buf_len) {
168319 @memcpy(self.buf[self.buf_len..][0..input.len], input);
169320 self.buf_len += input.len;
......@@ -175,59 +326,85 @@ pub const XxHash32 = struct {
175326 if (self.buf_len > 0) {
176327 i = 16 - self.buf_len;
177328 @memcpy(self.buf[self.buf_len..][0..i], input[0..i]);
178 self.processStripe(&self.buf);
329 self.accumulator.processStripe(&self.buf);
330 self.byte_count += self.buf_len;
179331 self.buf_len = 0;
180332 }
181333
182 while (i + 16 <= input.len) : (i += 16) {
183 self.processStripe(input[i..][0..16]);
184 }
334 i += self.accumulator.updateEmpty(input[i..], 16);
335 self.byte_count += i;
185336
186337 const remaining_bytes = input[i..];
187338 @memcpy(self.buf[0..remaining_bytes.len], remaining_bytes);
188339 self.buf_len = remaining_bytes.len;
189340 }
190341
191 inline fn processStripe(self: *XxHash32, buf: *const [16]u8) void {
192 self.acc1 = round(self.acc1, mem.readIntLittle(u32, buf[0..4]));
193 self.acc2 = round(self.acc2, mem.readIntLittle(u32, buf[4..8]));
194 self.acc3 = round(self.acc3, mem.readIntLittle(u32, buf[8..12]));
195 self.acc4 = round(self.acc4, mem.readIntLittle(u32, buf[12..16]));
196 self.byte_count += 16;
197 }
198
199 inline fn round(acc: u32, lane: u32) u32 {
342 fn round(acc: u32, lane: u32) u32 {
200343 const a = acc +% (lane *% prime_2);
201344 const b = rotl(u32, a, 13);
202345 return b *% prime_1;
203346 }
204347
205348 pub fn final(self: *XxHash32) u32 {
206 var acc: u32 = undefined;
349 const unfinished = if (self.byte_count < 16)
350 self.seed +% prime_5
351 else
352 self.accumulator.merge();
207353
208 if (self.byte_count < 16) {
209 acc = self.seed +% prime_5;
210 } else {
211 acc = rotl(u32, self.acc1, 1) +% rotl(u32, self.acc2, 7) +%
212 rotl(u32, self.acc3, 12) +% rotl(u32, self.acc4, 18);
354 return finalize(unfinished, self.byte_count, self.buf[0..self.buf_len]);
355 }
356
357 fn finalize(unfinished: u32, byte_count: usize, partial: anytype) u32 {
358 std.debug.assert(partial.len < 16);
359 var acc = unfinished +% @as(u32, @intCast(byte_count)) +% @as(u32, @intCast(partial.len));
360
361 switch (partial.len) {
362 inline 0, 1, 2, 3 => |count| {
363 inline for (0..count) |i| acc = finalize1(acc, partial[i]);
364 return avalanche(acc);
365 },
366 inline 4, 5, 6, 7 => |count| {
367 acc = finalize4(acc, partial[0..4]);
368 inline for (4..count) |i| acc = finalize1(acc, partial[i]);
369 return avalanche(acc);
370 },
371 inline 8, 9, 10, 11 => |count| {
372 acc = finalize4(acc, partial[0..4]);
373 acc = finalize4(acc, partial[4..8]);
374 inline for (8..count) |i| acc = finalize1(acc, partial[i]);
375 return avalanche(acc);
376 },
377 inline 12, 13, 14, 15 => |count| {
378 acc = finalize4(acc, partial[0..4]);
379 acc = finalize4(acc, partial[4..8]);
380 acc = finalize4(acc, partial[8..12]);
381 inline for (12..count) |i| acc = finalize1(acc, partial[i]);
382 return avalanche(acc);
383 },
384 else => unreachable,
213385 }
214386
215 acc = acc +% @as(u32, @intCast(self.byte_count)) +% @as(u32, @intCast(self.buf_len));
387 return avalanche(acc);
388 }
216389
217 var pos: usize = 0;
218 while (pos + 4 <= self.buf_len) : (pos += 4) {
219 const lane = mem.readIntLittle(u32, self.buf[pos..][0..4]);
220 acc +%= lane *% prime_3;
221 acc = rotl(u32, acc, 17) *% prime_4;
222 }
390 fn finalize4(v: u32, bytes: *const [4]u8) u32 {
391 var acc = v;
392 const lane = mem.readIntLittle(u32, bytes);
393 acc +%= lane *% prime_3;
394 acc = rotl(u32, acc, 17) *% prime_4;
395 return acc;
396 }
223397
224 while (pos < self.buf_len) : (pos += 1) {
225 const lane = @as(u32, self.buf[pos]);
226 acc +%= lane *% prime_5;
227 acc = rotl(u32, acc, 11) *% prime_1;
228 }
398 fn finalize1(v: u32, byte: u8) u32 {
399 var acc = v;
400 const lane = @as(u32, byte);
401 acc +%= lane *% prime_5;
402 acc = rotl(u32, acc, 11) *% prime_1;
403 return acc;
404 }
229405
230 acc ^= acc >> 15;
406 fn avalanche(value: u32) u32 {
407 var acc = value ^ value >> 15;
231408 acc *%= prime_2;
232409 acc ^= acc >> 13;
233410 acc *%= prime_3;
......@@ -236,33 +413,58 @@ pub const XxHash32 = struct {
236413 return acc;
237414 }
238415
239 pub fn hash(seed: u32, input: []const u8) u32 {
240 var hasher = XxHash32.init(seed);
241 hasher.update(input);
242 return hasher.final();
416 pub fn hash(seed: u32, input: anytype) u32 {
417 validateType(@TypeOf(input));
418
419 if (input.len < 16) {
420 return finalize(seed +% prime_5, 0, input);
421 } else {
422 var hasher = Accumulator.init(seed);
423 const i = hasher.updateEmpty(input, 0);
424 return finalize(hasher.merge(), i, input[i..]);
425 }
243426 }
244427};
245428
429fn validateType(comptime T: type) void {
430 comptime {
431 if (!((std.meta.trait.isSlice(T) or
432 std.meta.trait.is(.Array)(T) or
433 std.meta.trait.isPtrTo(.Array)(T)) and
434 std.meta.Elem(T) == u8))
435 {
436 @compileError("expect a slice, array or pointer to array of u8, got " ++ @typeName(T));
437 }
438 }
439}
440
441fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64) !void {
442 try expectEqual(expected, H.hash(0, input));
443
444 var hasher = H.init(seed);
445 hasher.update(input);
446 try expectEqual(expected, hasher.final());
447}
448
246449test "xxhash64" {
247 const hash = XxHash64.hash;
248
249 try expectEqual(hash(0, ""), 0xef46db3751d8e999);
250 try expectEqual(hash(0, "a"), 0xd24ec4f1a98c6e5b);
251 try expectEqual(hash(0, "abc"), 0x44bc2cf5ad770999);
252 try expectEqual(hash(0, "message digest"), 0x066ed728fceeb3be);
253 try expectEqual(hash(0, "abcdefghijklmnopqrstuvwxyz"), 0xcfe1f278fa89835c);
254 try expectEqual(hash(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0xaaa46907d3047814);
255 try expectEqual(hash(0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0xe04a477f19ee145d);
450 const H = XxHash64;
451 try testExpect(H, 0, "", 0xef46db3751d8e999);
452 try testExpect(H, 0, "a", 0xd24ec4f1a98c6e5b);
453 try testExpect(H, 0, "abc", 0x44bc2cf5ad770999);
454 try testExpect(H, 0, "message digest", 0x066ed728fceeb3be);
455 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0xcfe1f278fa89835c);
456 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xaaa46907d3047814);
457 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0xe04a477f19ee145d);
256458}
257459
258460test "xxhash32" {
259 const hash = XxHash32.hash;
260
261 try expectEqual(hash(0, ""), 0x02cc5d05);
262 try expectEqual(hash(0, "a"), 0x550d7456);
263 try expectEqual(hash(0, "abc"), 0x32d153ff);
264 try expectEqual(hash(0, "message digest"), 0x7c948494);
265 try expectEqual(hash(0, "abcdefghijklmnopqrstuvwxyz"), 0x63a14d5f);
266 try expectEqual(hash(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x9c285e64);
267 try expectEqual(hash(0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x9c05f475);
461 const H = XxHash32;
462
463 try testExpect(H, 0, "", 0x02cc5d05);
464 try testExpect(H, 0, "a", 0x550d7456);
465 try testExpect(H, 0, "abc", 0x32d153ff);
466 try testExpect(H, 0, "message digest", 0x7c948494);
467 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0x63a14d5f);
468 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x9c285e64);
469 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0x9c05f475);
268470}