authorgravatar for 4678790+dweiller@users.noreply.github.comDominic <4678790+dweiller@users.noreply.github.com> 2023-07-25 03:47:45+10:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-07-24 13:47:45-04:00
log559150e8440d288adfaeb84c8bc3ec400605287d
treecd786a3360a780067c1f12d9ec84beca49f087b7
parentd82b35901035a325ca7afd38b28ff2386f90ae84
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Xxhash perf (#15947)

Improvements for xxHash performance, both on small keys as well as large slices. * std.hash: make benchmarks consistent for xxhash There is some odd behaviour in the timings for the XXHash benchmarks introduced in 879f0b9, specifically the changes to the allocation in benchmarkHash. The problem is somewhere in the difference between 9628243 and 9362d61 (these are commit that were force-pushed over but can be found in PR #15917). * std.hash: correctly calculate throughput in benchmark * std.hash: add hashes per sec to small key output * std.hash: add exact and small xxhash routines * std.hash: add --small-only flag to benchmark * std.hash.xxhash: extract stateless Accumulator type * std.hash.xxhash: cleanup hash() and improve small key perf * std.hash.xxhash: port xxhash64 changes to xxhash32 * std.hash: change benchmark --small-only flag to --include-array

2 files changed, 564 insertions(+), 157 deletions(-)

lib/std/hash/benchmark.zig+224-19
...@@ -18,6 +18,7 @@ const Hash = struct {...@@ -18,6 +18,7 @@ const Hash = struct {
18 name: []const u8,18 name: []const u8,
19 has_iterative_api: bool = true,19 has_iterative_api: bool = true,
20 has_crypto_api: bool = false,20 has_crypto_api: bool = false,
21 has_anytype_api: ?[]const comptime_int = null,
21 init_u8s: ?[]const u8 = null,22 init_u8s: ?[]const u8 = null,
22 init_u64: ?u64 = null,23 init_u64: ?u64 = null,
23};24};
...@@ -27,11 +28,13 @@ const hashes = [_]Hash{...@@ -27,11 +28,13 @@ const hashes = [_]Hash{
27 .ty = hash.XxHash64,28 .ty = hash.XxHash64,
28 .name = "xxhash64",29 .name = "xxhash64",
29 .init_u64 = 0,30 .init_u64 = 0,
31 .has_anytype_api = @as([]const comptime_int, &[_]comptime_int{ 8, 16, 32, 48, 64, 80, 96, 112, 128 }),
30 },32 },
31 Hash{33 Hash{
32 .ty = hash.XxHash32,34 .ty = hash.XxHash32,
33 .name = "xxhash32",35 .name = "xxhash32",
34 .init_u64 = 0,36 .init_u64 = 0,
37 .has_anytype_api = @as([]const comptime_int, &[_]comptime_int{ 8, 16, 32, 48, 64, 80, 96, 112, 128 }),
35 },38 },
36 Hash{39 Hash{
37 .ty = hash.Wyhash,40 .ty = hash.Wyhash,
...@@ -99,14 +102,14 @@ const Result = struct {...@@ -99,14 +102,14 @@ const Result = struct {
99};102};
100103
101const block_size: usize = 8 * 8192;104const block_size: usize = 8 * 8192;
102const alignment: usize = 64;
103105
104pub fn benchmarkHash(comptime H: anytype, bytes: usize, allocator: std.mem.Allocator) !Result {106pub fn benchmarkHash(comptime H: anytype, bytes: usize, allocator: std.mem.Allocator) !Result {
105 const blocks_count = bytes / block_size;107 var blocks = try allocator.alloc(u8, bytes);
106 var blocks = try allocator.alloc(u8, block_size + alignment * (blocks_count - 1));
107 defer allocator.free(blocks);108 defer allocator.free(blocks);
108 random.bytes(blocks);109 random.bytes(blocks);
109110
111 const block_count = bytes / block_size;
112
110 var h = blk: {113 var h = blk: {
111 if (H.init_u8s) |init| {114 if (H.init_u8s) |init| {
112 break :blk H.ty.init(init[0..H.ty.key_length]);115 break :blk H.ty.init(init[0..H.ty.key_length]);
...@@ -118,17 +121,17 @@ pub fn benchmarkHash(comptime H: anytype, bytes: usize, allocator: std.mem.Alloc...@@ -118,17 +121,17 @@ pub fn benchmarkHash(comptime H: anytype, bytes: usize, allocator: std.mem.Alloc
118 };121 };
119122
120 var timer = try Timer.start();123 var timer = try Timer.start();
121 const start = timer.lap();124 for (0..block_count) |i| {
122 for (0..blocks_count) |i| {125 h.update(blocks[i * block_size ..][0..block_size]);
123 h.update(blocks[i * alignment ..][0..block_size]);
124 }126 }
125 const final = if (H.has_crypto_api) @as(u64, @truncate(h.finalInt())) else h.final();127 const final = if (H.has_crypto_api) @as(u64, @truncate(h.finalInt())) else h.final();
126 std.mem.doNotOptimizeAway(final);128 std.mem.doNotOptimizeAway(final);
127129
128 const end = timer.read();130 const elapsed_ns = timer.read();
129131
130 const elapsed_s = @as(f64, @floatFromInt(end - start)) / time.ns_per_s;132 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
131 const throughput = @as(u64, @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s));133 const size_float: f64 = @floatFromInt(block_size * block_count);
134 const throughput: u64 = @intFromFloat(size_float / elapsed_s);
132135
133 return Result{136 return Result{
134 .hash = final,137 .hash = final,
...@@ -144,7 +147,6 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize...@@ -144,7 +147,6 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize
144 const key_count = bytes / key_size;147 const key_count = bytes / key_size;
145148
146 var timer = try Timer.start();149 var timer = try Timer.start();
147 const start = timer.lap();
148150
149 var sum: u64 = 0;151 var sum: u64 = 0;
150 for (0..key_count) |i| {152 for (0..key_count) |i| {
...@@ -164,10 +166,59 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize...@@ -164,10 +166,59 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize
164 };166 };
165 sum +%= final;167 sum +%= final;
166 }168 }
167 const end = timer.read();169 const elapsed_ns = timer.read();
170
171 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
172 const size_float: f64 = @floatFromInt(key_count * key_size);
173 const throughput: u64 = @intFromFloat(size_float / elapsed_s);
174
175 std.mem.doNotOptimizeAway(sum);
176
177 return Result{
178 .hash = sum,
179 .throughput = throughput,
180 };
181}
182
183// the array and array pointer benchmarks for xxhash are very sensitive to in-lining,
184// if you see strange performance changes consider using `.never_inline` or `.always_inline`
185// to ensure the changes are not only due to the optimiser inlining the benchmark differently
186pub fn benchmarkHashSmallKeysArrayPtr(
187 comptime H: anytype,
188 comptime key_size: usize,
189 bytes: usize,
190 allocator: std.mem.Allocator,
191) !Result {
192 var blocks = try allocator.alloc(u8, bytes);
193 defer allocator.free(blocks);
194 random.bytes(blocks);
195
196 const key_count = bytes / key_size;
197
198 var timer = try Timer.start();
199
200 var sum: u64 = 0;
201 for (0..key_count) |i| {
202 const small_key = blocks[i * key_size ..][0..key_size];
203 const final: u64 = blk: {
204 if (H.init_u8s) |init| {
205 if (H.has_crypto_api) {
206 break :blk @truncate(H.ty.toInt(small_key, init[0..H.ty.key_length]));
207 } else {
208 break :blk H.ty.hash(init, small_key);
209 }
210 }
211 if (H.init_u64) |init| {
212 break :blk H.ty.hash(init, small_key);
213 }
214 break :blk H.ty.hash(small_key);
215 };
216 sum +%= final;
217 }
218 const elapsed_ns = timer.read();
168219
169 const elapsed_s = @as(f64, @floatFromInt(end - start)) / time.ns_per_s;220 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
170 const throughput = @as(u64, @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s));221 const throughput: u64 = @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s);
171222
172 std.mem.doNotOptimizeAway(sum);223 std.mem.doNotOptimizeAway(sum);
173224
...@@ -177,6 +228,95 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize...@@ -177,6 +228,95 @@ pub fn benchmarkHashSmallKeys(comptime H: anytype, key_size: usize, bytes: usize
177 };228 };
178}229}
179230
231// the array and array pointer benchmarks for xxhash are very sensitive to in-lining,
232// if you see strange performance changes consider using `.never_inline` or `.always_inline`
233// to ensure the changes are not only due to the optimiser inlining the benchmark differently
234pub fn benchmarkHashSmallKeysArray(
235 comptime H: anytype,
236 comptime key_size: usize,
237 bytes: usize,
238 allocator: std.mem.Allocator,
239) !Result {
240 var blocks = try allocator.alloc(u8, bytes);
241 defer allocator.free(blocks);
242 random.bytes(blocks);
243
244 const key_count = bytes / key_size;
245
246 var i: usize = 0;
247 var timer = try Timer.start();
248
249 var sum: u64 = 0;
250 while (i < key_count) : (i += 1) {
251 const small_key = blocks[i * key_size ..][0..key_size];
252 const final: u64 = blk: {
253 if (H.init_u8s) |init| {
254 if (H.has_crypto_api) {
255 break :blk @truncate(H.ty.toInt(small_key, init[0..H.ty.key_length]));
256 } else {
257 break :blk H.ty.hash(init, small_key.*);
258 }
259 }
260 if (H.init_u64) |init| {
261 break :blk H.ty.hash(init, small_key.*);
262 }
263 break :blk H.ty.hash(small_key.*);
264 };
265 sum +%= final;
266 }
267 const elapsed_ns = timer.read();
268
269 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
270 const throughput: u64 = @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s);
271
272 std.mem.doNotOptimizeAway(sum);
273
274 return Result{
275 .hash = sum,
276 .throughput = throughput,
277 };
278}
279
280pub fn benchmarkHashSmallApi(comptime H: anytype, key_size: usize, bytes: usize, allocator: std.mem.Allocator) !Result {
281 var blocks = try allocator.alloc(u8, bytes);
282 defer allocator.free(blocks);
283 random.bytes(blocks);
284
285 const key_count = bytes / key_size;
286
287 var timer = try Timer.start();
288
289 var sum: u64 = 0;
290 for (0..key_count) |i| {
291 const small_key = blocks[i * key_size ..][0..key_size];
292 const final: u64 = blk: {
293 if (H.init_u8s) |init| {
294 if (H.has_crypto_api) {
295 break :blk @truncate(H.ty.toInt(small_key, init[0..H.ty.key_length]));
296 } else {
297 break :blk H.ty.hashSmall(init, small_key);
298 }
299 }
300 if (H.init_u64) |init| {
301 break :blk H.ty.hashSmall(init, small_key);
302 }
303 break :blk H.ty.hashSmall(small_key);
304 };
305 sum +%= final;
306 }
307 const elapsed_ns = timer.read();
308
309 const elapsed_s = @as(f64, @floatFromInt(elapsed_ns)) / time.ns_per_s;
310 const throughput: u64 = @intFromFloat(@as(f64, @floatFromInt(bytes)) / elapsed_s);
311
312 std.mem.doNotOptimizeAway(sum);
313
314 return Result{
315 .throughput = throughput,
316 .hash = sum,
317 };
318}
319
180fn usage() void {320fn usage() void {
181 std.debug.print(321 std.debug.print(
182 \\throughput_test [options]322 \\throughput_test [options]
...@@ -205,9 +345,12 @@ pub fn main() !void {...@@ -205,9 +345,12 @@ pub fn main() !void {
205345
206 var filter: ?[]u8 = "";346 var filter: ?[]u8 = "";
207 var count: usize = mode(128 * MiB);347 var count: usize = mode(128 * MiB);
208 var key_size: usize = 32;348 var key_size: ?usize = null;
209 var seed: u32 = 0;349 var seed: u32 = 0;
210 var test_iterative_only = false;350 var test_iterative_only = false;
351 var test_arrays = false;
352
353 const default_small_key_size = 32;
211354
212 var i: usize = 1;355 var i: usize = 1;
213 while (i < args.len) : (i += 1) {356 while (i < args.len) : (i += 1) {
...@@ -248,12 +391,14 @@ pub fn main() !void {...@@ -248,12 +391,14 @@ pub fn main() !void {
248 }391 }
249392
250 key_size = try std.fmt.parseUnsigned(usize, args[i], 10);393 key_size = try std.fmt.parseUnsigned(usize, args[i], 10);
251 if (key_size > block_size) {394 if (key_size.? > block_size) {
252 try stdout.print("key_size cannot exceed block size of {}\n", .{block_size});395 try stdout.print("key_size cannot exceed block size of {}\n", .{block_size});
253 std.os.exit(1);396 std.os.exit(1);
254 }397 }
255 } else if (std.mem.eql(u8, args[i], "--iterative-only")) {398 } else if (std.mem.eql(u8, args[i], "--iterative-only")) {
256 test_iterative_only = true;399 test_iterative_only = true;
400 } else if (std.mem.eql(u8, args[i], "--include-array")) {
401 test_arrays = true;
257 } else if (std.mem.eql(u8, args[i], "--help")) {402 } else if (std.mem.eql(u8, args[i], "--help")) {
258 usage();403 usage();
259 return;404 return;
...@@ -268,7 +413,7 @@ pub fn main() !void {...@@ -268,7 +413,7 @@ pub fn main() !void {
268 const allocator = gpa.allocator();413 const allocator = gpa.allocator();
269414
270 inline for (hashes) |H| {415 inline for (hashes) |H| {
271 if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) {416 if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) hash: {
272 if (!test_iterative_only or H.has_iterative_api) {417 if (!test_iterative_only or H.has_iterative_api) {
273 try stdout.print("{s}\n", .{H.name});418 try stdout.print("{s}\n", .{H.name});
274419
...@@ -281,9 +426,69 @@ pub fn main() !void {...@@ -281,9 +426,69 @@ pub fn main() !void {
281 }426 }
282427
283 if (!test_iterative_only) {428 if (!test_iterative_only) {
284 prng.seed(seed);429 if (key_size) |size| {
285 const result_small = try benchmarkHashSmallKeys(H, key_size, count, allocator);430 prng.seed(seed);
286 try stdout.print(" small keys: {:5} MiB/s [{x:0<16}]\n", .{ result_small.throughput / (1 * MiB), result_small.hash });431 const result_small = try benchmarkHashSmallKeys(H, size, count, allocator);
432 try stdout.print(" small keys: {:3}B {:5} MiB/s {} Hashes/s [{x:0<16}]\n", .{
433 size,
434 result_small.throughput / (1 * MiB),
435 result_small.throughput / size,
436 result_small.hash,
437 });
438
439 if (!test_arrays) break :hash;
440 if (H.has_anytype_api) |sizes| {
441 inline for (sizes) |exact_size| {
442 if (size == exact_size) {
443 prng.seed(seed);
444 const result_array = try benchmarkHashSmallKeysArray(H, exact_size, count, allocator);
445 prng.seed(seed);
446 const result_ptr = try benchmarkHashSmallKeysArrayPtr(H, exact_size, count, allocator);
447 try stdout.print(" array: {:5} MiB/s [{x:0<16}]\n", .{
448 result_array.throughput / (1 * MiB),
449 result_array.hash,
450 });
451 try stdout.print(" array ptr: {:5} MiB/s [{x:0<16}]\n", .{
452 result_ptr.throughput / (1 * MiB),
453 result_ptr.hash,
454 });
455 }
456 }
457 }
458 } else {
459 prng.seed(seed);
460 const result_small = try benchmarkHashSmallKeys(H, default_small_key_size, count, allocator);
461 try stdout.print(" small keys: {:3}B {:5} MiB/s {} Hashes/s [{x:0<16}]\n", .{
462 default_small_key_size,
463 result_small.throughput / (1 * MiB),
464 result_small.throughput / default_small_key_size,
465 result_small.hash,
466 });
467
468 if (!test_arrays) break :hash;
469 if (H.has_anytype_api) |sizes| {
470 try stdout.print(" array:\n", .{});
471 inline for (sizes) |exact_size| {
472 prng.seed(seed);
473 const result = try benchmarkHashSmallKeysArray(H, exact_size, count, allocator);
474 try stdout.print(" {d: >3}B {:5} MiB/s [{x:0<16}]\n", .{
475 exact_size,
476 result.throughput / (1 * MiB),
477 result.hash,
478 });
479 }
480 try stdout.print(" array ptr: \n", .{});
481 inline for (sizes) |exact_size| {
482 prng.seed(seed);
483 const result = try benchmarkHashSmallKeysArrayPtr(H, exact_size, count, allocator);
484 try stdout.print(" {d: >3}B {:5} MiB/s [{x:0<16}]\n", .{
485 exact_size,
486 result.throughput / (1 * MiB),
487 result.hash,
488 });
489 }
490 }
491 }
287 }492 }
288 }493 }
289 }494 }
lib/std/hash/xxhash.zig+340-138
...@@ -5,11 +5,7 @@ const expectEqual = std.testing.expectEqual;...@@ -5,11 +5,7 @@ const expectEqual = std.testing.expectEqual;
5const rotl = std.math.rotl;5const rotl = std.math.rotl;
66
7pub const XxHash64 = struct {7pub const XxHash64 = struct {
8 acc1: u64,8 accumulator: Accumulator,
9 acc2: u64,
10 acc3: u64,
11 acc4: u64,
12
13 seed: u64,9 seed: u64,
14 buf: [32]u8,10 buf: [32]u8,
15 buf_len: usize,11 buf_len: usize,
...@@ -21,20 +17,174 @@ pub const XxHash64 = struct {...@@ -21,20 +17,174 @@ pub const XxHash64 = struct {
21 const prime_4 = 0x85EBCA77C2B2AE63; // 0b100001011110101111001010011101111100001010110010101011100110001117 const prime_4 = 0x85EBCA77C2B2AE63; // 0b1000010111101011110010100111011111000010101100101010111001100011
22 const prime_5 = 0x27D4EB2F165667C5; // 0b001001111101010011101011001011110001011001010110011001111100010118 const prime_5 = 0x27D4EB2F165667C5; // 0b0010011111010100111010110010111100010110010101100110011111000101
2319
20 const Accumulator = struct {
21 acc1: u64,
22 acc2: u64,
23 acc3: u64,
24 acc4: u64,
25
26 fn init(seed: u64) Accumulator {
27 return .{
28 .acc1 = seed +% prime_1 +% prime_2,
29 .acc2 = seed +% prime_2,
30 .acc3 = seed,
31 .acc4 = seed -% prime_1,
32 };
33 }
34
35 fn updateEmpty(self: *Accumulator, input: anytype, comptime unroll_count: usize) usize {
36 var i: usize = 0;
37
38 if (unroll_count > 0) {
39 const unrolled_bytes = unroll_count * 32;
40 while (i + unrolled_bytes <= input.len) : (i += unrolled_bytes) {
41 inline for (0..unroll_count) |j| {
42 self.processStripe(input[i + j * 32 ..][0..32]);
43 }
44 }
45 }
46
47 while (i + 32 <= input.len) : (i += 32) {
48 self.processStripe(input[i..][0..32]);
49 }
50
51 return i;
52 }
53
54 fn processStripe(self: *Accumulator, buf: *const [32]u8) void {
55 self.acc1 = round(self.acc1, mem.readIntLittle(u64, buf[0..8]));
56 self.acc2 = round(self.acc2, mem.readIntLittle(u64, buf[8..16]));
57 self.acc3 = round(self.acc3, mem.readIntLittle(u64, buf[16..24]));
58 self.acc4 = round(self.acc4, mem.readIntLittle(u64, buf[24..32]));
59 }
60
61 fn merge(self: Accumulator) u64 {
62 var acc = rotl(u64, self.acc1, 1) +% rotl(u64, self.acc2, 7) +%
63 rotl(u64, self.acc3, 12) +% rotl(u64, self.acc4, 18);
64 acc = mergeAccumulator(acc, self.acc1);
65 acc = mergeAccumulator(acc, self.acc2);
66 acc = mergeAccumulator(acc, self.acc3);
67 acc = mergeAccumulator(acc, self.acc4);
68 return acc;
69 }
70
71 fn mergeAccumulator(acc: u64, other: u64) u64 {
72 const a = acc ^ round(0, other);
73 const b = a *% prime_1;
74 return b +% prime_4;
75 }
76 };
77
78 fn finalize(
79 unfinished: u64,
80 byte_count: usize,
81 partial: anytype,
82 ) u64 {
83 std.debug.assert(partial.len < 32);
84 var acc = unfinished +% @as(u64, byte_count) +% @as(u64, partial.len);
85
86 switch (partial.len) {
87 inline 0, 1, 2, 3 => |count| {
88 inline for (0..count) |i| acc = finalize1(acc, partial[i]);
89 return avalanche(acc);
90 },
91 inline 4, 5, 6, 7 => |count| {
92 acc = finalize4(acc, partial[0..4]);
93 inline for (4..count) |i| acc = finalize1(acc, partial[i]);
94 return avalanche(acc);
95 },
96 inline 8, 9, 10, 11 => |count| {
97 acc = finalize8(acc, partial[0..8]);
98 inline for (8..count) |i| acc = finalize1(acc, partial[i]);
99 return avalanche(acc);
100 },
101 inline 12, 13, 14, 15 => |count| {
102 acc = finalize8(acc, partial[0..8]);
103 acc = finalize4(acc, partial[8..12]);
104 inline for (12..count) |i| acc = finalize1(acc, partial[i]);
105 return avalanche(acc);
106 },
107 inline 16, 17, 18, 19 => |count| {
108 acc = finalize8(acc, partial[0..8]);
109 acc = finalize8(acc, partial[8..16]);
110 inline for (16..count) |i| acc = finalize1(acc, partial[i]);
111 return avalanche(acc);
112 },
113 inline 20, 21, 22, 23 => |count| {
114 acc = finalize8(acc, partial[0..8]);
115 acc = finalize8(acc, partial[8..16]);
116 acc = finalize4(acc, partial[16..20]);
117 inline for (20..count) |i| acc = finalize1(acc, partial[i]);
118 return avalanche(acc);
119 },
120 inline 24, 25, 26, 27 => |count| {
121 acc = finalize8(acc, partial[0..8]);
122 acc = finalize8(acc, partial[8..16]);
123 acc = finalize8(acc, partial[16..24]);
124 inline for (24..count) |i| acc = finalize1(acc, partial[i]);
125 return avalanche(acc);
126 },
127 inline 28, 29, 30, 31 => |count| {
128 acc = finalize8(acc, partial[0..8]);
129 acc = finalize8(acc, partial[8..16]);
130 acc = finalize8(acc, partial[16..24]);
131 acc = finalize4(acc, partial[24..28]);
132 inline for (28..count) |i| acc = finalize1(acc, partial[i]);
133 return avalanche(acc);
134 },
135 else => unreachable,
136 }
137 }
138
139 fn finalize8(v: u64, bytes: *const [8]u8) u64 {
140 var acc = v;
141 const lane = mem.readIntLittle(u64, bytes);
142 acc ^= round(0, lane);
143 acc = rotl(u64, acc, 27) *% prime_1;
144 acc +%= prime_4;
145 return acc;
146 }
147
148 fn finalize4(v: u64, bytes: *const [4]u8) u64 {
149 var acc = v;
150 const lane = @as(u64, mem.readIntLittle(u32, bytes));
151 acc ^= lane *% prime_1;
152 acc = rotl(u64, acc, 23) *% prime_2;
153 acc +%= prime_3;
154 return acc;
155 }
156
157 fn finalize1(v: u64, byte: u8) u64 {
158 var acc = v;
159 const lane = @as(u64, byte);
160 acc ^= lane *% prime_5;
161 acc = rotl(u64, acc, 11) *% prime_1;
162 return acc;
163 }
164
165 fn avalanche(value: u64) u64 {
166 var result = value ^ (value >> 33);
167 result *%= prime_2;
168 result ^= result >> 29;
169 result *%= prime_3;
170 result ^= result >> 32;
171
172 return result;
173 }
174
24 pub fn init(seed: u64) XxHash64 {175 pub fn init(seed: u64) XxHash64 {
25 return XxHash64{176 return XxHash64{
177 .accumulator = Accumulator.init(seed),
26 .seed = seed,178 .seed = seed,
27 .acc1 = seed +% prime_1 +% prime_2,
28 .acc2 = seed +% prime_2,
29 .acc3 = seed,
30 .acc4 = seed -% prime_1,
31 .buf = undefined,179 .buf = undefined,
32 .buf_len = 0,180 .buf_len = 0,
33 .byte_count = 0,181 .byte_count = 0,
34 };182 };
35 }183 }
36184
37 pub fn update(self: *XxHash64, input: []const u8) void {185 pub fn update(self: *XxHash64, input: anytype) void {
186 validateType(@TypeOf(input));
187
38 if (input.len < 32 - self.buf_len) {188 if (input.len < 32 - self.buf_len) {
39 @memcpy(self.buf[self.buf_len..][0..input.len], input);189 @memcpy(self.buf[self.buf_len..][0..input.len], input);
40 self.buf_len += input.len;190 self.buf_len += input.len;
...@@ -46,99 +196,54 @@ pub const XxHash64 = struct {...@@ -46,99 +196,54 @@ pub const XxHash64 = struct {
46 if (self.buf_len > 0) {196 if (self.buf_len > 0) {
47 i = 32 - self.buf_len;197 i = 32 - self.buf_len;
48 @memcpy(self.buf[self.buf_len..][0..i], input[0..i]);198 @memcpy(self.buf[self.buf_len..][0..i], input[0..i]);
49 self.processStripe(&self.buf);199 self.accumulator.processStripe(&self.buf);
50 self.buf_len = 0;200 self.byte_count += self.buf_len;
51 }201 }
52202
53 while (i + 32 <= input.len) : (i += 32) {203 i += self.accumulator.updateEmpty(input[i..], 32);
54 self.processStripe(input[i..][0..32]);204 self.byte_count += i;
55 }
56205
57 const remaining_bytes = input[i..];206 const remaining_bytes = input[i..];
58 @memcpy(self.buf[0..remaining_bytes.len], remaining_bytes);207 @memcpy(self.buf[0..remaining_bytes.len], remaining_bytes);
59 self.buf_len = remaining_bytes.len;208 self.buf_len = remaining_bytes.len;
60 }209 }
61210
62 inline fn processStripe(self: *XxHash64, buf: *const [32]u8) void {211 fn round(acc: u64, lane: u64) u64 {
63 self.acc1 = round(self.acc1, mem.readIntLittle(u64, buf[0..8]));
64 self.acc2 = round(self.acc2, mem.readIntLittle(u64, buf[8..16]));
65 self.acc3 = round(self.acc3, mem.readIntLittle(u64, buf[16..24]));
66 self.acc4 = round(self.acc4, mem.readIntLittle(u64, buf[24..32]));
67 self.byte_count += 32;
68 }
69
70 inline fn round(acc: u64, lane: u64) u64 {
71 const a = acc +% (lane *% prime_2);212 const a = acc +% (lane *% prime_2);
72 const b = rotl(u64, a, 31);213 const b = rotl(u64, a, 31);
73 return b *% prime_1;214 return b *% prime_1;
74 }215 }
75216
76 pub fn final(self: *XxHash64) u64 {217 pub fn final(self: *XxHash64) u64 {
77 var acc: u64 = undefined;218 const unfinished = if (self.byte_count < 32)
78219 self.seed +% prime_5
79 if (self.byte_count < 32) {220 else
80 acc = self.seed +% prime_5;221 self.accumulator.merge();
81 } else {
82 acc = rotl(u64, self.acc1, 1) +% rotl(u64, self.acc2, 7) +%
83 rotl(u64, self.acc3, 12) +% rotl(u64, self.acc4, 18);
84 acc = mergeAccumulator(acc, self.acc1);
85 acc = mergeAccumulator(acc, self.acc2);
86 acc = mergeAccumulator(acc, self.acc3);
87 acc = mergeAccumulator(acc, self.acc4);
88 }
89222
90 acc = acc +% @as(u64, self.byte_count) +% @as(u64, self.buf_len);223 return finalize(unfinished, self.byte_count, self.buf[0..self.buf_len]);
224 }
91225
92 var pos: usize = 0;226 const Size = enum {
93 while (pos + 8 <= self.buf_len) : (pos += 8) {227 small,
94 const lane = mem.readIntLittle(u64, self.buf[pos..][0..8]);228 large,
95 acc ^= round(0, lane);229 unknown,
96 acc = rotl(u64, acc, 27) *% prime_1;230 };
97 acc +%= prime_4;
98 }
99231
100 if (pos + 4 <= self.buf_len) {232 pub fn hash(seed: u64, input: anytype) u64 {
101 const lane = @as(u64, mem.readIntLittle(u32, self.buf[pos..][0..4]));233 validateType(@TypeOf(input));
102 acc ^= lane *% prime_1;
103 acc = rotl(u64, acc, 23) *% prime_2;
104 acc +%= prime_3;
105 pos += 4;
106 }
107234
108 while (pos < self.buf_len) : (pos += 1) {235 if (input.len < 32) {
109 const lane = @as(u64, self.buf[pos]);236 return finalize(seed +% prime_5, 0, input);
110 acc ^= lane *% prime_5;237 } else {
111 acc = rotl(u64, acc, 11) *% prime_1;238 var hasher = Accumulator.init(seed);
239 const i = hasher.updateEmpty(input, 0);
240 return finalize(hasher.merge(), i, input[i..]);
112 }241 }
113
114 acc ^= acc >> 33;
115 acc *%= prime_2;
116 acc ^= acc >> 29;
117 acc *%= prime_3;
118 acc ^= acc >> 32;
119
120 return acc;
121 }
122
123 inline fn mergeAccumulator(acc: u64, other: u64) u64 {
124 const a = acc ^ round(0, other);
125 const b = a *% prime_1;
126 return b +% prime_4;
127 }
128
129 pub fn hash(seed: u64, input: []const u8) u64 {
130 var hasher = XxHash64.init(seed);
131 hasher.update(input);
132 return hasher.final();
133 }242 }
134};243};
135244
136pub const XxHash32 = struct {245pub const XxHash32 = struct {
137 acc1: u32,246 accumulator: Accumulator,
138 acc2: u32,
139 acc3: u32,
140 acc4: u32,
141
142 seed: u32,247 seed: u32,
143 buf: [16]u8,248 buf: [16]u8,
144 buf_len: usize,249 buf_len: usize,
...@@ -150,13 +255,57 @@ pub const XxHash32 = struct {...@@ -150,13 +255,57 @@ pub const XxHash32 = struct {
150 const prime_4 = 0x27D4EB2F; // 0b00100111110101001110101100101111255 const prime_4 = 0x27D4EB2F; // 0b00100111110101001110101100101111
151 const prime_5 = 0x165667B1; // 0b00010110010101100110011110110001256 const prime_5 = 0x165667B1; // 0b00010110010101100110011110110001
152257
258 const Accumulator = struct {
259 acc1: u32,
260 acc2: u32,
261 acc3: u32,
262 acc4: u32,
263
264 fn init(seed: u32) Accumulator {
265 return .{
266 .acc1 = seed +% prime_1 +% prime_2,
267 .acc2 = seed +% prime_2,
268 .acc3 = seed,
269 .acc4 = seed -% prime_1,
270 };
271 }
272
273 fn updateEmpty(self: *Accumulator, input: anytype, comptime unroll_count: usize) usize {
274 var i: usize = 0;
275
276 if (unroll_count > 0) {
277 const unrolled_bytes = unroll_count * 16;
278 while (i + unrolled_bytes <= input.len) : (i += unrolled_bytes) {
279 inline for (0..unroll_count) |j| {
280 self.processStripe(input[i + j * 16 ..][0..16]);
281 }
282 }
283 }
284
285 while (i + 16 <= input.len) : (i += 16) {
286 self.processStripe(input[i..][0..16]);
287 }
288
289 return i;
290 }
291
292 fn processStripe(self: *Accumulator, buf: *const [16]u8) void {
293 self.acc1 = round(self.acc1, mem.readIntLittle(u32, buf[0..4]));
294 self.acc2 = round(self.acc2, mem.readIntLittle(u32, buf[4..8]));
295 self.acc3 = round(self.acc3, mem.readIntLittle(u32, buf[8..12]));
296 self.acc4 = round(self.acc4, mem.readIntLittle(u32, buf[12..16]));
297 }
298
299 fn merge(self: Accumulator) u32 {
300 return rotl(u32, self.acc1, 1) +% rotl(u32, self.acc2, 7) +%
301 rotl(u32, self.acc3, 12) +% rotl(u32, self.acc4, 18);
302 }
303 };
304
153 pub fn init(seed: u32) XxHash32 {305 pub fn init(seed: u32) XxHash32 {
154 return XxHash32{306 return XxHash32{
307 .accumulator = Accumulator.init(seed),
155 .seed = seed,308 .seed = seed,
156 .acc1 = seed +% prime_1 +% prime_2,
157 .acc2 = seed +% prime_2,
158 .acc3 = seed,
159 .acc4 = seed -% prime_1,
160 .buf = undefined,309 .buf = undefined,
161 .buf_len = 0,310 .buf_len = 0,
162 .byte_count = 0,311 .byte_count = 0,
...@@ -164,6 +313,8 @@ pub const XxHash32 = struct {...@@ -164,6 +313,8 @@ pub const XxHash32 = struct {
164 }313 }
165314
166 pub fn update(self: *XxHash32, input: []const u8) void {315 pub fn update(self: *XxHash32, input: []const u8) void {
316 validateType(@TypeOf(input));
317
167 if (input.len < 16 - self.buf_len) {318 if (input.len < 16 - self.buf_len) {
168 @memcpy(self.buf[self.buf_len..][0..input.len], input);319 @memcpy(self.buf[self.buf_len..][0..input.len], input);
169 self.buf_len += input.len;320 self.buf_len += input.len;
...@@ -175,59 +326,85 @@ pub const XxHash32 = struct {...@@ -175,59 +326,85 @@ pub const XxHash32 = struct {
175 if (self.buf_len > 0) {326 if (self.buf_len > 0) {
176 i = 16 - self.buf_len;327 i = 16 - self.buf_len;
177 @memcpy(self.buf[self.buf_len..][0..i], input[0..i]);328 @memcpy(self.buf[self.buf_len..][0..i], input[0..i]);
178 self.processStripe(&self.buf);329 self.accumulator.processStripe(&self.buf);
330 self.byte_count += self.buf_len;
179 self.buf_len = 0;331 self.buf_len = 0;
180 }332 }
181333
182 while (i + 16 <= input.len) : (i += 16) {334 i += self.accumulator.updateEmpty(input[i..], 16);
183 self.processStripe(input[i..][0..16]);335 self.byte_count += i;
184 }
185336
186 const remaining_bytes = input[i..];337 const remaining_bytes = input[i..];
187 @memcpy(self.buf[0..remaining_bytes.len], remaining_bytes);338 @memcpy(self.buf[0..remaining_bytes.len], remaining_bytes);
188 self.buf_len = remaining_bytes.len;339 self.buf_len = remaining_bytes.len;
189 }340 }
190341
191 inline fn processStripe(self: *XxHash32, buf: *const [16]u8) void {342 fn round(acc: u32, lane: u32) u32 {
192 self.acc1 = round(self.acc1, mem.readIntLittle(u32, buf[0..4]));
193 self.acc2 = round(self.acc2, mem.readIntLittle(u32, buf[4..8]));
194 self.acc3 = round(self.acc3, mem.readIntLittle(u32, buf[8..12]));
195 self.acc4 = round(self.acc4, mem.readIntLittle(u32, buf[12..16]));
196 self.byte_count += 16;
197 }
198
199 inline fn round(acc: u32, lane: u32) u32 {
200 const a = acc +% (lane *% prime_2);343 const a = acc +% (lane *% prime_2);
201 const b = rotl(u32, a, 13);344 const b = rotl(u32, a, 13);
202 return b *% prime_1;345 return b *% prime_1;
203 }346 }
204347
205 pub fn final(self: *XxHash32) u32 {348 pub fn final(self: *XxHash32) u32 {
206 var acc: u32 = undefined;349 const unfinished = if (self.byte_count < 16)
350 self.seed +% prime_5
351 else
352 self.accumulator.merge();
207353
208 if (self.byte_count < 16) {354 return finalize(unfinished, self.byte_count, self.buf[0..self.buf_len]);
209 acc = self.seed +% prime_5;355 }
210 } else {356
211 acc = rotl(u32, self.acc1, 1) +% rotl(u32, self.acc2, 7) +%357 fn finalize(unfinished: u32, byte_count: usize, partial: anytype) u32 {
212 rotl(u32, self.acc3, 12) +% rotl(u32, self.acc4, 18);358 std.debug.assert(partial.len < 16);
359 var acc = unfinished +% @as(u32, @intCast(byte_count)) +% @as(u32, @intCast(partial.len));
360
361 switch (partial.len) {
362 inline 0, 1, 2, 3 => |count| {
363 inline for (0..count) |i| acc = finalize1(acc, partial[i]);
364 return avalanche(acc);
365 },
366 inline 4, 5, 6, 7 => |count| {
367 acc = finalize4(acc, partial[0..4]);
368 inline for (4..count) |i| acc = finalize1(acc, partial[i]);
369 return avalanche(acc);
370 },
371 inline 8, 9, 10, 11 => |count| {
372 acc = finalize4(acc, partial[0..4]);
373 acc = finalize4(acc, partial[4..8]);
374 inline for (8..count) |i| acc = finalize1(acc, partial[i]);
375 return avalanche(acc);
376 },
377 inline 12, 13, 14, 15 => |count| {
378 acc = finalize4(acc, partial[0..4]);
379 acc = finalize4(acc, partial[4..8]);
380 acc = finalize4(acc, partial[8..12]);
381 inline for (12..count) |i| acc = finalize1(acc, partial[i]);
382 return avalanche(acc);
383 },
384 else => unreachable,
213 }385 }
214386
215 acc = acc +% @as(u32, @intCast(self.byte_count)) +% @as(u32, @intCast(self.buf_len));387 return avalanche(acc);
388 }
216389
217 var pos: usize = 0;390 fn finalize4(v: u32, bytes: *const [4]u8) u32 {
218 while (pos + 4 <= self.buf_len) : (pos += 4) {391 var acc = v;
219 const lane = mem.readIntLittle(u32, self.buf[pos..][0..4]);392 const lane = mem.readIntLittle(u32, bytes);
220 acc +%= lane *% prime_3;393 acc +%= lane *% prime_3;
221 acc = rotl(u32, acc, 17) *% prime_4;394 acc = rotl(u32, acc, 17) *% prime_4;
222 }395 return acc;
396 }
223397
224 while (pos < self.buf_len) : (pos += 1) {398 fn finalize1(v: u32, byte: u8) u32 {
225 const lane = @as(u32, self.buf[pos]);399 var acc = v;
226 acc +%= lane *% prime_5;400 const lane = @as(u32, byte);
227 acc = rotl(u32, acc, 11) *% prime_1;401 acc +%= lane *% prime_5;
228 }402 acc = rotl(u32, acc, 11) *% prime_1;
403 return acc;
404 }
229405
230 acc ^= acc >> 15;406 fn avalanche(value: u32) u32 {
407 var acc = value ^ value >> 15;
231 acc *%= prime_2;408 acc *%= prime_2;
232 acc ^= acc >> 13;409 acc ^= acc >> 13;
233 acc *%= prime_3;410 acc *%= prime_3;
...@@ -236,33 +413,58 @@ pub const XxHash32 = struct {...@@ -236,33 +413,58 @@ pub const XxHash32 = struct {
236 return acc;413 return acc;
237 }414 }
238415
239 pub fn hash(seed: u32, input: []const u8) u32 {416 pub fn hash(seed: u32, input: anytype) u32 {
240 var hasher = XxHash32.init(seed);417 validateType(@TypeOf(input));
241 hasher.update(input);418
242 return hasher.final();419 if (input.len < 16) {
420 return finalize(seed +% prime_5, 0, input);
421 } else {
422 var hasher = Accumulator.init(seed);
423 const i = hasher.updateEmpty(input, 0);
424 return finalize(hasher.merge(), i, input[i..]);
425 }
243 }426 }
244};427};
245428
429fn validateType(comptime T: type) void {
430 comptime {
431 if (!((std.meta.trait.isSlice(T) or
432 std.meta.trait.is(.Array)(T) or
433 std.meta.trait.isPtrTo(.Array)(T)) and
434 std.meta.Elem(T) == u8))
435 {
436 @compileError("expect a slice, array or pointer to array of u8, got " ++ @typeName(T));
437 }
438 }
439}
440
441fn testExpect(comptime H: type, seed: anytype, input: []const u8, expected: u64) !void {
442 try expectEqual(expected, H.hash(0, input));
443
444 var hasher = H.init(seed);
445 hasher.update(input);
446 try expectEqual(expected, hasher.final());
447}
448
246test "xxhash64" {449test "xxhash64" {
247 const hash = XxHash64.hash;450 const H = XxHash64;
248451 try testExpect(H, 0, "", 0xef46db3751d8e999);
249 try expectEqual(hash(0, ""), 0xef46db3751d8e999);452 try testExpect(H, 0, "a", 0xd24ec4f1a98c6e5b);
250 try expectEqual(hash(0, "a"), 0xd24ec4f1a98c6e5b);453 try testExpect(H, 0, "abc", 0x44bc2cf5ad770999);
251 try expectEqual(hash(0, "abc"), 0x44bc2cf5ad770999);454 try testExpect(H, 0, "message digest", 0x066ed728fceeb3be);
252 try expectEqual(hash(0, "message digest"), 0x066ed728fceeb3be);455 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0xcfe1f278fa89835c);
253 try expectEqual(hash(0, "abcdefghijklmnopqrstuvwxyz"), 0xcfe1f278fa89835c);456 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0xaaa46907d3047814);
254 try expectEqual(hash(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0xaaa46907d3047814);457 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0xe04a477f19ee145d);
255 try expectEqual(hash(0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0xe04a477f19ee145d);
256}458}
257459
258test "xxhash32" {460test "xxhash32" {
259 const hash = XxHash32.hash;461 const H = XxHash32;
260462
261 try expectEqual(hash(0, ""), 0x02cc5d05);463 try testExpect(H, 0, "", 0x02cc5d05);
262 try expectEqual(hash(0, "a"), 0x550d7456);464 try testExpect(H, 0, "a", 0x550d7456);
263 try expectEqual(hash(0, "abc"), 0x32d153ff);465 try testExpect(H, 0, "abc", 0x32d153ff);
264 try expectEqual(hash(0, "message digest"), 0x7c948494);466 try testExpect(H, 0, "message digest", 0x7c948494);
265 try expectEqual(hash(0, "abcdefghijklmnopqrstuvwxyz"), 0x63a14d5f);467 try testExpect(H, 0, "abcdefghijklmnopqrstuvwxyz", 0x63a14d5f);
266 try expectEqual(hash(0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x9c285e64);468 try testExpect(H, 0, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 0x9c285e64);
267 try expectEqual(hash(0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x9c05f475);469 try testExpect(H, 0, "12345678901234567890123456789012345678901234567890123456789012345678901234567890", 0x9c05f475);
268}470}