authorgravatar for marc@tiehu.isMarc Tiehuis <marc@tiehu.is> 2019-07-01 23:23:26+12:00
committergravatar for sahnvour@pm.meSahnvour <sahnvour@pm.me> 2019-08-04 12:34:05+02:00
log83dffc70afe4956c56f570ce5c854b17cbd6f218
tree4da3ccfc0de4043296d610e341536329062e0250
parentc9ce43f59fc777055612aeea58db0849390bc204

Add iterative wyhash api

3 files changed, 113 insertions(+), 63 deletions(-)

std/hash.zig+3-1
......@@ -17,6 +17,7 @@ pub const SipHash128 = siphash.SipHash128;
1717pub const murmur = @import("hash/murmur.zig");
1818pub const Murmur2_32 = murmur.Murmur2_32;
1919
20
2021pub const Murmur2_64 = murmur.Murmur2_64;
2122pub const Murmur3_32 = murmur.Murmur3_32;
2223
......@@ -24,7 +25,8 @@ pub const cityhash = @import("hash/cityhash.zig");
2425pub const CityHash32 = cityhash.CityHash32;
2526pub const CityHash64 = cityhash.CityHash64;
2627
27pub const wyhash = @import("hash/wyhash.zig").hash;
28const wyhash = @import("hash/wyhash.zig");
29pub const Wyhash = wyhash.Wyhash;
2830
2931test "hash" {
3032 _ = @import("hash/adler.zig");
std/hash/wyhash.zig+107-59
......@@ -10,7 +10,7 @@ const primes = [_]u64{
1010};
1111
1212fn read_bytes(comptime bytes: u8, data: []const u8) u64 {
13 return mem.readVarInt(u64, data[0..bytes], @import("builtin").endian);
13 return mem.readVarInt(u64, data[0..bytes], .Little);
1414}
1515
1616fn read_8bytes_swapped(data: []const u8) u64 {
......@@ -18,7 +18,7 @@ fn read_8bytes_swapped(data: []const u8) u64 {
1818}
1919
2020fn mum(a: u64, b: u64) u64 {
21 var r: u128 = @intCast(u128, a) * @intCast(u128, b);
21 var r = std.math.mulWide(u64, a, b);
2222 r = (r >> 64) ^ r;
2323 return @truncate(u64, r);
2424}
......@@ -31,69 +31,117 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {
3131 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);
3232}
3333
34pub fn hash(key: []const u8, initial_seed: u64) u64 {
35 var seed = initial_seed;
34pub const Wyhash = struct {
35 seed: u64,
3636
37 var i: usize = 0;
38 while (i + 32 <= key.len) : (i += 32) {
39 seed = mix0(
40 read_bytes(8, key[i..]),
41 read_bytes(8, key[i + 8 ..]),
42 seed,
37 buf: [32]u8,
38 buf_len: usize,
39 msg_len: usize,
40
41 pub fn init(seed: u64) Wyhash {
42 return Wyhash{
43 .seed = seed,
44 .buf = undefined,
45 .buf_len = 0,
46 .msg_len = 0,
47 };
48 }
49
50 fn round(self: *Wyhash, b: []const u8) void {
51 std.debug.assert(b.len == 32);
52
53 self.seed = mix0(
54 read_bytes(8, b[0..]),
55 read_bytes(8, b[8..]),
56 self.seed,
4357 ) ^ mix1(
44 read_bytes(8, key[i + 16 ..]),
45 read_bytes(8, key[i + 24 ..]),
46 seed,
58 read_bytes(8, b[16..]),
59 read_bytes(8, b[24..]),
60 self.seed,
4761 );
4862 }
4963
50 const rem_len = @truncate(u5, key.len);
51 const rem_key = key[i..];
52 seed = switch (rem_len) {
53 0 => seed,
54 1 => mix0(read_bytes(1, rem_key), primes[4], seed),
55 2 => mix0(read_bytes(2, rem_key), primes[4], seed),
56 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed),
57 4 => mix0(read_bytes(4, rem_key), primes[4], seed),
58 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed),
59 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed),
60 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed),
61 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),
62 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),
63 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),
64 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed),
65 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),
66 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed),
67 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed),
68 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed),
69 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),
70 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),
71 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),
72 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed),
73 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),
74 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed),
75 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed),
76 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed),
77 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),
78 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),
79 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),
80 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed),
81 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),
82 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed),
83 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),
84 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),
85 };
86
87 return mum(seed ^ key.len, primes[4]);
88}
64 pub fn update(self: *Wyhash, b: []const u8) void {
65 var off: usize = 0;
66
67 // Partial from previous.
68 if (self.buf_len != 0 and self.buf_len + b.len > 32) {
69 off += 32 - self.buf_len;
70 mem.copy(u8, self.buf[self.buf_len..], b[0..off]);
71 self.round(self.buf[0..]);
72 self.buf_len = 0;
73 }
74
75 // Full middle blocks.
76 while (off + 32 <= b.len) : (off += 32) {
77 @inlineCall(self.round, b[off .. off + 32]);
78 }
79
80 // Remainder for next pass.
81 mem.copy(u8, self.buf[self.buf_len..], b[off..]);
82 self.buf_len += @intCast(u8, b[off..].len);
83 self.msg_len += b.len;
84 }
85
86 pub fn final(self: *Wyhash) u64 {
87 const seed = self.seed;
88 const rem_len = @intCast(u5, self.buf_len);
89 const rem_key = self.buf[0..self.buf_len];
90
91 self.seed = switch (rem_len) {
92 0 => seed,
93 1 => mix0(read_bytes(1, rem_key), primes[4], seed),
94 2 => mix0(read_bytes(2, rem_key), primes[4], seed),
95 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed),
96 4 => mix0(read_bytes(4, rem_key), primes[4], seed),
97 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed),
98 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed),
99 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed),
100 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),
101 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),
102 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),
103 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed),
104 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),
105 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed),
106 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed),
107 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed),
108 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),
109 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),
110 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),
111 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed),
112 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),
113 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed),
114 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed),
115 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed),
116 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),
117 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),
118 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),
119 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed),
120 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),
121 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed),
122 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),
123 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),
124 };
125
126 return mum(self.seed ^ self.msg_len, primes[4]);
127 }
128
129 pub fn hash(seed: u64, input: []const u8) u64 {
130 var c = Wyhash.init(seed);
131 c.update(input);
132 return c.final();
133 }
134};
89135
90136test "test vectors" {
91137 const expectEqual = std.testing.expectEqual;
92 expectEqual(hash("", 0), 0x0);
93 expectEqual(hash("a", 1), 0xbed235177f41d328);
94 expectEqual(hash("abc", 2), 0xbe348debe59b27c3);
95 expectEqual(hash("message digest", 3), 0x37320f657213a290);
96 expectEqual(hash("abcdefghijklmnopqrstuvwxyz", 4), 0xd0b270e1d8a7019c);
97 expectEqual(hash("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", 5), 0x602a1894d3bbfe7f);
98 expectEqual(hash("12345678901234567890123456789012345678901234567890123456789012345678901234567890", 6), 0x829e9c148b75970e);
138 const hash = Wyhash.hash;
139
140 expectEqual(hash(0, ""), 0x0);
141 expectEqual(hash(1, "a"), 0xbed235177f41d328);
142 expectEqual(hash(2, "abc"), 0xbe348debe59b27c3);
143 expectEqual(hash(3, "message digest"), 0x37320f657213a290);
144 expectEqual(hash(4, "abcdefghijklmnopqrstuvwxyz"), 0xd0b270e1d8a7019c);
145 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
146 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
99147}
std/hash_map.zig+3-3
......@@ -5,7 +5,7 @@ const testing = std.testing;
55const math = std.math;
66const mem = std.mem;
77const meta = std.meta;
8const wyhash = std.hash.wyhash;
8const Wyhash = std.hash.Wyhash;
99const Allocator = mem.Allocator;
1010const builtin = @import("builtin");
1111
......@@ -557,7 +557,7 @@ pub fn autoHash(key: var, seed: u64) u64 {
557557 builtin.TypeId.EnumLiteral,
558558 => @compileError("cannot hash this type"),
559559
560 builtin.TypeId.Int => return wyhash(std.mem.asBytes(&key), seed),
560 builtin.TypeId.Int => return Wyhash.hash(seed, std.mem.asBytes(&key)),
561561
562562 builtin.TypeId.Float => |info| return autoHash(@bitCast(@IntType(false, info.bits), key), seed),
563563
......@@ -594,7 +594,7 @@ pub fn autoHash(key: var, seed: u64) u64 {
594594 // If there's no unused bits in the child type, we can just hash
595595 // this as an array of bytes.
596596 if (info.child.bit_count % 8 == 0) {
597 return wyhash(mem.asBytes(&key), seed);
597 return Wyhash.hash(seed, mem.asBytes(&key));
598598 }
599599
600600 // Otherwise, hash every element.