authorgravatar for sahnvour@pm.meSahnvour <sahnvour@pm.me> 2019-08-24 12:52:00+02:00
committergravatar for sahnvour@pm.meSahnvour <sahnvour@pm.me> 2019-08-24 15:37:47+02:00
log90e921f7a7356cda56e98367687889eb34ce0384
tree3d3d9977d38a52f8c669a954c0f2fae811f4599a
parent4c882e731f26aa595a2b6c17725f01bbc1974e37

wyhash: readd the stateful streaming version so that both are available

and rename the stateless one so that's it is not the default option

3 files changed, 157 insertions(+), 5 deletions(-)

std/hash.zig+1
...@@ -29,6 +29,7 @@ pub const CityHash64 = cityhash.CityHash64;...@@ -29,6 +29,7 @@ pub const CityHash64 = cityhash.CityHash64;
2929
30const wyhash = @import("hash/wyhash.zig");30const wyhash = @import("hash/wyhash.zig");
31pub const Wyhash = wyhash.Wyhash;31pub const Wyhash = wyhash.Wyhash;
32pub const WyhashStateless = wyhash.WyhashStateless;
3233
33test "hash" {34test "hash" {
34 _ = @import("hash/adler.zig");35 _ = @import("hash/adler.zig");
std/hash/benchmark.zig+5
...@@ -28,6 +28,11 @@ const hashes = [_]Hash{...@@ -28,6 +28,11 @@ const hashes = [_]Hash{
28 .name = "wyhash",28 .name = "wyhash",
29 .init_u64 = 0,29 .init_u64 = 0,
30 },30 },
31 Hash{
32 .ty = hash.WyhashStateless,
33 .name = "wyhash-stateless",
34 .init_u64 = 0,
35 },
31 Hash{36 Hash{
32 .ty = hash.SipHash64(1, 3),37 .ty = hash.SipHash64(1, 3),
33 .name = "siphash(1,3)",38 .name = "siphash(1,3)",
std/hash/wyhash.zig+151-5
...@@ -31,13 +31,20 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {...@@ -31,13 +31,20 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {
31 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);31 return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);
32}32}
3333
34/// Fast non-cryptographic 64bit hash function.
35/// See https://github.com/wangyi-fudan/wyhash
34pub const Wyhash = struct {36pub const Wyhash = struct {
35 seed: u64,37 seed: u64,
38
39 buf: [32]u8,
40 buf_len: usize,
36 msg_len: usize,41 msg_len: usize,
3742
38 pub fn init(seed: u64) Wyhash {43 pub fn init(seed: u64) Wyhash {
39 return Wyhash{44 return Wyhash{
40 .seed = seed,45 .seed = seed,
46 .buf = undefined,
47 .buf_len = 0,
41 .msg_len = 0,48 .msg_len = 0,
42 };49 };
43 }50 }
...@@ -56,7 +63,110 @@ pub const Wyhash = struct {...@@ -56,7 +63,110 @@ pub const Wyhash = struct {
56 );63 );
57 }64 }
5865
59 fn partial(self: *Wyhash, b: []const u8) void {66 pub fn update(self: *Wyhash, b: []const u8) void {
67 var off: usize = 0;
68
69 // Partial from previous.
70 if (self.buf_len != 0 and self.buf_len + b.len > 32) {
71 off += 32 - self.buf_len;
72 mem.copy(u8, self.buf[self.buf_len..], b[0..off]);
73 self.round(self.buf[0..]);
74 self.buf_len = 0;
75 }
76
77 // Full middle blocks.
78 while (off + 32 <= b.len) : (off += 32) {
79 @inlineCall(self.round, b[off .. off + 32]);
80 }
81
82 // Remainder for next pass.
83 mem.copy(u8, self.buf[self.buf_len..], b[off..]);
84 self.buf_len += @intCast(u8, b[off..].len);
85 self.msg_len += b.len;
86 }
87
88 pub fn final(self: *Wyhash) u64 {
89 const seed = self.seed;
90 const rem_len = @intCast(u5, self.buf_len);
91 const rem_key = self.buf[0..self.buf_len];
92
93 self.seed = switch (rem_len) {
94 0 => seed,
95 1 => mix0(read_bytes(1, rem_key), primes[4], seed),
96 2 => mix0(read_bytes(2, rem_key), primes[4], seed),
97 3 => mix0((read_bytes(2, rem_key) << 8) | read_bytes(1, rem_key[2..]), primes[4], seed),
98 4 => mix0(read_bytes(4, rem_key), primes[4], seed),
99 5 => mix0((read_bytes(4, rem_key) << 8) | read_bytes(1, rem_key[4..]), primes[4], seed),
100 6 => mix0((read_bytes(4, rem_key) << 16) | read_bytes(2, rem_key[4..]), primes[4], seed),
101 7 => mix0((read_bytes(4, rem_key) << 24) | (read_bytes(2, rem_key[4..]) << 8) | read_bytes(1, rem_key[6..]), primes[4], seed),
102 8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),
103 9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),
104 10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),
105 11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) | read_bytes(1, rem_key[10..]), seed),
106 12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),
107 13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) | read_bytes(1, rem_key[12..]), seed),
108 14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) | read_bytes(2, rem_key[12..]), seed),
109 15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) | (read_bytes(2, rem_key[12..]) << 8) | read_bytes(1, rem_key[14..]), seed),
110 16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),
111 17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),
112 18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),
113 19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) | read_bytes(1, rem_key[18..]), primes[4], seed),
114 20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),
115 21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) | read_bytes(1, rem_key[20..]), primes[4], seed),
116 22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) | read_bytes(2, rem_key[20..]), primes[4], seed),
117 23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) | (read_bytes(2, rem_key[20..]) << 8) | read_bytes(1, rem_key[22..]), primes[4], seed),
118 24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),
119 25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),
120 26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),
121 27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) | read_bytes(1, rem_key[26..]), seed),
122 28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),
123 29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) | read_bytes(1, rem_key[28..]), seed),
124 30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) | read_bytes(2, rem_key[28..]), seed),
125 31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) | (read_bytes(2, rem_key[28..]) << 8) | read_bytes(1, rem_key[30..]), seed),
126 };
127
128 return mum(self.seed ^ self.msg_len, primes[4]);
129 }
130
131 pub fn hash(seed: u64, input: []const u8) u64 {
132 var c = Wyhash.init(seed);
133 @inlineCall(c.update, input);
134 return @inlineCall(c.final);
135 }
136};
137
138/// Wyhash version where state is not preserved between successive `update`
139/// calls, ie. it will have different results between hashing the data in
140/// one or several steps.
141/// This allows it to be faster.
142pub const WyhashStateless = struct {
143 seed: u64,
144 msg_len: usize,
145
146 const Self = @This();
147
148 pub fn init(seed: u64) Self {
149 return Self{
150 .seed = seed,
151 .msg_len = 0,
152 };
153 }
154
155 fn round(self: *Self, b: []const u8) void {
156 std.debug.assert(b.len == 32);
157
158 self.seed = mix0(
159 read_bytes(8, b[0..]),
160 read_bytes(8, b[8..]),
161 self.seed,
162 ) ^ mix1(
163 read_bytes(8, b[16..]),
164 read_bytes(8, b[24..]),
165 self.seed,
166 );
167 }
168
169 fn partial(self: *Self, b: []const u8) void {
60 const rem_key = b;170 const rem_key = b;
61 const rem_len = b.len;171 const rem_len = b.len;
62172
...@@ -98,7 +208,7 @@ pub const Wyhash = struct {...@@ -98,7 +208,7 @@ pub const Wyhash = struct {
98 self.seed = seed;208 self.seed = seed;
99 }209 }
100210
101 pub fn update(self: *Wyhash, b: []const u8) void {211 pub fn update(self: *Self, b: []const u8) void {
102 var off: usize = 0;212 var off: usize = 0;
103213
104 // Full middle blocks.214 // Full middle blocks.
...@@ -110,19 +220,20 @@ pub const Wyhash = struct {...@@ -110,19 +220,20 @@ pub const Wyhash = struct {
110 self.msg_len += b.len;220 self.msg_len += b.len;
111 }221 }
112222
113 pub fn final(self: *Wyhash) u64 {223 pub fn final(self: *Self) u64 {
114 return mum(self.seed ^ self.msg_len, primes[4]);224 return mum(self.seed ^ self.msg_len, primes[4]);
115 }225 }
116226
117 pub fn hash(seed: u64, input: []const u8) u64 {227 pub fn hash(seed: u64, input: []const u8) u64 {
118 var c = Wyhash.init(seed);228 var c = Self.init(seed);
119 @inlineCall(c.update, input);229 @inlineCall(c.update, input);
120 return @inlineCall(c.final);230 return @inlineCall(c.final);
121 }231 }
122};232};
123233
234const expectEqual = std.testing.expectEqual;
235
124test "test vectors" {236test "test vectors" {
125 const expectEqual = std.testing.expectEqual;
126 const hash = Wyhash.hash;237 const hash = Wyhash.hash;
127238
128 expectEqual(hash(0, ""), 0x0);239 expectEqual(hash(0, ""), 0x0);
...@@ -133,3 +244,38 @@ test "test vectors" {...@@ -133,3 +244,38 @@ test "test vectors" {
133 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);244 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
134 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);245 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
135}246}
247
248test "test vectors streaming" {
249 var wh = Wyhash.init(5);
250 for ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") |e| {
251 wh.update(mem.asBytes(&e));
252 }
253 expectEqual(wh.final(), 0x602a1894d3bbfe7f);
254
255 const pattern = "1234567890";
256 const count = 8;
257 const result = 0x829e9c148b75970e;
258 expectEqual(Wyhash.hash(6, pattern ** 8), result);
259
260 wh = Wyhash.init(6);
261 var i: u32 = 0;
262 while (i < count) : (i += 1) {
263 wh.update(pattern);
264 }
265 expectEqual(wh.final(), result);
266}
267
268test "test vectors stateless" {
269 const hash = WyhashStateless.hash;
270
271 expectEqual(hash(0, ""), 0x0);
272 expectEqual(hash(1, "a"), 0xbed235177f41d328);
273 expectEqual(hash(2, "abc"), 0xbe348debe59b27c3);
274 expectEqual(hash(3, "message digest"), 0x37320f657213a290);
275 expectEqual(hash(4, "abcdefghijklmnopqrstuvwxyz"), 0xd0b270e1d8a7019c);
276 expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
277 expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
278
279 // We don't check for the streaming API having the same results, as it is
280 // not required to.
281}