Commit 90e921f7a7356cda56e98367687889eb34ce0384 | zig

author	Sahnvour <sahnvour@pm.me> 2019-08-24 12:52:00+02:00
committer	Sahnvour <sahnvour@pm.me> 2019-08-24 15:37:47+02:00
log	90e921f7a7356cda56e98367687889eb34ce0384
tree	3d3d9977d38a52f8c669a954c0f2fae811f4599a
parent	4c882e731f26aa595a2b6c17725f01bbc1974e37

wyhash: readd the stateful streaming version so that both are available

and rename the stateless one so that's it is not the default option

3 files changed, 157 insertions(+), 5 deletions(-)

std
- hash.zig
- hash
  - benchmark.zig
  - wyhash.zig

std/hash.zig+1

...	...	@@ -29,6 +29,7 @@ pub const CityHash64 = cityhash.CityHash64;
29	29
30	30	const wyhash = @import("hash/wyhash.zig");
31	31	pub const Wyhash = wyhash.Wyhash;
	32	pub const WyhashStateless = wyhash.WyhashStateless;
32	33
33	34	test "hash" {
34	35	_ = @import("hash/adler.zig");

std/hash/benchmark.zig+5

...	...	@@ -28,6 +28,11 @@ const hashes = [_]Hash{
28	28	.name = "wyhash",
29	29	.init_u64 = 0,
30	30	},
	31	Hash{
	32	.ty = hash.WyhashStateless,
	33	.name = "wyhash-stateless",
	34	.init_u64 = 0,
	35	},
31	36	Hash{
32	37	.ty = hash.SipHash64(1, 3),
33	38	.name = "siphash(1,3)",

std/hash/wyhash.zig+151-5

...	...	@@ -31,13 +31,20 @@ fn mix1(a: u64, b: u64, seed: u64) u64 {
31	31	return mum(a ^ seed ^ primes[2], b ^ seed ^ primes[3]);
32	32	}
33	33
	34	/// Fast non-cryptographic 64bit hash function.
	35	/// See https://github.com/wangyi-fudan/wyhash
34	36	pub const Wyhash = struct {
35	37	seed: u64,
	38
	39	buf: [32]u8,
	40	buf_len: usize,
36	41	msg_len: usize,
37	42
38	43	pub fn init(seed: u64) Wyhash {
39	44	return Wyhash{
40	45	.seed = seed,
	46	.buf = undefined,
	47	.buf_len = 0,
41	48	.msg_len = 0,
42	49	};
43	50	}
...	...	@@ -56,7 +63,110 @@ pub const Wyhash = struct {
56	63	);
57	64	}
58	65
59		fn partial(self: *Wyhash, b: []const u8) void {
	66	pub fn update(self: *Wyhash, b: []const u8) void {
	67	var off: usize = 0;
	68
	69	// Partial from previous.
	70	if (self.buf_len != 0 and self.buf_len + b.len > 32) {
	71	off += 32 - self.buf_len;
	72	mem.copy(u8, self.buf[self.buf_len..], b[0..off]);
	73	self.round(self.buf[0..]);
	74	self.buf_len = 0;
	75	}
	76
	77	// Full middle blocks.
	78	while (off + 32 <= b.len) : (off += 32) {
	79	@inlineCall(self.round, b[off .. off + 32]);
	80	}
	81
	82	// Remainder for next pass.
	83	mem.copy(u8, self.buf[self.buf_len..], b[off..]);
	84	self.buf_len += @intCast(u8, b[off..].len);
	85	self.msg_len += b.len;
	86	}
	87
	88	pub fn final(self: *Wyhash) u64 {
	89	const seed = self.seed;
	90	const rem_len = @intCast(u5, self.buf_len);
	91	const rem_key = self.buf[0..self.buf_len];
	92
	93	self.seed = switch (rem_len) {
	94	0 => seed,
	95	1 => mix0(read_bytes(1, rem_key), primes[4], seed),
	96	2 => mix0(read_bytes(2, rem_key), primes[4], seed),
	97	3 => mix0((read_bytes(2, rem_key) << 8) \| read_bytes(1, rem_key[2..]), primes[4], seed),
	98	4 => mix0(read_bytes(4, rem_key), primes[4], seed),
	99	5 => mix0((read_bytes(4, rem_key) << 8) \| read_bytes(1, rem_key[4..]), primes[4], seed),
	100	6 => mix0((read_bytes(4, rem_key) << 16) \| read_bytes(2, rem_key[4..]), primes[4], seed),
	101	7 => mix0((read_bytes(4, rem_key) << 24) \| (read_bytes(2, rem_key[4..]) << 8) \| read_bytes(1, rem_key[6..]), primes[4], seed),
	102	8 => mix0(read_8bytes_swapped(rem_key), primes[4], seed),
	103	9 => mix0(read_8bytes_swapped(rem_key), read_bytes(1, rem_key[8..]), seed),
	104	10 => mix0(read_8bytes_swapped(rem_key), read_bytes(2, rem_key[8..]), seed),
	105	11 => mix0(read_8bytes_swapped(rem_key), (read_bytes(2, rem_key[8..]) << 8) \| read_bytes(1, rem_key[10..]), seed),
	106	12 => mix0(read_8bytes_swapped(rem_key), read_bytes(4, rem_key[8..]), seed),
	107	13 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 8) \| read_bytes(1, rem_key[12..]), seed),
	108	14 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 16) \| read_bytes(2, rem_key[12..]), seed),
	109	15 => mix0(read_8bytes_swapped(rem_key), (read_bytes(4, rem_key[8..]) << 24) \| (read_bytes(2, rem_key[12..]) << 8) \| read_bytes(1, rem_key[14..]), seed),
	110	16 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed),
	111	17 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(1, rem_key[16..]), primes[4], seed),
	112	18 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(2, rem_key[16..]), primes[4], seed),
	113	19 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(2, rem_key[16..]) << 8) \| read_bytes(1, rem_key[18..]), primes[4], seed),
	114	20 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_bytes(4, rem_key[16..]), primes[4], seed),
	115	21 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 8) \| read_bytes(1, rem_key[20..]), primes[4], seed),
	116	22 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 16) \| read_bytes(2, rem_key[20..]), primes[4], seed),
	117	23 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1((read_bytes(4, rem_key[16..]) << 24) \| (read_bytes(2, rem_key[20..]) << 8) \| read_bytes(1, rem_key[22..]), primes[4], seed),
	118	24 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), primes[4], seed),
	119	25 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(1, rem_key[24..]), seed),
	120	26 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(2, rem_key[24..]), seed),
	121	27 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(2, rem_key[24..]) << 8) \| read_bytes(1, rem_key[26..]), seed),
	122	28 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), read_bytes(4, rem_key[24..]), seed),
	123	29 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 8) \| read_bytes(1, rem_key[28..]), seed),
	124	30 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 16) \| read_bytes(2, rem_key[28..]), seed),
	125	31 => mix0(read_8bytes_swapped(rem_key), read_8bytes_swapped(rem_key[8..]), seed) ^ mix1(read_8bytes_swapped(rem_key[16..]), (read_bytes(4, rem_key[24..]) << 24) \| (read_bytes(2, rem_key[28..]) << 8) \| read_bytes(1, rem_key[30..]), seed),
	126	};
	127
	128	return mum(self.seed ^ self.msg_len, primes[4]);
	129	}
	130
	131	pub fn hash(seed: u64, input: []const u8) u64 {
	132	var c = Wyhash.init(seed);
	133	@inlineCall(c.update, input);
	134	return @inlineCall(c.final);
	135	}
	136	};
	137
	138	/// Wyhash version where state is not preserved between successive `update`
	139	/// calls, ie. it will have different results between hashing the data in
	140	/// one or several steps.
	141	/// This allows it to be faster.
	142	pub const WyhashStateless = struct {
	143	seed: u64,
	144	msg_len: usize,
	145
	146	const Self = @This();
	147
	148	pub fn init(seed: u64) Self {
	149	return Self{
	150	.seed = seed,
	151	.msg_len = 0,
	152	};
	153	}
	154
	155	fn round(self: *Self, b: []const u8) void {
	156	std.debug.assert(b.len == 32);
	157
	158	self.seed = mix0(
	159	read_bytes(8, b[0..]),
	160	read_bytes(8, b[8..]),
	161	self.seed,
	162	) ^ mix1(
	163	read_bytes(8, b[16..]),
	164	read_bytes(8, b[24..]),
	165	self.seed,
	166	);
	167	}
	168
	169	fn partial(self: *Self, b: []const u8) void {
60	170	const rem_key = b;
61	171	const rem_len = b.len;
62	172
...	...	@@ -98,7 +208,7 @@ pub const Wyhash = struct {
98	208	self.seed = seed;
99	209	}
100	210
101		pub fn update(self: *Wyhash, b: []const u8) void {
	211	pub fn update(self: *Self, b: []const u8) void {
102	212	var off: usize = 0;
103	213
104	214	// Full middle blocks.
...	...	@@ -110,19 +220,20 @@ pub const Wyhash = struct {
110	220	self.msg_len += b.len;
111	221	}
112	222
113		pub fn final(self: *Wyhash) u64 {
	223	pub fn final(self: *Self) u64 {
114	224	return mum(self.seed ^ self.msg_len, primes[4]);
115	225	}
116	226
117	227	pub fn hash(seed: u64, input: []const u8) u64 {
118		var c = Wyhash.init(seed);
	228	var c = Self.init(seed);
119	229	@inlineCall(c.update, input);
120	230	return @inlineCall(c.final);
121	231	}
122	232	};
123	233
	234	const expectEqual = std.testing.expectEqual;
	235
124	236	test "test vectors" {
125		const expectEqual = std.testing.expectEqual;
126	237	const hash = Wyhash.hash;
127	238
128	239	expectEqual(hash(0, ""), 0x0);
...	...	@@ -133,3 +244,38 @@ test "test vectors" {
133	244	expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
134	245	expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
135	246	}
	247
	248	test "test vectors streaming" {
	249	var wh = Wyhash.init(5);
	250	for ("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") \|e\| {
	251	wh.update(mem.asBytes(&e));
	252	}
	253	expectEqual(wh.final(), 0x602a1894d3bbfe7f);
	254
	255	const pattern = "1234567890";
	256	const count = 8;
	257	const result = 0x829e9c148b75970e;
	258	expectEqual(Wyhash.hash(6, pattern ** 8), result);
	259
	260	wh = Wyhash.init(6);
	261	var i: u32 = 0;
	262	while (i < count) : (i += 1) {
	263	wh.update(pattern);
	264	}
	265	expectEqual(wh.final(), result);
	266	}
	267
	268	test "test vectors stateless" {
	269	const hash = WyhashStateless.hash;
	270
	271	expectEqual(hash(0, ""), 0x0);
	272	expectEqual(hash(1, "a"), 0xbed235177f41d328);
	273	expectEqual(hash(2, "abc"), 0xbe348debe59b27c3);
	274	expectEqual(hash(3, "message digest"), 0x37320f657213a290);
	275	expectEqual(hash(4, "abcdefghijklmnopqrstuvwxyz"), 0xd0b270e1d8a7019c);
	276	expectEqual(hash(5, "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"), 0x602a1894d3bbfe7f);
	277	expectEqual(hash(6, "12345678901234567890123456789012345678901234567890123456789012345678901234567890"), 0x829e9c148b75970e);
	278
	279	// We don't check for the streaming API having the same results, as it is
	280	// not required to.
	281	}