Commit 8161de7fa44393c08cf96c9f9aa5379f98925472 | zig

author	Frank Denis <124872+jedisct1@users.noreply.github.com> 2020-09-30 22:34:17+02:00
committer	Frank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-01 02:05:07+02:00
log	8161de7fa44393c08cf96c9f9aa5379f98925472
tree	a748195f42f256885b10c03b4d37dfaa8ef71947
parent	f1ad94437baaae40109f388a7d44d698c10a56d3

Implement ghash aggregated reduction

Performance increases from ~400 MiB/s to 450 MiB/s at the expense of
extra code. Thus, aggregation is disabled on ReleaseSmall.

Since the multiplication cost is significant compared to the reduction,
aggregating more than 2 blocks is probably not worth it.

1 files changed, 118 insertions(+), 12 deletions(-)

lib
- std
  - crypto
    ghash.zig

lib/std/crypto/ghash.zig+118-12

...	...	@@ -31,6 +31,13 @@ pub const Ghash = struct {
31	31	h1r: u64,
32	32	h2r: u64,
33	33
	34	hh0: u64 = undefined,
	35	hh1: u64 = undefined,
	36	hh2: u64 = undefined,
	37	hh0r: u64 = undefined,
	38	hh1r: u64 = undefined,
	39	hh2r: u64 = undefined,
	40
34	41	leftover: usize = 0,
35	42	buf: [block_size]u8 align(16) = undefined,
36	43
...	...	@@ -42,14 +49,49 @@ pub const Ghash = struct {
42	49	const h2 = h0 ^ h1;
43	50	const h2r = h0r ^ h1r;
44	51
45		return Ghash{
46		.h0 = h0,
47		.h1 = h1,
48		.h2 = h2,
49		.h0r = h0r,
50		.h1r = h1r,
51		.h2r = h2r,
52		};
	52	if (std.builtin.mode == .ReleaseSmall) {
	53	return Ghash{
	54	.h0 = h0,
	55	.h1 = h1,
	56	.h2 = h2,
	57	.h0r = h0r,
	58	.h1r = h1r,
	59	.h2r = h2r,
	60	};
	61	} else {
	62	// Precompute H^2
	63	var hh = Ghash{
	64	.h0 = h0,
	65	.h1 = h1,
	66	.h2 = h2,
	67	.h0r = h0r,
	68	.h1r = h1r,
	69	.h2r = h2r,
	70	};
	71	hh.update(key);
	72	const hh1 = hh.y1;
	73	const hh0 = hh.y0;
	74	const hh1r = @bitReverse(u64, hh1);
	75	const hh0r = @bitReverse(u64, hh0);
	76	const hh2 = hh0 ^ hh1;
	77	const hh2r = hh0r ^ hh1r;
	78
	79	return Ghash{
	80	.h0 = h0,
	81	.h1 = h1,
	82	.h2 = h2,
	83	.h0r = h0r,
	84	.h1r = h1r,
	85	.h2r = h2r,
	86
	87	.hh0 = hh0,
	88	.hh1 = hh1,
	89	.hh2 = hh2,
	90	.hh0r = hh0r,
	91	.hh1r = hh1r,
	92	.hh2r = hh2r,
	93	};
	94	}
53	95	}
54	96
55	97	fn bmul(x: u64, y: u64) u64 {
...	...	@@ -79,6 +121,71 @@ pub const Ghash = struct {
79	121	var y0 = st.y0;
80	122
81	123	var i: usize = 0;
	124
	125	// 2-blocks aggregated reduction
	126	if (std.builtin.mode != .ReleaseSmall) {
	127	while (i + 32 <= msg.len) : (i += 32) {
	128	// B0 * H^2 unreduced
	129	y1 ^= mem.readIntBig(u64, msg[i..][0..8]);
	130	y0 ^= mem.readIntBig(u64, msg[i..][8..16]);
	131
	132	const y1r = @bitReverse(u64, y1);
	133	const y0r = @bitReverse(u64, y0);
	134	const y2 = y0 ^ y1;
	135	const y2r = y0r ^ y1r;
	136
	137	var z0 = bmul(y0, st.hh0);
	138	var z1 = bmul(y1, st.hh1);
	139	var z2 = bmul(y2, st.hh2) ^ z0 ^ z1;
	140	var z0h = bmul(y0r, st.hh0r);
	141	var z1h = bmul(y1r, st.hh1r);
	142	var z2h = bmul(y2r, st.hh2r) ^ z0h ^ z1h;
	143
	144	// B1 * H unreduced
	145	const sy1 = mem.readIntBig(u64, msg[i..][16..24]);
	146	const sy0 = mem.readIntBig(u64, msg[i..][24..32]);
	147
	148	const sy1r = @bitReverse(u64, sy1);
	149	const sy0r = @bitReverse(u64, sy0);
	150	const sy2 = sy0 ^ sy1;
	151	const sy2r = sy0r ^ sy1r;
	152
	153	const sz0 = bmul(sy0, st.h0);
	154	const sz1 = bmul(sy1, st.h1);
	155	const sz2 = bmul(sy2, st.h2) ^ sz0 ^ sz1;
	156	const sz0h = bmul(sy0r, st.h0r);
	157	const sz1h = bmul(sy1r, st.h1r);
	158	const sz2h = bmul(sy2r, st.h2r) ^ sz0h ^ sz1h;
	159
	160	// ((B0 * H^2) + B1 * H) (mod M)
	161	z0 ^= sz0;
	162	z1 ^= sz1;
	163	z2 ^= sz2;
	164	z0h ^= sz0h;
	165	z1h ^= sz1h;
	166	z2h ^= sz2h;
	167	z0h = @bitReverse(u64, z0h) >> 1;
	168	z1h = @bitReverse(u64, z1h) >> 1;
	169	z2h = @bitReverse(u64, z2h) >> 1;
	170
	171	var v3 = z1h;
	172	var v2 = z1 ^ z2h;
	173	var v1 = z0h ^ z2;
	174	var v0 = z0;
	175
	176	v3 = (v3 << 1) \| (v2 >> 63);
	177	v2 = (v2 << 1) \| (v1 >> 63);
	178	v1 = (v1 << 1) \| (v0 >> 63);
	179	v0 = (v0 << 1);
	180
	181	v2 ^= v0 ^ (v0 >> 1) ^ (v0 >> 2) ^ (v0 >> 7);
	182	v1 ^= (v0 << 63) ^ (v0 << 62) ^ (v0 << 57);
	183	y1 = v3 ^ v1 ^ (v1 >> 1) ^ (v1 >> 2) ^ (v1 >> 7);
	184	y0 = v2 ^ (v1 << 63) ^ (v1 << 62) ^ (v1 << 57);
	185	}
	186	}
	187
	188	// single block
82	189	while (i + 16 <= msg.len) : (i += 16) {
83	190	y1 ^= mem.readIntBig(u64, msg[i..][0..8]);
84	191	y0 ^= mem.readIntBig(u64, msg[i..][8..16]);
...	...	@@ -90,16 +197,15 @@ pub const Ghash = struct {
90	197
91	198	const z0 = bmul(y0, st.h0);
92	199	const z1 = bmul(y1, st.h1);
93		var z2 = bmul(y2, st.h2);
	200	var z2 = bmul(y2, st.h2) ^ z0 ^ z1;
94	201	var z0h = bmul(y0r, st.h0r);
95	202	var z1h = bmul(y1r, st.h1r);
96		var z2h = bmul(y2r, st.h2r);
97		z2 ^= z0 ^ z1;
98		z2h ^= z0h ^ z1h;
	203	var z2h = bmul(y2r, st.h2r) ^ z0h ^ z1h;
99	204	z0h = @bitReverse(u64, z0h) >> 1;
100	205	z1h = @bitReverse(u64, z1h) >> 1;
101	206	z2h = @bitReverse(u64, z2h) >> 1;
102	207
	208	// shift & reduce
103	209	var v3 = z1h;
104	210	var v2 = z1 ^ z2h;
105	211	var v1 = z0h ^ z2;