authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-09-30 22:34:17+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-01 02:05:07+02:00
log8161de7fa44393c08cf96c9f9aa5379f98925472
treea748195f42f256885b10c03b4d37dfaa8ef71947
parentf1ad94437baaae40109f388a7d44d698c10a56d3

Implement ghash aggregated reduction

Performance increases from ~400 MiB/s to 450 MiB/s at the expense of extra code. Thus, aggregation is disabled on ReleaseSmall. Since the multiplication cost is significant compared to the reduction, aggregating more than 2 blocks is probably not worth it.

1 files changed, 118 insertions(+), 12 deletions(-)

lib/std/crypto/ghash.zig+118-12
...@@ -31,6 +31,13 @@ pub const Ghash = struct {...@@ -31,6 +31,13 @@ pub const Ghash = struct {
31 h1r: u64,31 h1r: u64,
32 h2r: u64,32 h2r: u64,
3333
34 hh0: u64 = undefined,
35 hh1: u64 = undefined,
36 hh2: u64 = undefined,
37 hh0r: u64 = undefined,
38 hh1r: u64 = undefined,
39 hh2r: u64 = undefined,
40
34 leftover: usize = 0,41 leftover: usize = 0,
35 buf: [block_size]u8 align(16) = undefined,42 buf: [block_size]u8 align(16) = undefined,
3643
...@@ -42,14 +49,49 @@ pub const Ghash = struct {...@@ -42,14 +49,49 @@ pub const Ghash = struct {
42 const h2 = h0 ^ h1;49 const h2 = h0 ^ h1;
43 const h2r = h0r ^ h1r;50 const h2r = h0r ^ h1r;
4451
45 return Ghash{52 if (std.builtin.mode == .ReleaseSmall) {
46 .h0 = h0,53 return Ghash{
47 .h1 = h1,54 .h0 = h0,
48 .h2 = h2,55 .h1 = h1,
49 .h0r = h0r,56 .h2 = h2,
50 .h1r = h1r,57 .h0r = h0r,
51 .h2r = h2r,58 .h1r = h1r,
52 };59 .h2r = h2r,
60 };
61 } else {
62 // Precompute H^2
63 var hh = Ghash{
64 .h0 = h0,
65 .h1 = h1,
66 .h2 = h2,
67 .h0r = h0r,
68 .h1r = h1r,
69 .h2r = h2r,
70 };
71 hh.update(key);
72 const hh1 = hh.y1;
73 const hh0 = hh.y0;
74 const hh1r = @bitReverse(u64, hh1);
75 const hh0r = @bitReverse(u64, hh0);
76 const hh2 = hh0 ^ hh1;
77 const hh2r = hh0r ^ hh1r;
78
79 return Ghash{
80 .h0 = h0,
81 .h1 = h1,
82 .h2 = h2,
83 .h0r = h0r,
84 .h1r = h1r,
85 .h2r = h2r,
86
87 .hh0 = hh0,
88 .hh1 = hh1,
89 .hh2 = hh2,
90 .hh0r = hh0r,
91 .hh1r = hh1r,
92 .hh2r = hh2r,
93 };
94 }
53 }95 }
5496
55 fn bmul(x: u64, y: u64) u64 {97 fn bmul(x: u64, y: u64) u64 {
...@@ -79,6 +121,71 @@ pub const Ghash = struct {...@@ -79,6 +121,71 @@ pub const Ghash = struct {
79 var y0 = st.y0;121 var y0 = st.y0;
80122
81 var i: usize = 0;123 var i: usize = 0;
124
125 // 2-blocks aggregated reduction
126 if (std.builtin.mode != .ReleaseSmall) {
127 while (i + 32 <= msg.len) : (i += 32) {
128 // B0 * H^2 unreduced
129 y1 ^= mem.readIntBig(u64, msg[i..][0..8]);
130 y0 ^= mem.readIntBig(u64, msg[i..][8..16]);
131
132 const y1r = @bitReverse(u64, y1);
133 const y0r = @bitReverse(u64, y0);
134 const y2 = y0 ^ y1;
135 const y2r = y0r ^ y1r;
136
137 var z0 = bmul(y0, st.hh0);
138 var z1 = bmul(y1, st.hh1);
139 var z2 = bmul(y2, st.hh2) ^ z0 ^ z1;
140 var z0h = bmul(y0r, st.hh0r);
141 var z1h = bmul(y1r, st.hh1r);
142 var z2h = bmul(y2r, st.hh2r) ^ z0h ^ z1h;
143
144 // B1 * H unreduced
145 const sy1 = mem.readIntBig(u64, msg[i..][16..24]);
146 const sy0 = mem.readIntBig(u64, msg[i..][24..32]);
147
148 const sy1r = @bitReverse(u64, sy1);
149 const sy0r = @bitReverse(u64, sy0);
150 const sy2 = sy0 ^ sy1;
151 const sy2r = sy0r ^ sy1r;
152
153 const sz0 = bmul(sy0, st.h0);
154 const sz1 = bmul(sy1, st.h1);
155 const sz2 = bmul(sy2, st.h2) ^ sz0 ^ sz1;
156 const sz0h = bmul(sy0r, st.h0r);
157 const sz1h = bmul(sy1r, st.h1r);
158 const sz2h = bmul(sy2r, st.h2r) ^ sz0h ^ sz1h;
159
160 // ((B0 * H^2) + B1 * H) (mod M)
161 z0 ^= sz0;
162 z1 ^= sz1;
163 z2 ^= sz2;
164 z0h ^= sz0h;
165 z1h ^= sz1h;
166 z2h ^= sz2h;
167 z0h = @bitReverse(u64, z0h) >> 1;
168 z1h = @bitReverse(u64, z1h) >> 1;
169 z2h = @bitReverse(u64, z2h) >> 1;
170
171 var v3 = z1h;
172 var v2 = z1 ^ z2h;
173 var v1 = z0h ^ z2;
174 var v0 = z0;
175
176 v3 = (v3 << 1) | (v2 >> 63);
177 v2 = (v2 << 1) | (v1 >> 63);
178 v1 = (v1 << 1) | (v0 >> 63);
179 v0 = (v0 << 1);
180
181 v2 ^= v0 ^ (v0 >> 1) ^ (v0 >> 2) ^ (v0 >> 7);
182 v1 ^= (v0 << 63) ^ (v0 << 62) ^ (v0 << 57);
183 y1 = v3 ^ v1 ^ (v1 >> 1) ^ (v1 >> 2) ^ (v1 >> 7);
184 y0 = v2 ^ (v1 << 63) ^ (v1 << 62) ^ (v1 << 57);
185 }
186 }
187
188 // single block
82 while (i + 16 <= msg.len) : (i += 16) {189 while (i + 16 <= msg.len) : (i += 16) {
83 y1 ^= mem.readIntBig(u64, msg[i..][0..8]);190 y1 ^= mem.readIntBig(u64, msg[i..][0..8]);
84 y0 ^= mem.readIntBig(u64, msg[i..][8..16]);191 y0 ^= mem.readIntBig(u64, msg[i..][8..16]);
...@@ -90,16 +197,15 @@ pub const Ghash = struct {...@@ -90,16 +197,15 @@ pub const Ghash = struct {
90197
91 const z0 = bmul(y0, st.h0);198 const z0 = bmul(y0, st.h0);
92 const z1 = bmul(y1, st.h1);199 const z1 = bmul(y1, st.h1);
93 var z2 = bmul(y2, st.h2);200 var z2 = bmul(y2, st.h2) ^ z0 ^ z1;
94 var z0h = bmul(y0r, st.h0r);201 var z0h = bmul(y0r, st.h0r);
95 var z1h = bmul(y1r, st.h1r);202 var z1h = bmul(y1r, st.h1r);
96 var z2h = bmul(y2r, st.h2r);203 var z2h = bmul(y2r, st.h2r) ^ z0h ^ z1h;
97 z2 ^= z0 ^ z1;
98 z2h ^= z0h ^ z1h;
99 z0h = @bitReverse(u64, z0h) >> 1;204 z0h = @bitReverse(u64, z0h) >> 1;
100 z1h = @bitReverse(u64, z1h) >> 1;205 z1h = @bitReverse(u64, z1h) >> 1;
101 z2h = @bitReverse(u64, z2h) >> 1;206 z2h = @bitReverse(u64, z2h) >> 1;
102207
208 // shift & reduce
103 var v3 = z1h;209 var v3 = z1h;
104 var v2 = z1 ^ z2h;210 var v2 = z1 ^ z2h;
105 var v1 = z0h ^ z2;211 var v1 = z0h ^ z2;