authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2023-05-24 08:38:28+02:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2023-05-24 08:38:28+02:00
log16dbb960fc8dada79cd42dd2ba1ebf0f66ccaaec
tree4a166ca84b061fd6d7bdeaf40b28ce066ab42814
parent3db3cf77904e664d589287602c14168a7a63f125
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.crypto: 2.5 times faster ghash and polyval on WebAssembly (#15835)

* std.crypto: faster ghash and polyval on WebAssembly Before: 91 MiB/s After : 243 MiB/s Some other platforms might benefit from this, but WebAssembly is the obvious one (simd128 doesn't make a difference).

1 files changed, 46 insertions(+), 2 deletions(-)

lib/std/crypto/ghash_polyval.zig+46-2
...@@ -155,8 +155,18 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {...@@ -155,8 +155,18 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
155 }155 }
156 }156 }
157157
158 // Software carryless multiplication of two 64-bit integers.158 /// clmulSoft128_64 is faster on platforms with no native 128-bit registers.
159 fn clmulSoft(x_: u128, y_: u128, comptime half: Selector) u128 {159 const clmulSoft = switch (builtin.cpu.arch) {
160 .wasm32, .wasm64 => clmulSoft128_64,
161 else => impl: {
162 const vector_size = std.simd.suggestVectorSize(u128) orelse 0;
163 if (vector_size < 128) break :impl clmulSoft128_64;
164 break :impl clmulSoft128;
165 },
166 };
167
168 // Software carryless multiplication of two 64-bit integers using native 128-bit registers.
169 fn clmulSoft128(x_: u128, y_: u128, comptime half: Selector) u128 {
160 const x = @truncate(u64, if (half == .hi or half == .hi_lo) x_ >> 64 else x_);170 const x = @truncate(u64, if (half == .hi or half == .hi_lo) x_ >> 64 else x_);
161 const y = @truncate(u64, if (half == .hi) y_ >> 64 else y_);171 const y = @truncate(u64, if (half == .hi) y_ >> 64 else y_);
162172
...@@ -186,6 +196,40 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {...@@ -186,6 +196,40 @@ fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
186 (z3 & 0x88888888888888888888888888888888) ^ extra;196 (z3 & 0x88888888888888888888888888888888) ^ extra;
187 }197 }
188198
199 // Software carryless multiplication of two 32-bit integers.
200 fn clmulSoft32(x: u32, y: u32) u64 {
201 const mulWide = math.mulWide;
202 const a0 = x & 0x11111111;
203 const a1 = x & 0x22222222;
204 const a2 = x & 0x44444444;
205 const a3 = x & 0x88888888;
206 const b0 = y & 0x11111111;
207 const b1 = y & 0x22222222;
208 const b2 = y & 0x44444444;
209 const b3 = y & 0x88888888;
210 const c0 = mulWide(u32, a0, b0) ^ mulWide(u32, a1, b3) ^ mulWide(u32, a2, b2) ^ mulWide(u32, a3, b1);
211 const c1 = mulWide(u32, a0, b1) ^ mulWide(u32, a1, b0) ^ mulWide(u32, a2, b3) ^ mulWide(u32, a3, b2);
212 const c2 = mulWide(u32, a0, b2) ^ mulWide(u32, a1, b1) ^ mulWide(u32, a2, b0) ^ mulWide(u32, a3, b3);
213 const c3 = mulWide(u32, a0, b3) ^ mulWide(u32, a1, b2) ^ mulWide(u32, a2, b1) ^ mulWide(u32, a3, b0);
214 return (c0 & 0x1111111111111111) | (c1 & 0x2222222222222222) | (c2 & 0x4444444444444444) | (c3 & 0x8888888888888888);
215 }
216
217 // Software carryless multiplication of two 128-bit integers using 64-bit registers.
218 fn clmulSoft128_64(x_: u128, y_: u128, comptime half: Selector) u128 {
219 const a = @truncate(u64, if (half == .hi or half == .hi_lo) x_ >> 64 else x_);
220 const b = @truncate(u64, if (half == .hi) y_ >> 64 else y_);
221 const a0 = @truncate(u32, a);
222 const a1 = @truncate(u32, a >> 32);
223 const b0 = @truncate(u32, b);
224 const b1 = @truncate(u32, b >> 32);
225 const lo = clmulSoft32(a0, b0);
226 const hi = clmulSoft32(a1, b1);
227 const mid = clmulSoft32(a0 ^ a1, b0 ^ b1) ^ lo ^ hi;
228 const res_lo = lo ^ (mid << 32);
229 const res_hi = hi ^ (mid >> 32);
230 return @as(u128, res_lo) | (@as(u128, res_hi) << 64);
231 }
232
189 const I256 = struct {233 const I256 = struct {
190 hi: u128,234 hi: u128,
191 lo: u128,235 lo: u128,