authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-10-22 23:50:38-07:00
committergravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-10-28 15:21:10-07:00
log10edb6d352173dfbc9962ce3db064384319e77f8
tree84b964d90c0488872a9ec56b856910aef82f3e2c
parentc616141241047d6d6c811d43f644eb1b7d2b26ce

crypto.sha2: Use intrinsics for SHA-256 on x86-64 and AArch64

There's probably plenty of room to optimize these further in the future, but for the moment this gives ~3x improvement on Intel x86-64 processors, ~5x on AMD, and ~10x on M1 Macs. These extensions are very new - Most processors prior to 2020 do not support them. AVX-512 is a slightly older alternative that we could use on Intel for a much bigger performance bump, but it's been fused off on Intel's latest hybrid architectures and it relies on computing independent SHA hashes in parallel. In contrast, these SHA intrinsics provide the usual single-threaded, single-stream interface, and should continue working on new processors. AArch64 also has SHA-512 intrinsics that we could take advantage of in the future

1 files changed, 164 insertions(+), 70 deletions(-)

lib/std/crypto/sha2.zig+164-70
......@@ -1,4 +1,5 @@
11const std = @import("../std.zig");
2const builtin = @import("builtin");
23const mem = std.mem;
34const math = std.math;
45const htest = @import("test.zig");
......@@ -16,10 +17,9 @@ const RoundParam256 = struct {
1617 g: usize,
1718 h: usize,
1819 i: usize,
19 k: u32,
2020};
2121
22fn roundParam256(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize, h: usize, i: usize, k: u32) RoundParam256 {
22fn roundParam256(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize, h: usize, i: usize) RoundParam256 {
2323 return RoundParam256{
2424 .a = a,
2525 .b = b,
......@@ -30,7 +30,6 @@ fn roundParam256(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g:
3030 .g = g,
3131 .h = h,
3232 .i = i,
33 .k = k,
3433 };
3534}
3635
......@@ -70,6 +69,8 @@ const Sha256Params = Sha2Params32{
7069 .digest_bits = 256,
7170};
7271
72const v4u32 = @Vector(4, u32);
73
7374/// SHA-224
7475pub const Sha224 = Sha2x32(Sha224Params);
7576
......@@ -83,7 +84,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
8384 pub const digest_length = params.digest_bits / 8;
8485 pub const Options = struct {};
8586
86 s: [8]u32,
87 s: [8]u32 align(16),
8788 // Streaming Cache
8889 buf: [64]u8 = undefined,
8990 buf_len: u8 = 0,
......@@ -168,8 +169,19 @@ fn Sha2x32(comptime params: Sha2Params32) type {
168169 }
169170 }
170171
172 const W = [64]u32{
173 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
174 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
175 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
176 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
177 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
178 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
179 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
180 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
181 };
182
171183 fn round(d: *Self, b: *const [64]u8) void {
172 var s: [64]u32 = undefined;
184 var s: [64]u32 align(16) = undefined;
173185
174186 var i: usize = 0;
175187 while (i < 16) : (i += 1) {
......@@ -179,6 +191,88 @@ fn Sha2x32(comptime params: Sha2Params32) type {
179191 s[i] |= @as(u32, b[i * 4 + 2]) << 8;
180192 s[i] |= @as(u32, b[i * 4 + 3]) << 0;
181193 }
194
195 if (builtin.cpu.arch == .aarch64 and builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) {
196 var x: v4u32 = d.s[0..4].*;
197 var y: v4u32 = d.s[4..8].*;
198 const s_v = @ptrCast(*[16]v4u32, &s);
199
200 comptime var k: u8 = 0;
201 inline while (k < 16) : (k += 1) {
202 if (k > 3) {
203 s_v[k] = asm (
204 \\sha256su0.4s %[w0_3], %[w4_7]
205 \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15]
206 : [w0_3] "=w" (-> v4u32),
207 : [_] "0" (s_v[k - 4]),
208 [w4_7] "w" (s_v[k - 3]),
209 [w8_11] "w" (s_v[k - 2]),
210 [w12_15] "w" (s_v[k - 1]),
211 );
212 }
213
214 const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*);
215 asm volatile (
216 \\mov.4s v0, %[x]
217 \\sha256h.4s %[x], %[y], %[w]
218 \\sha256h2.4s %[y], v0, %[w]
219 : [x] "=w" (x),
220 [y] "=w" (y),
221 : [_] "0" (x),
222 [_] "1" (y),
223 [w] "w" (w),
224 : "v0"
225 );
226 }
227
228 d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*);
229 d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*);
230 return;
231 } else if (builtin.cpu.arch == .x86_64 and builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) {
232 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
233 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
234 const s_v = @ptrCast(*[16]v4u32, &s);
235
236 comptime var k: u8 = 0;
237 inline while (k < 16) : (k += 1) {
238 if (k < 12) {
239 const r = asm ("sha256msg1 %[w4_7], %[w0_3]"
240 : [w0_3] "=x" (-> v4u32),
241 : [_] "0" (s_v[k]),
242 [w4_7] "x" (s_v[k + 1]),
243 );
244 const t = @shuffle(u32, s_v[k + 2], s_v[k + 3], [_]i32{ 1, 2, 3, -1 });
245 s_v[k + 4] = asm ("sha256msg2 %[w12_15], %[t]"
246 : [t] "=x" (-> v4u32),
247 : [_] "0" (r +% t),
248 [w12_15] "x" (s_v[k + 3]),
249 );
250 }
251
252 const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*);
253 asm volatile (
254 \\sha256rnds2 %[x], %[y]
255 \\pshufd $0xe, %%xmm0, %%xmm0
256 \\sha256rnds2 %[y], %[x]
257 : [y] "=x" (y),
258 [x] "=x" (x),
259 : [_] "0" (y),
260 [_] "1" (x),
261 [_] "{xmm0}" (w),
262 );
263 }
264
265 d.s[0] +%= x[3];
266 d.s[1] +%= x[2];
267 d.s[4] +%= x[1];
268 d.s[5] +%= x[0];
269 d.s[2] +%= y[3];
270 d.s[3] +%= y[2];
271 d.s[6] +%= y[1];
272 d.s[7] +%= y[0];
273 return;
274 }
275
182276 while (i < 64) : (i += 1) {
183277 s[i] = s[i - 16] +% s[i - 7] +% (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +% (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
184278 }
......@@ -195,73 +289,73 @@ fn Sha2x32(comptime params: Sha2Params32) type {
195289 };
196290
197291 const round0 = comptime [_]RoundParam256{
198 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 0, 0x428A2F98),
199 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 1, 0x71374491),
200 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 2, 0xB5C0FBCF),
201 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 3, 0xE9B5DBA5),
202 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 4, 0x3956C25B),
203 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 5, 0x59F111F1),
204 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 6, 0x923F82A4),
205 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 7, 0xAB1C5ED5),
206 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 8, 0xD807AA98),
207 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 9, 0x12835B01),
208 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 10, 0x243185BE),
209 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 11, 0x550C7DC3),
210 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 12, 0x72BE5D74),
211 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 13, 0x80DEB1FE),
212 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 14, 0x9BDC06A7),
213 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 15, 0xC19BF174),
214 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 16, 0xE49B69C1),
215 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 17, 0xEFBE4786),
216 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 18, 0x0FC19DC6),
217 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 19, 0x240CA1CC),
218 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 20, 0x2DE92C6F),
219 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 21, 0x4A7484AA),
220 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 22, 0x5CB0A9DC),
221 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 23, 0x76F988DA),
222 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 24, 0x983E5152),
223 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 25, 0xA831C66D),
224 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 26, 0xB00327C8),
225 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 27, 0xBF597FC7),
226 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 28, 0xC6E00BF3),
227 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 29, 0xD5A79147),
228 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 30, 0x06CA6351),
229 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 31, 0x14292967),
230 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 32, 0x27B70A85),
231 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 33, 0x2E1B2138),
232 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 34, 0x4D2C6DFC),
233 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 35, 0x53380D13),
234 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 36, 0x650A7354),
235 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 37, 0x766A0ABB),
236 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 38, 0x81C2C92E),
237 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 39, 0x92722C85),
238 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 40, 0xA2BFE8A1),
239 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 41, 0xA81A664B),
240 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 42, 0xC24B8B70),
241 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 43, 0xC76C51A3),
242 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 44, 0xD192E819),
243 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 45, 0xD6990624),
244 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 46, 0xF40E3585),
245 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 47, 0x106AA070),
246 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 48, 0x19A4C116),
247 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 49, 0x1E376C08),
248 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 50, 0x2748774C),
249 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 51, 0x34B0BCB5),
250 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 52, 0x391C0CB3),
251 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 53, 0x4ED8AA4A),
252 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 54, 0x5B9CCA4F),
253 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 55, 0x682E6FF3),
254 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 56, 0x748F82EE),
255 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 57, 0x78A5636F),
256 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 58, 0x84C87814),
257 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 59, 0x8CC70208),
258 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 60, 0x90BEFFFA),
259 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 61, 0xA4506CEB),
260 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 62, 0xBEF9A3F7),
261 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 63, 0xC67178F2),
292 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 0),
293 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 1),
294 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 2),
295 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 3),
296 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 4),
297 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 5),
298 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 6),
299 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 7),
300 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 8),
301 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 9),
302 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 10),
303 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 11),
304 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 12),
305 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 13),
306 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 14),
307 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 15),
308 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 16),
309 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 17),
310 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 18),
311 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 19),
312 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 20),
313 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 21),
314 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 22),
315 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 23),
316 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 24),
317 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 25),
318 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 26),
319 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 27),
320 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 28),
321 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 29),
322 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 30),
323 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 31),
324 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 32),
325 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 33),
326 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 34),
327 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 35),
328 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 36),
329 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 37),
330 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 38),
331 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 39),
332 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 40),
333 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 41),
334 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 42),
335 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 43),
336 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 44),
337 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 45),
338 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 46),
339 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 47),
340 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 48),
341 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 49),
342 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 50),
343 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 51),
344 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 52),
345 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 53),
346 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 54),
347 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 55),
348 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 56),
349 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 57),
350 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 58),
351 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 59),
352 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 60),
353 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 61),
354 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 62),
355 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 63),
262356 };
263357 inline for (round0) |r| {
264 v[r.h] = v[r.h] +% (math.rotr(u32, v[r.e], @as(u32, 6)) ^ math.rotr(u32, v[r.e], @as(u32, 11)) ^ math.rotr(u32, v[r.e], @as(u32, 25))) +% (v[r.g] ^ (v[r.e] & (v[r.f] ^ v[r.g]))) +% r.k +% s[r.i];
358 v[r.h] = v[r.h] +% (math.rotr(u32, v[r.e], @as(u32, 6)) ^ math.rotr(u32, v[r.e], @as(u32, 11)) ^ math.rotr(u32, v[r.e], @as(u32, 25))) +% (v[r.g] ^ (v[r.e] & (v[r.f] ^ v[r.g]))) +% W[r.i] +% s[r.i];
265359
266360 v[r.d] = v[r.d] +% v[r.h];
267361