authorgravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-10-24 09:47:31-07:00
committergravatar for topolarity@tapscott.meCody Tapscott <topolarity@tapscott.me> 2022-10-28 15:21:10-07:00
log4c1f71e866088a1a2e943331256115ed7e3daf98
treeff801e8aa7b5f1f578434198144c28a8350da6ed
parentee241c47ee675050e4e4b0eabd6ba06a82cc626e

std.crypto: Optimize SHA-256 intrinsics for AMD x86-64

This gets us most of the way back to the performance I had when I was using the LLVM intrinsics: - Intel Intel(R) Core(TM) i7-1068NG7 CPU @ 2.30GHz: 190.67 MB/s (w/o intrinsics) -> 1285.08 MB/s - AMD EPYC 7763 (VM) @ 2.45 GHz: 240.09 MB/s (w/o intrinsics) -> 1360.78 MB/s - Apple M1: 216.96 MB/s (w/o intrinsics) -> 2133.69 MB/s Minor changes to this source can swing performance from 400 MB/s to 1400 MB/s or... 20 MB/s, depending on how it interacts with the optimizer. I have a sneaking suspicion that despite LLVM inheriting GCC's extremely strict inline assembly semantics, its passes are rather skittish around inline assembly (and almost certainly, its instruction cost models can assume nothing)

1 files changed, 23 insertions(+), 23 deletions(-)

lib/std/crypto/sha2.zig+23-23
...@@ -182,14 +182,8 @@ fn Sha2x32(comptime params: Sha2Params32) type {...@@ -182,14 +182,8 @@ fn Sha2x32(comptime params: Sha2Params32) type {
182182
183 fn round(d: *Self, b: *const [64]u8) void {183 fn round(d: *Self, b: *const [64]u8) void {
184 var s: [64]u32 align(16) = undefined;184 var s: [64]u32 align(16) = undefined;
185185 for (@ptrCast(*align(1) const [16]u32, b)) |*elem, i| {
186 var i: usize = 0;186 s[i] = mem.readIntBig(u32, mem.asBytes(elem));
187 while (i < 16) : (i += 1) {
188 s[i] = 0;
189 s[i] |= @as(u32, b[i * 4 + 0]) << 24;
190 s[i] |= @as(u32, b[i * 4 + 1]) << 16;
191 s[i] |= @as(u32, b[i * 4 + 2]) << 8;
192 s[i] |= @as(u32, b[i * 4 + 3]) << 0;
193 }187 }
194188
195 switch (builtin.cpu.arch) {189 switch (builtin.cpu.arch) {
...@@ -238,30 +232,35 @@ fn Sha2x32(comptime params: Sha2Params32) type {...@@ -238,30 +232,35 @@ fn Sha2x32(comptime params: Sha2Params32) type {
238 comptime var k: u8 = 0;232 comptime var k: u8 = 0;
239 inline while (k < 16) : (k += 1) {233 inline while (k < 16) : (k += 1) {
240 if (k < 12) {234 if (k < 12) {
241 const r = asm ("sha256msg1 %[w4_7], %[w0_3]"235 var tmp = s_v[k];
242 : [w0_3] "=x" (-> v4u32),236 s_v[k + 4] = asm (
243 : [_] "0" (s_v[k]),237 \\ sha256msg1 %[w4_7], %[tmp]
238 \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result]
239 \\ paddd %[tmp], %[result]
240 \\ sha256msg2 %[w12_15], %[result]
241 : [tmp] "=&x" (tmp),
242 [result] "=&x" (-> v4u32),
243 : [_] "0" (tmp),
244 [w4_7] "x" (s_v[k + 1]),244 [w4_7] "x" (s_v[k + 1]),
245 );245 [w8_11] "x" (s_v[k + 2]),
246 const t = @shuffle(u32, s_v[k + 2], s_v[k + 3], [_]i32{ 1, 2, 3, -1 });
247 s_v[k + 4] = asm ("sha256msg2 %[w12_15], %[t]"
248 : [t] "=x" (-> v4u32),
249 : [_] "0" (r +% t),
250 [w12_15] "x" (s_v[k + 3]),246 [w12_15] "x" (s_v[k + 3]),
251 );247 );
252 }248 }
253249
254 const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*);250 const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*);
255 asm volatile (251 y = asm ("sha256rnds2 %[x], %[y]"
256 \\sha256rnds2 %[x], %[y]252 : [y] "=x" (-> v4u32),
257 \\pshufd $0xe, %%xmm0, %%xmm0
258 \\sha256rnds2 %[y], %[x]
259 : [y] "=x" (y),
260 [x] "=x" (x),
261 : [_] "0" (y),253 : [_] "0" (y),
262 [_] "1" (x),254 [x] "x" (x),
263 [_] "{xmm0}" (w),255 [_] "{xmm0}" (w),
264 );256 );
257
258 x = asm ("sha256rnds2 %[y], %[x]"
259 : [x] "=x" (-> v4u32),
260 : [_] "0" (x),
261 [y] "x" (y),
262 [_] "{xmm0}" (@bitCast(v4u32, @bitCast(u128, w) >> 64)),
263 );
265 }264 }
266265
267 d.s[0] +%= x[3];266 d.s[0] +%= x[3];
...@@ -277,6 +276,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {...@@ -277,6 +276,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
277 else => {},276 else => {},
278 }277 }
279278
279 var i: usize = 16;
280 while (i < 64) : (i += 1) {280 while (i < 64) : (i += 1) {
281 s[i] = s[i - 16] +% s[i - 7] +% (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +% (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));281 s[i] = s[i - 16] +% s[i - 7] +% (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +% (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
282 }282 }