| ... | ... | @@ -192,85 +192,89 @@ fn Sha2x32(comptime params: Sha2Params32) type { |
| 192 | 192 | s[i] |= @as(u32, b[i * 4 + 3]) << 0; |
| 193 | 193 | } |
| 194 | 194 | |
| 195 | | if (builtin.cpu.arch == .aarch64 and builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) { |
| 196 | | var x: v4u32 = d.s[0..4].*; |
| 197 | | var y: v4u32 = d.s[4..8].*; |
| 198 | | const s_v = @ptrCast(*[16]v4u32, &s); |
| 199 | | |
| 200 | | comptime var k: u8 = 0; |
| 201 | | inline while (k < 16) : (k += 1) { |
| 202 | | if (k > 3) { |
| 203 | | s_v[k] = asm ( |
| 204 | | \\sha256su0.4s %[w0_3], %[w4_7] |
| 205 | | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] |
| 206 | | : [w0_3] "=w" (-> v4u32), |
| 207 | | : [_] "0" (s_v[k - 4]), |
| 208 | | [w4_7] "w" (s_v[k - 3]), |
| 209 | | [w8_11] "w" (s_v[k - 2]), |
| 210 | | [w12_15] "w" (s_v[k - 1]), |
| 195 | switch (builtin.cpu.arch) { |
| 196 | .aarch64 => if (comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.aarch64.Feature.sha2))) { |
| 197 | var x: v4u32 = d.s[0..4].*; |
| 198 | var y: v4u32 = d.s[4..8].*; |
| 199 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 200 | |
| 201 | comptime var k: u8 = 0; |
| 202 | inline while (k < 16) : (k += 1) { |
| 203 | if (k > 3) { |
| 204 | s_v[k] = asm ( |
| 205 | \\sha256su0.4s %[w0_3], %[w4_7] |
| 206 | \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15] |
| 207 | : [w0_3] "=w" (-> v4u32), |
| 208 | : [_] "0" (s_v[k - 4]), |
| 209 | [w4_7] "w" (s_v[k - 3]), |
| 210 | [w8_11] "w" (s_v[k - 2]), |
| 211 | [w12_15] "w" (s_v[k - 1]), |
| 212 | ); |
| 213 | } |
| 214 | |
| 215 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 216 | asm volatile ( |
| 217 | \\mov.4s v0, %[x] |
| 218 | \\sha256h.4s %[x], %[y], %[w] |
| 219 | \\sha256h2.4s %[y], v0, %[w] |
| 220 | : [x] "=w" (x), |
| 221 | [y] "=w" (y), |
| 222 | : [_] "0" (x), |
| 223 | [_] "1" (y), |
| 224 | [w] "w" (w), |
| 225 | : "v0" |
| 211 | 226 | ); |
| 212 | 227 | } |
| 213 | 228 | |
| 214 | | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 215 | | asm volatile ( |
| 216 | | \\mov.4s v0, %[x] |
| 217 | | \\sha256h.4s %[x], %[y], %[w] |
| 218 | | \\sha256h2.4s %[y], v0, %[w] |
| 219 | | : [x] "=w" (x), |
| 220 | | [y] "=w" (y), |
| 221 | | : [_] "0" (x), |
| 222 | | [_] "1" (y), |
| 223 | | [w] "w" (w), |
| 224 | | : "v0" |
| 225 | | ); |
| 226 | | } |
| 227 | | |
| 228 | | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); |
| 229 | | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); |
| 230 | | return; |
| 231 | | } else if (builtin.cpu.arch == .x86_64 and builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) { |
| 232 | | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; |
| 233 | | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; |
| 234 | | const s_v = @ptrCast(*[16]v4u32, &s); |
| 235 | | |
| 236 | | comptime var k: u8 = 0; |
| 237 | | inline while (k < 16) : (k += 1) { |
| 238 | | if (k < 12) { |
| 239 | | const r = asm ("sha256msg1 %[w4_7], %[w0_3]" |
| 240 | | : [w0_3] "=x" (-> v4u32), |
| 241 | | : [_] "0" (s_v[k]), |
| 242 | | [w4_7] "x" (s_v[k + 1]), |
| 243 | | ); |
| 244 | | const t = @shuffle(u32, s_v[k + 2], s_v[k + 3], [_]i32{ 1, 2, 3, -1 }); |
| 245 | | s_v[k + 4] = asm ("sha256msg2 %[w12_15], %[t]" |
| 246 | | : [t] "=x" (-> v4u32), |
| 247 | | : [_] "0" (r +% t), |
| 248 | | [w12_15] "x" (s_v[k + 3]), |
| 229 | d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*); |
| 230 | d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*); |
| 231 | return; |
| 232 | }, |
| 233 | .x86_64 => if (comptime builtin.cpu.features.isEnabled(@enumToInt(std.Target.x86.Feature.sha))) { |
| 234 | var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] }; |
| 235 | var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] }; |
| 236 | const s_v = @ptrCast(*[16]v4u32, &s); |
| 237 | |
| 238 | comptime var k: u8 = 0; |
| 239 | inline while (k < 16) : (k += 1) { |
| 240 | if (k < 12) { |
| 241 | const r = asm ("sha256msg1 %[w4_7], %[w0_3]" |
| 242 | : [w0_3] "=x" (-> v4u32), |
| 243 | : [_] "0" (s_v[k]), |
| 244 | [w4_7] "x" (s_v[k + 1]), |
| 245 | ); |
| 246 | const t = @shuffle(u32, s_v[k + 2], s_v[k + 3], [_]i32{ 1, 2, 3, -1 }); |
| 247 | s_v[k + 4] = asm ("sha256msg2 %[w12_15], %[t]" |
| 248 | : [t] "=x" (-> v4u32), |
| 249 | : [_] "0" (r +% t), |
| 250 | [w12_15] "x" (s_v[k + 3]), |
| 251 | ); |
| 252 | } |
| 253 | |
| 254 | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 255 | asm volatile ( |
| 256 | \\sha256rnds2 %[x], %[y] |
| 257 | \\pshufd $0xe, %%xmm0, %%xmm0 |
| 258 | \\sha256rnds2 %[y], %[x] |
| 259 | : [y] "=x" (y), |
| 260 | [x] "=x" (x), |
| 261 | : [_] "0" (y), |
| 262 | [_] "1" (x), |
| 263 | [_] "{xmm0}" (w), |
| 249 | 264 | ); |
| 250 | 265 | } |
| 251 | 266 | |
| 252 | | const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*); |
| 253 | | asm volatile ( |
| 254 | | \\sha256rnds2 %[x], %[y] |
| 255 | | \\pshufd $0xe, %%xmm0, %%xmm0 |
| 256 | | \\sha256rnds2 %[y], %[x] |
| 257 | | : [y] "=x" (y), |
| 258 | | [x] "=x" (x), |
| 259 | | : [_] "0" (y), |
| 260 | | [_] "1" (x), |
| 261 | | [_] "{xmm0}" (w), |
| 262 | | ); |
| 263 | | } |
| 264 | | |
| 265 | | d.s[0] +%= x[3]; |
| 266 | | d.s[1] +%= x[2]; |
| 267 | | d.s[4] +%= x[1]; |
| 268 | | d.s[5] +%= x[0]; |
| 269 | | d.s[2] +%= y[3]; |
| 270 | | d.s[3] +%= y[2]; |
| 271 | | d.s[6] +%= y[1]; |
| 272 | | d.s[7] +%= y[0]; |
| 273 | | return; |
| 267 | d.s[0] +%= x[3]; |
| 268 | d.s[1] +%= x[2]; |
| 269 | d.s[4] +%= x[1]; |
| 270 | d.s[5] +%= x[0]; |
| 271 | d.s[2] +%= y[3]; |
| 272 | d.s[3] +%= y[2]; |
| 273 | d.s[6] +%= y[1]; |
| 274 | d.s[7] +%= y[0]; |
| 275 | return; |
| 276 | }, |
| 277 | else => {}, |
| 274 | 278 | } |
| 275 | 279 | |
| 276 | 280 | while (i < 64) : (i += 1) { |
Cody Tapscott <topolarity@tapscott.me> 2022-10-24 00:38:10-07:00
lib