authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-10-29 03:31:42-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-10-29 03:31:42-04:00
log20925b2f5c5c0ae20fdc0574e5d4e5740d17b4d6
tree38e540d0d03be14f8ad013b0c586e59ec5638d1b
parentc36eb4ede9b0ba65f275f0d230fdbee2b71e88df
parent67fa3262b1329316cbf62e00ba3890d68a9f5f6d
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #13272 from topolarity/sha2-intrinsics

crypto.sha2: Use intrinsics for SHA-256 on x86-64 and AArch64

1 files changed, 183 insertions(+), 77 deletions(-)

lib/std/crypto/sha2.zig+183-77
......@@ -1,4 +1,5 @@
11const std = @import("../std.zig");
2const builtin = @import("builtin");
23const mem = std.mem;
34const math = std.math;
45const htest = @import("test.zig");
......@@ -16,10 +17,9 @@ const RoundParam256 = struct {
1617 g: usize,
1718 h: usize,
1819 i: usize,
19 k: u32,
2020};
2121
22fn roundParam256(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize, h: usize, i: usize, k: u32) RoundParam256 {
22fn roundParam256(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g: usize, h: usize, i: usize) RoundParam256 {
2323 return RoundParam256{
2424 .a = a,
2525 .b = b,
......@@ -30,7 +30,6 @@ fn roundParam256(a: usize, b: usize, c: usize, d: usize, e: usize, f: usize, g:
3030 .g = g,
3131 .h = h,
3232 .i = i,
33 .k = k,
3433 };
3534}
3635
......@@ -70,6 +69,14 @@ const Sha256Params = Sha2Params32{
7069 .digest_bits = 256,
7170};
7271
72const v4u32 = @Vector(4, u32);
73
74// TODO: Remove once https://github.com/ziglang/zig/issues/868 is resolved.
75fn isComptime() bool {
76 var a: u8 = 0;
77 return @typeInfo(@TypeOf(.{a})).Struct.fields[0].is_comptime;
78}
79
7380/// SHA-224
7481pub const Sha224 = Sha2x32(Sha224Params);
7582
......@@ -83,7 +90,7 @@ fn Sha2x32(comptime params: Sha2Params32) type {
8390 pub const digest_length = params.digest_bits / 8;
8491 pub const Options = struct {};
8592
86 s: [8]u32,
93 s: [8]u32 align(16),
8794 // Streaming Cache
8895 buf: [64]u8 = undefined,
8996 buf_len: u8 = 0,
......@@ -168,17 +175,116 @@ fn Sha2x32(comptime params: Sha2Params32) type {
168175 }
169176 }
170177
178 const W = [64]u32{
179 0x428A2F98, 0x71374491, 0xB5C0FBCF, 0xE9B5DBA5, 0x3956C25B, 0x59F111F1, 0x923F82A4, 0xAB1C5ED5,
180 0xD807AA98, 0x12835B01, 0x243185BE, 0x550C7DC3, 0x72BE5D74, 0x80DEB1FE, 0x9BDC06A7, 0xC19BF174,
181 0xE49B69C1, 0xEFBE4786, 0x0FC19DC6, 0x240CA1CC, 0x2DE92C6F, 0x4A7484AA, 0x5CB0A9DC, 0x76F988DA,
182 0x983E5152, 0xA831C66D, 0xB00327C8, 0xBF597FC7, 0xC6E00BF3, 0xD5A79147, 0x06CA6351, 0x14292967,
183 0x27B70A85, 0x2E1B2138, 0x4D2C6DFC, 0x53380D13, 0x650A7354, 0x766A0ABB, 0x81C2C92E, 0x92722C85,
184 0xA2BFE8A1, 0xA81A664B, 0xC24B8B70, 0xC76C51A3, 0xD192E819, 0xD6990624, 0xF40E3585, 0x106AA070,
185 0x19A4C116, 0x1E376C08, 0x2748774C, 0x34B0BCB5, 0x391C0CB3, 0x4ED8AA4A, 0x5B9CCA4F, 0x682E6FF3,
186 0x748F82EE, 0x78A5636F, 0x84C87814, 0x8CC70208, 0x90BEFFFA, 0xA4506CEB, 0xBEF9A3F7, 0xC67178F2,
187 };
188
171189 fn round(d: *Self, b: *const [64]u8) void {
172 var s: [64]u32 = undefined;
190 var s: [64]u32 align(16) = undefined;
191 for (@ptrCast(*align(1) const [16]u32, b)) |*elem, i| {
192 s[i] = mem.readIntBig(u32, mem.asBytes(elem));
193 }
173194
174 var i: usize = 0;
175 while (i < 16) : (i += 1) {
176 s[i] = 0;
177 s[i] |= @as(u32, b[i * 4 + 0]) << 24;
178 s[i] |= @as(u32, b[i * 4 + 1]) << 16;
179 s[i] |= @as(u32, b[i * 4 + 2]) << 8;
180 s[i] |= @as(u32, b[i * 4 + 3]) << 0;
195 if (!isComptime()) {
196 switch (builtin.cpu.arch) {
197 .aarch64 => if (comptime std.Target.aarch64.featureSetHas(builtin.cpu.features, .sha2)) {
198 var x: v4u32 = d.s[0..4].*;
199 var y: v4u32 = d.s[4..8].*;
200 const s_v = @ptrCast(*[16]v4u32, &s);
201
202 comptime var k: u8 = 0;
203 inline while (k < 16) : (k += 1) {
204 if (k > 3) {
205 s_v[k] = asm (
206 \\sha256su0.4s %[w0_3], %[w4_7]
207 \\sha256su1.4s %[w0_3], %[w8_11], %[w12_15]
208 : [w0_3] "=w" (-> v4u32),
209 : [_] "0" (s_v[k - 4]),
210 [w4_7] "w" (s_v[k - 3]),
211 [w8_11] "w" (s_v[k - 2]),
212 [w12_15] "w" (s_v[k - 1]),
213 );
214 }
215
216 const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*);
217 asm volatile (
218 \\mov.4s v0, %[x]
219 \\sha256h.4s %[x], %[y], %[w]
220 \\sha256h2.4s %[y], v0, %[w]
221 : [x] "=w" (x),
222 [y] "=w" (y),
223 : [_] "0" (x),
224 [_] "1" (y),
225 [w] "w" (w),
226 : "v0"
227 );
228 }
229
230 d.s[0..4].* = x +% @as(v4u32, d.s[0..4].*);
231 d.s[4..8].* = y +% @as(v4u32, d.s[4..8].*);
232 return;
233 },
234 .x86_64 => if (comptime std.Target.x86.featureSetHas(builtin.cpu.features, .sha)) {
235 var x: v4u32 = [_]u32{ d.s[5], d.s[4], d.s[1], d.s[0] };
236 var y: v4u32 = [_]u32{ d.s[7], d.s[6], d.s[3], d.s[2] };
237 const s_v = @ptrCast(*[16]v4u32, &s);
238
239 comptime var k: u8 = 0;
240 inline while (k < 16) : (k += 1) {
241 if (k < 12) {
242 var tmp = s_v[k];
243 s_v[k + 4] = asm (
244 \\ sha256msg1 %[w4_7], %[tmp]
245 \\ vpalignr $0x4, %[w8_11], %[w12_15], %[result]
246 \\ paddd %[tmp], %[result]
247 \\ sha256msg2 %[w12_15], %[result]
248 : [tmp] "=&x" (tmp),
249 [result] "=&x" (-> v4u32),
250 : [_] "0" (tmp),
251 [w4_7] "x" (s_v[k + 1]),
252 [w8_11] "x" (s_v[k + 2]),
253 [w12_15] "x" (s_v[k + 3]),
254 );
255 }
256
257 const w: v4u32 = s_v[k] +% @as(v4u32, W[4 * k ..][0..4].*);
258 y = asm ("sha256rnds2 %[x], %[y]"
259 : [y] "=x" (-> v4u32),
260 : [_] "0" (y),
261 [x] "x" (x),
262 [_] "{xmm0}" (w),
263 );
264
265 x = asm ("sha256rnds2 %[y], %[x]"
266 : [x] "=x" (-> v4u32),
267 : [_] "0" (x),
268 [y] "x" (y),
269 [_] "{xmm0}" (@bitCast(v4u32, @bitCast(u128, w) >> 64)),
270 );
271 }
272
273 d.s[0] +%= x[3];
274 d.s[1] +%= x[2];
275 d.s[4] +%= x[1];
276 d.s[5] +%= x[0];
277 d.s[2] +%= y[3];
278 d.s[3] +%= y[2];
279 d.s[6] +%= y[1];
280 d.s[7] +%= y[0];
281 return;
282 },
283 else => {},
284 }
181285 }
286
287 var i: usize = 16;
182288 while (i < 64) : (i += 1) {
183289 s[i] = s[i - 16] +% s[i - 7] +% (math.rotr(u32, s[i - 15], @as(u32, 7)) ^ math.rotr(u32, s[i - 15], @as(u32, 18)) ^ (s[i - 15] >> 3)) +% (math.rotr(u32, s[i - 2], @as(u32, 17)) ^ math.rotr(u32, s[i - 2], @as(u32, 19)) ^ (s[i - 2] >> 10));
184290 }
......@@ -195,73 +301,73 @@ fn Sha2x32(comptime params: Sha2Params32) type {
195301 };
196302
197303 const round0 = comptime [_]RoundParam256{
198 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 0, 0x428A2F98),
199 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 1, 0x71374491),
200 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 2, 0xB5C0FBCF),
201 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 3, 0xE9B5DBA5),
202 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 4, 0x3956C25B),
203 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 5, 0x59F111F1),
204 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 6, 0x923F82A4),
205 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 7, 0xAB1C5ED5),
206 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 8, 0xD807AA98),
207 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 9, 0x12835B01),
208 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 10, 0x243185BE),
209 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 11, 0x550C7DC3),
210 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 12, 0x72BE5D74),
211 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 13, 0x80DEB1FE),
212 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 14, 0x9BDC06A7),
213 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 15, 0xC19BF174),
214 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 16, 0xE49B69C1),
215 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 17, 0xEFBE4786),
216 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 18, 0x0FC19DC6),
217 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 19, 0x240CA1CC),
218 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 20, 0x2DE92C6F),
219 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 21, 0x4A7484AA),
220 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 22, 0x5CB0A9DC),
221 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 23, 0x76F988DA),
222 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 24, 0x983E5152),
223 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 25, 0xA831C66D),
224 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 26, 0xB00327C8),
225 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 27, 0xBF597FC7),
226 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 28, 0xC6E00BF3),
227 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 29, 0xD5A79147),
228 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 30, 0x06CA6351),
229 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 31, 0x14292967),
230 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 32, 0x27B70A85),
231 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 33, 0x2E1B2138),
232 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 34, 0x4D2C6DFC),
233 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 35, 0x53380D13),
234 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 36, 0x650A7354),
235 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 37, 0x766A0ABB),
236 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 38, 0x81C2C92E),
237 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 39, 0x92722C85),
238 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 40, 0xA2BFE8A1),
239 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 41, 0xA81A664B),
240 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 42, 0xC24B8B70),
241 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 43, 0xC76C51A3),
242 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 44, 0xD192E819),
243 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 45, 0xD6990624),
244 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 46, 0xF40E3585),
245 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 47, 0x106AA070),
246 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 48, 0x19A4C116),
247 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 49, 0x1E376C08),
248 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 50, 0x2748774C),
249 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 51, 0x34B0BCB5),
250 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 52, 0x391C0CB3),
251 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 53, 0x4ED8AA4A),
252 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 54, 0x5B9CCA4F),
253 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 55, 0x682E6FF3),
254 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 56, 0x748F82EE),
255 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 57, 0x78A5636F),
256 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 58, 0x84C87814),
257 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 59, 0x8CC70208),
258 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 60, 0x90BEFFFA),
259 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 61, 0xA4506CEB),
260 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 62, 0xBEF9A3F7),
261 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 63, 0xC67178F2),
304 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 0),
305 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 1),
306 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 2),
307 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 3),
308 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 4),
309 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 5),
310 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 6),
311 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 7),
312 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 8),
313 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 9),
314 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 10),
315 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 11),
316 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 12),
317 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 13),
318 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 14),
319 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 15),
320 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 16),
321 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 17),
322 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 18),
323 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 19),
324 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 20),
325 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 21),
326 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 22),
327 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 23),
328 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 24),
329 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 25),
330 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 26),
331 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 27),
332 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 28),
333 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 29),
334 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 30),
335 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 31),
336 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 32),
337 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 33),
338 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 34),
339 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 35),
340 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 36),
341 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 37),
342 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 38),
343 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 39),
344 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 40),
345 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 41),
346 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 42),
347 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 43),
348 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 44),
349 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 45),
350 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 46),
351 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 47),
352 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 48),
353 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 49),
354 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 50),
355 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 51),
356 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 52),
357 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 53),
358 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 54),
359 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 55),
360 roundParam256(0, 1, 2, 3, 4, 5, 6, 7, 56),
361 roundParam256(7, 0, 1, 2, 3, 4, 5, 6, 57),
362 roundParam256(6, 7, 0, 1, 2, 3, 4, 5, 58),
363 roundParam256(5, 6, 7, 0, 1, 2, 3, 4, 59),
364 roundParam256(4, 5, 6, 7, 0, 1, 2, 3, 60),
365 roundParam256(3, 4, 5, 6, 7, 0, 1, 2, 61),
366 roundParam256(2, 3, 4, 5, 6, 7, 0, 1, 62),
367 roundParam256(1, 2, 3, 4, 5, 6, 7, 0, 63),
262368 };
263369 inline for (round0) |r| {
264 v[r.h] = v[r.h] +% (math.rotr(u32, v[r.e], @as(u32, 6)) ^ math.rotr(u32, v[r.e], @as(u32, 11)) ^ math.rotr(u32, v[r.e], @as(u32, 25))) +% (v[r.g] ^ (v[r.e] & (v[r.f] ^ v[r.g]))) +% r.k +% s[r.i];
370 v[r.h] = v[r.h] +% (math.rotr(u32, v[r.e], @as(u32, 6)) ^ math.rotr(u32, v[r.e], @as(u32, 11)) ^ math.rotr(u32, v[r.e], @as(u32, 25))) +% (v[r.g] ^ (v[r.e] & (v[r.f] ^ v[r.g]))) +% W[r.i] +% s[r.i];
265371
266372 v[r.d] = v[r.d] +% v[r.h];
267373