authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-28 23:25:34+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-29 14:34:58-04:00
log5764c550ed5b891e71107857822d7143cd510796
tree724fed13102f7f6acb8cd56a9115339f36663d64
parent17575019a0b46a8f03418587655ce98bc1b56899

std/crypto: vectorize Salsa20

20% faster on x86_64, slower on aarch64 as usual :/

1 files changed, 201 insertions(+), 14 deletions(-)

lib/std/crypto/salsa20.zig+201-14
......@@ -9,11 +9,185 @@ const crypto = std.crypto;
99const debug = std.debug;
1010const math = std.math;
1111const mem = std.mem;
12const Vector = std.meta.Vector;
1213
1314const Poly1305 = crypto.onetimeauth.Poly1305;
1415const Blake2b = crypto.hash.blake2.Blake2b;
1516const X25519 = crypto.dh.X25519;
1617
18const Salsa20VecImpl = struct {
19 const Lane = Vector(4, u32);
20 const Half = Vector(2, u32);
21 const BlockVec = [4]Lane;
22
23 fn initContext(key: [8]u32, d: [4]u32) BlockVec {
24 const c = "expand 32-byte k";
25 const constant_le = comptime [4]u32{
26 mem.readIntLittle(u32, c[0..4]),
27 mem.readIntLittle(u32, c[4..8]),
28 mem.readIntLittle(u32, c[8..12]),
29 mem.readIntLittle(u32, c[12..16]),
30 };
31 return BlockVec{
32 Lane{ key[0], key[1], key[2], key[3] },
33 Lane{ key[4], key[5], key[6], key[7] },
34 Lane{ constant_le[0], constant_le[1], constant_le[2], constant_le[3] },
35 Lane{ d[0], d[1], d[2], d[3] },
36 };
37 }
38
39 inline fn rot(x: Lane, comptime n: u5) Lane {
40 return (x << @splat(4, @as(u5, n))) | (x >> @splat(4, @as(u5, 1 +% ~n)));
41 }
42
43 inline fn salsa20Core(x: *BlockVec, input: BlockVec, comptime feedback: bool) void {
44 const n1n2n3n0 = Lane{ input[3][1], input[3][2], input[3][3], input[3][0] };
45 const n1n2 = Half{ n1n2n3n0[0], n1n2n3n0[1] };
46 const n3n0 = Half{ n1n2n3n0[2], n1n2n3n0[3] };
47 const k0k1 = Half{ input[0][0], input[0][1] };
48 const k2k3 = Half{ input[0][2], input[0][3] };
49 const k4k5 = Half{ input[1][0], input[1][1] };
50 const k6k7 = Half{ input[1][2], input[1][3] };
51 const n0k0 = Half{ n3n0[1], k0k1[0] };
52 const k0n0 = Half{ n0k0[1], n0k0[0] };
53 const k4k5k0n0 = Lane{ k4k5[0], k4k5[1], k0n0[0], k0n0[1] };
54 const k1k6 = Half{ k0k1[1], k6k7[0] };
55 const k6k1 = Half{ k1k6[1], k1k6[0] };
56 const n1n2k6k1 = Lane{ n1n2[0], n1n2[1], k6k1[0], k6k1[1] };
57 const k7n3 = Half{ k6k7[1], n3n0[0] };
58 const n3k7 = Half{ k7n3[1], k7n3[0] };
59 const k2k3n3k7 = Lane{ k2k3[0], k2k3[1], n3k7[0], n3k7[1] };
60
61 var diag0 = input[2];
62 var diag1 = @shuffle(u32, k4k5k0n0, undefined, [_]i32{ 1, 2, 3, 0 });
63 var diag2 = @shuffle(u32, n1n2k6k1, undefined, [_]i32{ 1, 2, 3, 0 });
64 var diag3 = @shuffle(u32, k2k3n3k7, undefined, [_]i32{ 1, 2, 3, 0 });
65
66 const start0 = diag0;
67 const start1 = diag1;
68 const start2 = diag2;
69 const start3 = diag3;
70
71 var i: usize = 0;
72 while (i < 20) : (i += 2) {
73 var a0 = diag1 +% diag0;
74 diag3 ^= rot(a0, 7);
75 var a1 = diag0 +% diag3;
76 diag2 ^= rot(a1, 9);
77 var a2 = diag3 +% diag2;
78 diag1 ^= rot(a2, 13);
79 var a3 = diag2 +% diag1;
80 diag0 ^= rot(a3, 18);
81
82 var diag3_shift = @shuffle(u32, diag3, undefined, [_]i32{ 3, 0, 1, 2 });
83 var diag2_shift = @shuffle(u32, diag2, undefined, [_]i32{ 2, 3, 0, 1 });
84 var diag1_shift = @shuffle(u32, diag1, undefined, [_]i32{ 1, 2, 3, 0 });
85 diag3 = diag3_shift;
86 diag2 = diag2_shift;
87 diag1 = diag1_shift;
88
89 a0 = diag3 +% diag0;
90 diag1 ^= rot(a0, 7);
91 a1 = diag0 +% diag1;
92 diag2 ^= rot(a1, 9);
93 a2 = diag1 +% diag2;
94 diag3 ^= rot(a2, 13);
95 a3 = diag2 +% diag3;
96 diag0 ^= rot(a3, 18);
97
98 diag1_shift = @shuffle(u32, diag1, undefined, [_]i32{ 3, 0, 1, 2 });
99 diag2_shift = @shuffle(u32, diag2, undefined, [_]i32{ 2, 3, 0, 1 });
100 diag3_shift = @shuffle(u32, diag3, undefined, [_]i32{ 1, 2, 3, 0 });
101 diag1 = diag1_shift;
102 diag2 = diag2_shift;
103 diag3 = diag3_shift;
104 }
105
106 if (feedback) {
107 diag0 +%= start0;
108 diag1 +%= start1;
109 diag2 +%= start2;
110 diag3 +%= start3;
111 }
112
113 const x0x1x10x11 = Lane{ diag0[0], diag1[1], diag0[2], diag1[3] };
114 const x12x13x6x7 = Lane{ diag1[0], diag2[1], diag1[2], diag2[3] };
115 const x8x9x2x3 = Lane{ diag2[0], diag3[1], diag2[2], diag3[3] };
116 const x4x5x14x15 = Lane{ diag3[0], diag0[1], diag3[2], diag0[3] };
117
118 x[0] = Lane{ x0x1x10x11[0], x0x1x10x11[1], x8x9x2x3[2], x8x9x2x3[3] };
119 x[1] = Lane{ x4x5x14x15[0], x4x5x14x15[1], x12x13x6x7[2], x12x13x6x7[3] };
120 x[2] = Lane{ x8x9x2x3[0], x8x9x2x3[1], x0x1x10x11[2], x0x1x10x11[3] };
121 x[3] = Lane{ x12x13x6x7[0], x12x13x6x7[1], x4x5x14x15[2], x4x5x14x15[3] };
122 }
123
124 fn hashToBytes(out: *[64]u8, x: BlockVec) void {
125 var i: usize = 0;
126 while (i < 4) : (i += 1) {
127 mem.writeIntLittle(u32, out[16 * i + 0 ..][0..4], x[i][0]);
128 mem.writeIntLittle(u32, out[16 * i + 4 ..][0..4], x[i][1]);
129 mem.writeIntLittle(u32, out[16 * i + 8 ..][0..4], x[i][2]);
130 mem.writeIntLittle(u32, out[16 * i + 12 ..][0..4], x[i][3]);
131 }
132 }
133
134 fn salsa20Xor(out: []u8, in: []const u8, key: [8]u32, d: [4]u32) void {
135 var ctx = initContext(key, d);
136 var x: BlockVec = undefined;
137 var buf: [64]u8 = undefined;
138 var i: usize = 0;
139 while (i + 64 <= in.len) : (i += 64) {
140 salsa20Core(x[0..], ctx, true);
141 hashToBytes(buf[0..], x);
142 var xout = out[i..];
143 const xin = in[i..];
144 var j: usize = 0;
145 while (j < 64) : (j += 1) {
146 xout[j] = xin[j];
147 }
148 j = 0;
149 while (j < 64) : (j += 1) {
150 xout[j] ^= buf[j];
151 }
152 ctx[2][0] +%= 1;
153 if (ctx[2][0] == 0) {
154 ctx[2][1] += 1;
155 }
156 }
157 if (i < in.len) {
158 salsa20Core(x[0..], ctx, true);
159 hashToBytes(buf[0..], x);
160
161 var xout = out[i..];
162 const xin = in[i..];
163 var j: usize = 0;
164 while (j < in.len % 64) : (j += 1) {
165 xout[j] = xin[j] ^ buf[j];
166 }
167 }
168 }
169
170 fn hsalsa20(input: [16]u8, key: [32]u8) [32]u8 {
171 var c: [4]u32 = undefined;
172 for (c) |_, i| {
173 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
174 }
175 const ctx = initContext(keyToWords(key), c);
176 var x: BlockVec = undefined;
177 salsa20Core(x[0..], ctx, false);
178 var out: [32]u8 = undefined;
179 mem.writeIntLittle(u32, out[0..4], x[0][0]);
180 mem.writeIntLittle(u32, out[4..8], x[1][1]);
181 mem.writeIntLittle(u32, out[8..12], x[2][2]);
182 mem.writeIntLittle(u32, out[12..16], x[3][3]);
183 mem.writeIntLittle(u32, out[16..20], x[1][2]);
184 mem.writeIntLittle(u32, out[20..24], x[1][3]);
185 mem.writeIntLittle(u32, out[24..28], x[2][0]);
186 mem.writeIntLittle(u32, out[28..32], x[2][1]);
187 return out;
188 }
189};
190
17191const Salsa20NonVecImpl = struct {
18192 const BlockVec = [16]u32;
19193
......@@ -49,7 +223,7 @@ const Salsa20NonVecImpl = struct {
49223 };
50224 }
51225
52 inline fn salsa20Core(x: *BlockVec, input: BlockVec) void {
226 inline fn salsa20Core(x: *BlockVec, input: BlockVec, comptime feedback: bool) void {
53227 const arx_steps = comptime [_]QuarterRound{
54228 Rp(4, 0, 12, 7), Rp(8, 4, 0, 9), Rp(12, 8, 4, 13), Rp(0, 12, 8, 18),
55229 Rp(9, 5, 1, 7), Rp(13, 9, 5, 9), Rp(1, 13, 9, 13), Rp(5, 1, 13, 18),
......@@ -67,6 +241,12 @@ const Salsa20NonVecImpl = struct {
67241 x[r.a] ^= math.rotl(u32, x[r.b] +% x[r.c], r.d);
68242 }
69243 }
244 if (feedback) {
245 j = 0;
246 while (j < 16) : (j += 1) {
247 x[j] +%= input[j];
248 }
249 }
70250 }
71251
72252 fn hashToBytes(out: *[64]u8, x: BlockVec) void {
......@@ -75,21 +255,13 @@ const Salsa20NonVecImpl = struct {
75255 }
76256 }
77257
78 fn contextFeedback(x: *BlockVec, ctx: BlockVec) void {
79 var i: usize = 0;
80 while (i < 16) : (i += 1) {
81 x[i] +%= ctx[i];
82 }
83 }
84
85258 fn salsa20Xor(out: []u8, in: []const u8, key: [8]u32, d: [4]u32) void {
86259 var ctx = initContext(key, d);
87260 var x: BlockVec = undefined;
88261 var buf: [64]u8 = undefined;
89262 var i: usize = 0;
90263 while (i + 64 <= in.len) : (i += 64) {
91 salsa20Core(x[0..], ctx);
92 contextFeedback(&x, ctx);
264 salsa20Core(x[0..], ctx, true);
93265 hashToBytes(buf[0..], x);
94266 var xout = out[i..];
95267 const xin = in[i..];
......@@ -104,8 +276,7 @@ const Salsa20NonVecImpl = struct {
104276 ctx[9] += @boolToInt(@addWithOverflow(u32, ctx[8], 1, &ctx[8]));
105277 }
106278 if (i < in.len) {
107 salsa20Core(x[0..], ctx);
108 contextFeedback(&x, ctx);
279 salsa20Core(x[0..], ctx, true);
109280 hashToBytes(buf[0..], x);
110281
111282 var xout = out[i..];
......@@ -124,7 +295,7 @@ const Salsa20NonVecImpl = struct {
124295 }
125296 const ctx = initContext(keyToWords(key), c);
126297 var x: BlockVec = undefined;
127 salsa20Core(x[0..], ctx);
298 salsa20Core(x[0..], ctx, false);
128299 var out: [32]u8 = undefined;
129300 mem.writeIntLittle(u32, out[0..4], x[0]);
130301 mem.writeIntLittle(u32, out[4..8], x[5]);
......@@ -138,7 +309,7 @@ const Salsa20NonVecImpl = struct {
138309 }
139310};
140311
141const Salsa20Impl = Salsa20NonVecImpl;
312const Salsa20Impl = if (std.Target.current.cpu.arch == .x86_64) Salsa20VecImpl else Salsa20NonVecImpl;
142313
143314fn keyToWords(key: [32]u8) [8]u32 {
144315 var k: [8]u32 = undefined;
......@@ -381,6 +552,22 @@ pub const SealedBox = struct {
381552 }
382553};
383554
555const htest = @import("test.zig");
556
557test "(x)salsa20" {
558 const key = [_]u8{0x69} ** 32;
559 const nonce = [_]u8{0x42} ** 8;
560 const msg = [_]u8{0} ** 20;
561 var c: [msg.len]u8 = undefined;
562
563 Salsa20.xor(&c, msg[0..], 0, key, nonce);
564 htest.assertEqual("30ff9933aa6534ff5207142593cd1fca4b23bdd8", c[0..]);
565
566 const extended_nonce = [_]u8{0x42} ** 24;
567 XSalsa20.xor(&c, msg[0..], 0, key, extended_nonce);
568 htest.assertEqual("b4ab7d82e750ec07644fa3281bce6cd91d4243f9", c[0..]);
569}
570
384571test "xsalsa20poly1305" {
385572 var msg: [100]u8 = undefined;
386573 var msg2: [msg.len]u8 = undefined;