authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-28 23:25:34+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-29 14:34:58-04:00
log5764c550ed5b891e71107857822d7143cd510796
tree724fed13102f7f6acb8cd56a9115339f36663d64
parent17575019a0b46a8f03418587655ce98bc1b56899

std/crypto: vectorize Salsa20

20% faster on x86_64, slower on aarch64 as usual :/

1 files changed, 201 insertions(+), 14 deletions(-)

lib/std/crypto/salsa20.zig+201-14
...@@ -9,11 +9,185 @@ const crypto = std.crypto;...@@ -9,11 +9,185 @@ const crypto = std.crypto;
9const debug = std.debug;9const debug = std.debug;
10const math = std.math;10const math = std.math;
11const mem = std.mem;11const mem = std.mem;
12const Vector = std.meta.Vector;
1213
13const Poly1305 = crypto.onetimeauth.Poly1305;14const Poly1305 = crypto.onetimeauth.Poly1305;
14const Blake2b = crypto.hash.blake2.Blake2b;15const Blake2b = crypto.hash.blake2.Blake2b;
15const X25519 = crypto.dh.X25519;16const X25519 = crypto.dh.X25519;
1617
18const Salsa20VecImpl = struct {
19 const Lane = Vector(4, u32);
20 const Half = Vector(2, u32);
21 const BlockVec = [4]Lane;
22
23 fn initContext(key: [8]u32, d: [4]u32) BlockVec {
24 const c = "expand 32-byte k";
25 const constant_le = comptime [4]u32{
26 mem.readIntLittle(u32, c[0..4]),
27 mem.readIntLittle(u32, c[4..8]),
28 mem.readIntLittle(u32, c[8..12]),
29 mem.readIntLittle(u32, c[12..16]),
30 };
31 return BlockVec{
32 Lane{ key[0], key[1], key[2], key[3] },
33 Lane{ key[4], key[5], key[6], key[7] },
34 Lane{ constant_le[0], constant_le[1], constant_le[2], constant_le[3] },
35 Lane{ d[0], d[1], d[2], d[3] },
36 };
37 }
38
39 inline fn rot(x: Lane, comptime n: u5) Lane {
40 return (x << @splat(4, @as(u5, n))) | (x >> @splat(4, @as(u5, 1 +% ~n)));
41 }
42
43 inline fn salsa20Core(x: *BlockVec, input: BlockVec, comptime feedback: bool) void {
44 const n1n2n3n0 = Lane{ input[3][1], input[3][2], input[3][3], input[3][0] };
45 const n1n2 = Half{ n1n2n3n0[0], n1n2n3n0[1] };
46 const n3n0 = Half{ n1n2n3n0[2], n1n2n3n0[3] };
47 const k0k1 = Half{ input[0][0], input[0][1] };
48 const k2k3 = Half{ input[0][2], input[0][3] };
49 const k4k5 = Half{ input[1][0], input[1][1] };
50 const k6k7 = Half{ input[1][2], input[1][3] };
51 const n0k0 = Half{ n3n0[1], k0k1[0] };
52 const k0n0 = Half{ n0k0[1], n0k0[0] };
53 const k4k5k0n0 = Lane{ k4k5[0], k4k5[1], k0n0[0], k0n0[1] };
54 const k1k6 = Half{ k0k1[1], k6k7[0] };
55 const k6k1 = Half{ k1k6[1], k1k6[0] };
56 const n1n2k6k1 = Lane{ n1n2[0], n1n2[1], k6k1[0], k6k1[1] };
57 const k7n3 = Half{ k6k7[1], n3n0[0] };
58 const n3k7 = Half{ k7n3[1], k7n3[0] };
59 const k2k3n3k7 = Lane{ k2k3[0], k2k3[1], n3k7[0], n3k7[1] };
60
61 var diag0 = input[2];
62 var diag1 = @shuffle(u32, k4k5k0n0, undefined, [_]i32{ 1, 2, 3, 0 });
63 var diag2 = @shuffle(u32, n1n2k6k1, undefined, [_]i32{ 1, 2, 3, 0 });
64 var diag3 = @shuffle(u32, k2k3n3k7, undefined, [_]i32{ 1, 2, 3, 0 });
65
66 const start0 = diag0;
67 const start1 = diag1;
68 const start2 = diag2;
69 const start3 = diag3;
70
71 var i: usize = 0;
72 while (i < 20) : (i += 2) {
73 var a0 = diag1 +% diag0;
74 diag3 ^= rot(a0, 7);
75 var a1 = diag0 +% diag3;
76 diag2 ^= rot(a1, 9);
77 var a2 = diag3 +% diag2;
78 diag1 ^= rot(a2, 13);
79 var a3 = diag2 +% diag1;
80 diag0 ^= rot(a3, 18);
81
82 var diag3_shift = @shuffle(u32, diag3, undefined, [_]i32{ 3, 0, 1, 2 });
83 var diag2_shift = @shuffle(u32, diag2, undefined, [_]i32{ 2, 3, 0, 1 });
84 var diag1_shift = @shuffle(u32, diag1, undefined, [_]i32{ 1, 2, 3, 0 });
85 diag3 = diag3_shift;
86 diag2 = diag2_shift;
87 diag1 = diag1_shift;
88
89 a0 = diag3 +% diag0;
90 diag1 ^= rot(a0, 7);
91 a1 = diag0 +% diag1;
92 diag2 ^= rot(a1, 9);
93 a2 = diag1 +% diag2;
94 diag3 ^= rot(a2, 13);
95 a3 = diag2 +% diag3;
96 diag0 ^= rot(a3, 18);
97
98 diag1_shift = @shuffle(u32, diag1, undefined, [_]i32{ 3, 0, 1, 2 });
99 diag2_shift = @shuffle(u32, diag2, undefined, [_]i32{ 2, 3, 0, 1 });
100 diag3_shift = @shuffle(u32, diag3, undefined, [_]i32{ 1, 2, 3, 0 });
101 diag1 = diag1_shift;
102 diag2 = diag2_shift;
103 diag3 = diag3_shift;
104 }
105
106 if (feedback) {
107 diag0 +%= start0;
108 diag1 +%= start1;
109 diag2 +%= start2;
110 diag3 +%= start3;
111 }
112
113 const x0x1x10x11 = Lane{ diag0[0], diag1[1], diag0[2], diag1[3] };
114 const x12x13x6x7 = Lane{ diag1[0], diag2[1], diag1[2], diag2[3] };
115 const x8x9x2x3 = Lane{ diag2[0], diag3[1], diag2[2], diag3[3] };
116 const x4x5x14x15 = Lane{ diag3[0], diag0[1], diag3[2], diag0[3] };
117
118 x[0] = Lane{ x0x1x10x11[0], x0x1x10x11[1], x8x9x2x3[2], x8x9x2x3[3] };
119 x[1] = Lane{ x4x5x14x15[0], x4x5x14x15[1], x12x13x6x7[2], x12x13x6x7[3] };
120 x[2] = Lane{ x8x9x2x3[0], x8x9x2x3[1], x0x1x10x11[2], x0x1x10x11[3] };
121 x[3] = Lane{ x12x13x6x7[0], x12x13x6x7[1], x4x5x14x15[2], x4x5x14x15[3] };
122 }
123
124 fn hashToBytes(out: *[64]u8, x: BlockVec) void {
125 var i: usize = 0;
126 while (i < 4) : (i += 1) {
127 mem.writeIntLittle(u32, out[16 * i + 0 ..][0..4], x[i][0]);
128 mem.writeIntLittle(u32, out[16 * i + 4 ..][0..4], x[i][1]);
129 mem.writeIntLittle(u32, out[16 * i + 8 ..][0..4], x[i][2]);
130 mem.writeIntLittle(u32, out[16 * i + 12 ..][0..4], x[i][3]);
131 }
132 }
133
134 fn salsa20Xor(out: []u8, in: []const u8, key: [8]u32, d: [4]u32) void {
135 var ctx = initContext(key, d);
136 var x: BlockVec = undefined;
137 var buf: [64]u8 = undefined;
138 var i: usize = 0;
139 while (i + 64 <= in.len) : (i += 64) {
140 salsa20Core(x[0..], ctx, true);
141 hashToBytes(buf[0..], x);
142 var xout = out[i..];
143 const xin = in[i..];
144 var j: usize = 0;
145 while (j < 64) : (j += 1) {
146 xout[j] = xin[j];
147 }
148 j = 0;
149 while (j < 64) : (j += 1) {
150 xout[j] ^= buf[j];
151 }
152 ctx[2][0] +%= 1;
153 if (ctx[2][0] == 0) {
154 ctx[2][1] += 1;
155 }
156 }
157 if (i < in.len) {
158 salsa20Core(x[0..], ctx, true);
159 hashToBytes(buf[0..], x);
160
161 var xout = out[i..];
162 const xin = in[i..];
163 var j: usize = 0;
164 while (j < in.len % 64) : (j += 1) {
165 xout[j] = xin[j] ^ buf[j];
166 }
167 }
168 }
169
170 fn hsalsa20(input: [16]u8, key: [32]u8) [32]u8 {
171 var c: [4]u32 = undefined;
172 for (c) |_, i| {
173 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
174 }
175 const ctx = initContext(keyToWords(key), c);
176 var x: BlockVec = undefined;
177 salsa20Core(x[0..], ctx, false);
178 var out: [32]u8 = undefined;
179 mem.writeIntLittle(u32, out[0..4], x[0][0]);
180 mem.writeIntLittle(u32, out[4..8], x[1][1]);
181 mem.writeIntLittle(u32, out[8..12], x[2][2]);
182 mem.writeIntLittle(u32, out[12..16], x[3][3]);
183 mem.writeIntLittle(u32, out[16..20], x[1][2]);
184 mem.writeIntLittle(u32, out[20..24], x[1][3]);
185 mem.writeIntLittle(u32, out[24..28], x[2][0]);
186 mem.writeIntLittle(u32, out[28..32], x[2][1]);
187 return out;
188 }
189};
190
17const Salsa20NonVecImpl = struct {191const Salsa20NonVecImpl = struct {
18 const BlockVec = [16]u32;192 const BlockVec = [16]u32;
19193
...@@ -49,7 +223,7 @@ const Salsa20NonVecImpl = struct {...@@ -49,7 +223,7 @@ const Salsa20NonVecImpl = struct {
49 };223 };
50 }224 }
51225
52 inline fn salsa20Core(x: *BlockVec, input: BlockVec) void {226 inline fn salsa20Core(x: *BlockVec, input: BlockVec, comptime feedback: bool) void {
53 const arx_steps = comptime [_]QuarterRound{227 const arx_steps = comptime [_]QuarterRound{
54 Rp(4, 0, 12, 7), Rp(8, 4, 0, 9), Rp(12, 8, 4, 13), Rp(0, 12, 8, 18),228 Rp(4, 0, 12, 7), Rp(8, 4, 0, 9), Rp(12, 8, 4, 13), Rp(0, 12, 8, 18),
55 Rp(9, 5, 1, 7), Rp(13, 9, 5, 9), Rp(1, 13, 9, 13), Rp(5, 1, 13, 18),229 Rp(9, 5, 1, 7), Rp(13, 9, 5, 9), Rp(1, 13, 9, 13), Rp(5, 1, 13, 18),
...@@ -67,6 +241,12 @@ const Salsa20NonVecImpl = struct {...@@ -67,6 +241,12 @@ const Salsa20NonVecImpl = struct {
67 x[r.a] ^= math.rotl(u32, x[r.b] +% x[r.c], r.d);241 x[r.a] ^= math.rotl(u32, x[r.b] +% x[r.c], r.d);
68 }242 }
69 }243 }
244 if (feedback) {
245 j = 0;
246 while (j < 16) : (j += 1) {
247 x[j] +%= input[j];
248 }
249 }
70 }250 }
71251
72 fn hashToBytes(out: *[64]u8, x: BlockVec) void {252 fn hashToBytes(out: *[64]u8, x: BlockVec) void {
...@@ -75,21 +255,13 @@ const Salsa20NonVecImpl = struct {...@@ -75,21 +255,13 @@ const Salsa20NonVecImpl = struct {
75 }255 }
76 }256 }
77257
78 fn contextFeedback(x: *BlockVec, ctx: BlockVec) void {
79 var i: usize = 0;
80 while (i < 16) : (i += 1) {
81 x[i] +%= ctx[i];
82 }
83 }
84
85 fn salsa20Xor(out: []u8, in: []const u8, key: [8]u32, d: [4]u32) void {258 fn salsa20Xor(out: []u8, in: []const u8, key: [8]u32, d: [4]u32) void {
86 var ctx = initContext(key, d);259 var ctx = initContext(key, d);
87 var x: BlockVec = undefined;260 var x: BlockVec = undefined;
88 var buf: [64]u8 = undefined;261 var buf: [64]u8 = undefined;
89 var i: usize = 0;262 var i: usize = 0;
90 while (i + 64 <= in.len) : (i += 64) {263 while (i + 64 <= in.len) : (i += 64) {
91 salsa20Core(x[0..], ctx);264 salsa20Core(x[0..], ctx, true);
92 contextFeedback(&x, ctx);
93 hashToBytes(buf[0..], x);265 hashToBytes(buf[0..], x);
94 var xout = out[i..];266 var xout = out[i..];
95 const xin = in[i..];267 const xin = in[i..];
...@@ -104,8 +276,7 @@ const Salsa20NonVecImpl = struct {...@@ -104,8 +276,7 @@ const Salsa20NonVecImpl = struct {
104 ctx[9] += @boolToInt(@addWithOverflow(u32, ctx[8], 1, &ctx[8]));276 ctx[9] += @boolToInt(@addWithOverflow(u32, ctx[8], 1, &ctx[8]));
105 }277 }
106 if (i < in.len) {278 if (i < in.len) {
107 salsa20Core(x[0..], ctx);279 salsa20Core(x[0..], ctx, true);
108 contextFeedback(&x, ctx);
109 hashToBytes(buf[0..], x);280 hashToBytes(buf[0..], x);
110281
111 var xout = out[i..];282 var xout = out[i..];
...@@ -124,7 +295,7 @@ const Salsa20NonVecImpl = struct {...@@ -124,7 +295,7 @@ const Salsa20NonVecImpl = struct {
124 }295 }
125 const ctx = initContext(keyToWords(key), c);296 const ctx = initContext(keyToWords(key), c);
126 var x: BlockVec = undefined;297 var x: BlockVec = undefined;
127 salsa20Core(x[0..], ctx);298 salsa20Core(x[0..], ctx, false);
128 var out: [32]u8 = undefined;299 var out: [32]u8 = undefined;
129 mem.writeIntLittle(u32, out[0..4], x[0]);300 mem.writeIntLittle(u32, out[0..4], x[0]);
130 mem.writeIntLittle(u32, out[4..8], x[5]);301 mem.writeIntLittle(u32, out[4..8], x[5]);
...@@ -138,7 +309,7 @@ const Salsa20NonVecImpl = struct {...@@ -138,7 +309,7 @@ const Salsa20NonVecImpl = struct {
138 }309 }
139};310};
140311
141const Salsa20Impl = Salsa20NonVecImpl;312const Salsa20Impl = if (std.Target.current.cpu.arch == .x86_64) Salsa20VecImpl else Salsa20NonVecImpl;
142313
143fn keyToWords(key: [32]u8) [8]u32 {314fn keyToWords(key: [32]u8) [8]u32 {
144 var k: [8]u32 = undefined;315 var k: [8]u32 = undefined;
...@@ -381,6 +552,22 @@ pub const SealedBox = struct {...@@ -381,6 +552,22 @@ pub const SealedBox = struct {
381 }552 }
382};553};
383554
555const htest = @import("test.zig");
556
557test "(x)salsa20" {
558 const key = [_]u8{0x69} ** 32;
559 const nonce = [_]u8{0x42} ** 8;
560 const msg = [_]u8{0} ** 20;
561 var c: [msg.len]u8 = undefined;
562
563 Salsa20.xor(&c, msg[0..], 0, key, nonce);
564 htest.assertEqual("30ff9933aa6534ff5207142593cd1fca4b23bdd8", c[0..]);
565
566 const extended_nonce = [_]u8{0x42} ** 24;
567 XSalsa20.xor(&c, msg[0..], 0, key, extended_nonce);
568 htest.assertEqual("b4ab7d82e750ec07644fa3281bce6cd91d4243f9", c[0..]);
569}
570
384test "xsalsa20poly1305" {571test "xsalsa20poly1305" {
385 var msg: [100]u8 = undefined;572 var msg: [100]u8 = undefined;
386 var msg2: [msg.len]u8 = undefined;573 var msg2: [msg.len]u8 = undefined;