authorgravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2020-10-14 00:52:36-04:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2020-10-14 00:52:36-04:00
log3811602ad72ed1823e06d2e6d2cb646bc81ca5f9
tree89e49b65a7dc498e193906656c4837ff7044424b
parent0570df69b178c99f4d9d679a57d966f507dda484
parent9f109ba0ebb4167a18a80d8ff0c173d574fc4577
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

Merge pull request #6643 from jedisct1/chacha-vec

std/crypto: add a vectorized ChaCha20 implementation

1 files changed, 277 insertions(+), 118 deletions(-)

lib/std/crypto/chacha20.zig+277-118
...@@ -10,120 +10,300 @@ const mem = std.mem;...@@ -10,120 +10,300 @@ const mem = std.mem;
10const assert = std.debug.assert;10const assert = std.debug.assert;
11const testing = std.testing;11const testing = std.testing;
12const maxInt = std.math.maxInt;12const maxInt = std.math.maxInt;
13const Vector = std.meta.Vector;
13const Poly1305 = std.crypto.onetimeauth.Poly1305;14const Poly1305 = std.crypto.onetimeauth.Poly1305;
1415
15const QuarterRound = struct {16// Vectorized implementation of the core function
16 a: usize,17const ChaCha20VecImpl = struct {
17 b: usize,18 const Lane = Vector(4, u32);
18 c: usize,19 const BlockVec = [4]Lane;
19 d: usize,20
20};21 fn initContext(key: [8]u32, d: [4]u32) BlockVec {
22 const c = "expand 32-byte k";
23 const constant_le = comptime Lane{
24 mem.readIntLittle(u32, c[0..4]),
25 mem.readIntLittle(u32, c[4..8]),
26 mem.readIntLittle(u32, c[8..12]),
27 mem.readIntLittle(u32, c[12..16]),
28 };
29 return BlockVec{
30 constant_le,
31 Lane{ key[0], key[1], key[2], key[3] },
32 Lane{ key[4], key[5], key[6], key[7] },
33 Lane{ d[0], d[1], d[2], d[3] },
34 };
35 }
2136
22fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {37 inline fn rot(x: Lane, comptime n: comptime_int) Lane {
23 return QuarterRound{38 return (x << @splat(4, @as(u5, n))) | (x >> @splat(4, @as(u5, 32 - n)));
24 .a = a,39 }
25 .b = b,
26 .c = c,
27 .d = d,
28 };
29}
3040
31fn initContext(key: [8]u32, d: [4]u32) [16]u32 {41 inline fn chacha20Core(x: *BlockVec, input: BlockVec) void {
32 var ctx: [16]u32 = undefined;42 x.* = input;
33 const c = "expand 32-byte k";43
34 const constant_le = comptime [_]u32{44 var r: usize = 0;
35 mem.readIntLittle(u32, c[0..4]),45 while (r < 20) : (r += 2) {
36 mem.readIntLittle(u32, c[4..8]),46 x[0] +%= x[1];
37 mem.readIntLittle(u32, c[8..12]),47 x[3] ^= x[0];
38 mem.readIntLittle(u32, c[12..16]),48 x[3] = rot(x[3], 16);
39 };49
40 mem.copy(u32, ctx[0..], constant_le[0..4]);50 x[2] +%= x[3];
41 mem.copy(u32, ctx[4..12], key[0..8]);51 x[1] ^= x[2];
42 mem.copy(u32, ctx[12..16], d[0..4]);52 x[1] = rot(x[1], 12);
53
54 x[0] +%= x[1];
55 x[3] ^= x[0];
56 x[0] = @shuffle(u32, x[0], undefined, [_]i32{ 3, 0, 1, 2 });
57 x[3] = rot(x[3], 8);
58
59 x[2] +%= x[3];
60 x[3] = @shuffle(u32, x[3], undefined, [_]i32{ 2, 3, 0, 1 });
61 x[1] ^= x[2];
62 x[2] = @shuffle(u32, x[2], undefined, [_]i32{ 1, 2, 3, 0 });
63 x[1] = rot(x[1], 7);
64
65 x[0] +%= x[1];
66 x[3] ^= x[0];
67 x[3] = rot(x[3], 16);
68
69 x[2] +%= x[3];
70 x[1] ^= x[2];
71 x[1] = rot(x[1], 12);
72
73 x[0] +%= x[1];
74 x[3] ^= x[0];
75 x[0] = @shuffle(u32, x[0], undefined, [_]i32{ 1, 2, 3, 0 });
76 x[3] = rot(x[3], 8);
77
78 x[2] +%= x[3];
79 x[3] = @shuffle(u32, x[3], undefined, [_]i32{ 2, 3, 0, 1 });
80 x[1] ^= x[2];
81 x[2] = @shuffle(u32, x[2], undefined, [_]i32{ 3, 0, 1, 2 });
82 x[1] = rot(x[1], 7);
83 }
84 }
4385
44 return ctx;86 inline fn hashToBytes(out: *[64]u8, x: BlockVec) void {
45}87 var i: usize = 0;
88 while (i < 4) : (i += 1) {
89 mem.writeIntLittle(u32, out[16 * i + 0 ..][0..4], x[i][0]);
90 mem.writeIntLittle(u32, out[16 * i + 4 ..][0..4], x[i][1]);
91 mem.writeIntLittle(u32, out[16 * i + 8 ..][0..4], x[i][2]);
92 mem.writeIntLittle(u32, out[16 * i + 12 ..][0..4], x[i][3]);
93 }
94 }
4695
47// The chacha family of ciphers are based on the salsa family.96 inline fn contextFeedback(x: *BlockVec, ctx: BlockVec) void {
48inline fn chacha20Core(x: []u32, input: [16]u32) void {97 x[0] +%= ctx[0];
49 for (x) |_, i|98 x[1] +%= ctx[1];
50 x[i] = input[i];99 x[2] +%= ctx[2];
51100 x[3] +%= ctx[3];
52 const rounds = comptime [_]QuarterRound{101 }
53 Rp(0, 4, 8, 12),
54 Rp(1, 5, 9, 13),
55 Rp(2, 6, 10, 14),
56 Rp(3, 7, 11, 15),
57 Rp(0, 5, 10, 15),
58 Rp(1, 6, 11, 12),
59 Rp(2, 7, 8, 13),
60 Rp(3, 4, 9, 14),
61 };
62102
63 comptime var j: usize = 0;103 fn chaCha20Internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void {
64 inline while (j < 20) : (j += 2) {104 var ctx = initContext(key, counter);
65 // two-round cycles105 var x: BlockVec = undefined;
66 inline for (rounds) |r| {106 var buf: [64]u8 = undefined;
67 x[r.a] +%= x[r.b];107 var i: usize = 0;
68 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 16));108 while (i + 64 <= in.len) : (i += 64) {
69 x[r.c] +%= x[r.d];109 chacha20Core(x[0..], ctx);
70 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 12));110 contextFeedback(&x, ctx);
71 x[r.a] +%= x[r.b];111 hashToBytes(buf[0..], x);
72 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 8));112
73 x[r.c] +%= x[r.d];113 var xout = out[i..];
74 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 7));114 const xin = in[i..];
115 var j: usize = 0;
116 while (j < 64) : (j += 1) {
117 xout[j] = xin[j];
118 }
119 j = 0;
120 while (j < 64) : (j += 1) {
121 xout[j] ^= buf[j];
122 }
123 ctx[3][0] += 1;
124 }
125 if (i < in.len) {
126 chacha20Core(x[0..], ctx);
127 contextFeedback(&x, ctx);
128 hashToBytes(buf[0..], x);
129
130 var xout = out[i..];
131 const xin = in[i..];
132 var j: usize = 0;
133 while (j < in.len % 64) : (j += 1) {
134 xout[j] = xin[j] ^ buf[j];
135 }
75 }136 }
76 }137 }
77}
78138
79fn hashToBytes(out: []u8, x: [16]u32) void {139 fn hchacha20(input: [16]u8, key: [32]u8) [32]u8 {
80 for (x) |_, i| {140 var c: [4]u32 = undefined;
81 mem.writeIntLittle(u32, out[4 * i ..][0..4], x[i]);141 for (c) |_, i| {
142 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
143 }
144 const ctx = initContext(keyToWords(key), c);
145 var x: BlockVec = undefined;
146 chacha20Core(x[0..], ctx);
147 var out: [32]u8 = undefined;
148 mem.writeIntLittle(u32, out[0..4], x[0][0]);
149 mem.writeIntLittle(u32, out[4..8], x[0][1]);
150 mem.writeIntLittle(u32, out[8..12], x[0][2]);
151 mem.writeIntLittle(u32, out[12..16], x[0][3]);
152 mem.writeIntLittle(u32, out[16..20], x[3][0]);
153 mem.writeIntLittle(u32, out[20..24], x[3][1]);
154 mem.writeIntLittle(u32, out[24..28], x[3][2]);
155 mem.writeIntLittle(u32, out[28..32], x[3][3]);
156 return out;
82 }157 }
83}158};
84159
85fn chaCha20_internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void {160// Non-vectorized implementation of the core function
86 var ctx = initContext(key, counter);161const ChaCha20NonVecImpl = struct {
87 var remaining: usize = if (in.len > out.len) in.len else out.len;162 const BlockVec = [16]u32;
88 var cursor: usize = 0;163
164 fn initContext(key: [8]u32, d: [4]u32) BlockVec {
165 const c = "expand 32-byte k";
166 const constant_le = comptime [4]u32{
167 mem.readIntLittle(u32, c[0..4]),
168 mem.readIntLittle(u32, c[4..8]),
169 mem.readIntLittle(u32, c[8..12]),
170 mem.readIntLittle(u32, c[12..16]),
171 };
172 return BlockVec{
173 constant_le[0], constant_le[1], constant_le[2], constant_le[3],
174 key[0], key[1], key[2], key[3],
175 key[4], key[5], key[6], key[7],
176 d[0], d[1], d[2], d[3],
177 };
178 }
89179
90 while (true) {180 const QuarterRound = struct {
91 var x: [16]u32 = undefined;181 a: usize,
92 var buf: [64]u8 = undefined;182 b: usize,
93 chacha20Core(x[0..], ctx);183 c: usize,
94 for (x) |_, i| {184 d: usize,
95 x[i] +%= ctx[i];185 };
186
187 fn Rp(a: usize, b: usize, c: usize, d: usize) QuarterRound {
188 return QuarterRound{
189 .a = a,
190 .b = b,
191 .c = c,
192 .d = d,
193 };
194 }
195
196 inline fn chacha20Core(x: *BlockVec, input: BlockVec) void {
197 x.* = input;
198
199 const rounds = comptime [_]QuarterRound{
200 Rp(0, 4, 8, 12),
201 Rp(1, 5, 9, 13),
202 Rp(2, 6, 10, 14),
203 Rp(3, 7, 11, 15),
204 Rp(0, 5, 10, 15),
205 Rp(1, 6, 11, 12),
206 Rp(2, 7, 8, 13),
207 Rp(3, 4, 9, 14),
208 };
209
210 comptime var j: usize = 0;
211 inline while (j < 20) : (j += 2) {
212 inline for (rounds) |r| {
213 x[r.a] +%= x[r.b];
214 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 16));
215 x[r.c] +%= x[r.d];
216 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 12));
217 x[r.a] +%= x[r.b];
218 x[r.d] = std.math.rotl(u32, x[r.d] ^ x[r.a], @as(u32, 8));
219 x[r.c] +%= x[r.d];
220 x[r.b] = std.math.rotl(u32, x[r.b] ^ x[r.c], @as(u32, 7));
221 }
96 }222 }
97 hashToBytes(buf[0..], x);223 }
98 if (remaining < 64) {224
99 var i: usize = 0;225 inline fn hashToBytes(out: *[64]u8, x: BlockVec) void {
100 while (i < remaining) : (i += 1)226 var i: usize = 0;
101 out[cursor + i] = in[cursor + i] ^ buf[i];227 while (i < 4) : (i += 1) {
102 return;228 mem.writeIntLittle(u32, out[16 * i + 0 ..][0..4], x[i * 4 + 0]);
229 mem.writeIntLittle(u32, out[16 * i + 4 ..][0..4], x[i * 4 + 1]);
230 mem.writeIntLittle(u32, out[16 * i + 8 ..][0..4], x[i * 4 + 2]);
231 mem.writeIntLittle(u32, out[16 * i + 12 ..][0..4], x[i * 4 + 3]);
103 }232 }
233 }
104234
235 inline fn contextFeedback(x: *BlockVec, ctx: BlockVec) void {
105 var i: usize = 0;236 var i: usize = 0;
106 while (i < 64) : (i += 1)237 while (i < 16) : (i += 1) {
107 out[cursor + i] = in[cursor + i] ^ buf[i];238 x[i] +%= ctx[i];
239 }
240 }
108241
109 cursor += 64;242 fn chaCha20Internal(out: []u8, in: []const u8, key: [8]u32, counter: [4]u32) void {
110 remaining -= 64;243 var ctx = initContext(key, counter);
244 var x: BlockVec = undefined;
245 var buf: [64]u8 = undefined;
246 var i: usize = 0;
247 while (i + 64 <= in.len) : (i += 64) {
248 chacha20Core(x[0..], ctx);
249 contextFeedback(&x, ctx);
250 hashToBytes(buf[0..], x);
251
252 var xout = out[i..];
253 const xin = in[i..];
254 var j: usize = 0;
255 while (j < 64) : (j += 1) {
256 xout[j] = xin[j];
257 }
258 j = 0;
259 while (j < 64) : (j += 1) {
260 xout[j] ^= buf[j];
261 }
262 ctx[12] += 1;
263 }
264 if (i < in.len) {
265 chacha20Core(x[0..], ctx);
266 contextFeedback(&x, ctx);
267 hashToBytes(buf[0..], x);
268
269 var xout = out[i..];
270 const xin = in[i..];
271 var j: usize = 0;
272 while (j < in.len % 64) : (j += 1) {
273 xout[j] = xin[j] ^ buf[j];
274 }
275 }
276 }
111277
112 ctx[12] += 1;278 fn hchacha20(input: [16]u8, key: [32]u8) [32]u8 {
279 var c: [4]u32 = undefined;
280 for (c) |_, i| {
281 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
282 }
283 const ctx = initContext(keyToWords(key), c);
284 var x: BlockVec = undefined;
285 chacha20Core(x[0..], ctx);
286 var out: [32]u8 = undefined;
287 mem.writeIntLittle(u32, out[0..4], x[0]);
288 mem.writeIntLittle(u32, out[4..8], x[1]);
289 mem.writeIntLittle(u32, out[8..12], x[2]);
290 mem.writeIntLittle(u32, out[12..16], x[3]);
291 mem.writeIntLittle(u32, out[16..20], x[12]);
292 mem.writeIntLittle(u32, out[20..24], x[13]);
293 mem.writeIntLittle(u32, out[24..28], x[14]);
294 mem.writeIntLittle(u32, out[28..32], x[15]);
295 return out;
113 }296 }
114}297};
298
299const ChaCha20Impl = if (std.Target.current.cpu.arch == .x86_64) ChaCha20VecImpl else ChaCha20NonVecImpl;
115300
116fn keyToWords(key: [32]u8) [8]u32 {301fn keyToWords(key: [32]u8) [8]u32 {
117 var k: [8]u32 = undefined;302 var k: [8]u32 = undefined;
118 k[0] = mem.readIntLittle(u32, key[0..4]);303 var i: usize = 0;
119 k[1] = mem.readIntLittle(u32, key[4..8]);304 while (i < 8) : (i += 1) {
120 k[2] = mem.readIntLittle(u32, key[8..12]);305 k[i] = mem.readIntLittle(u32, key[i * 4 ..][0..4]);
121 k[3] = mem.readIntLittle(u32, key[12..16]);306 }
122 k[4] = mem.readIntLittle(u32, key[16..20]);
123 k[5] = mem.readIntLittle(u32, key[20..24]);
124 k[6] = mem.readIntLittle(u32, key[24..28]);
125 k[7] = mem.readIntLittle(u32, key[28..32]);
126
127 return k;307 return k;
128}308}
129309
...@@ -145,7 +325,7 @@ pub const ChaCha20IETF = struct {...@@ -145,7 +325,7 @@ pub const ChaCha20IETF = struct {
145 c[1] = mem.readIntLittle(u32, nonce[0..4]);325 c[1] = mem.readIntLittle(u32, nonce[0..4]);
146 c[2] = mem.readIntLittle(u32, nonce[4..8]);326 c[2] = mem.readIntLittle(u32, nonce[4..8]);
147 c[3] = mem.readIntLittle(u32, nonce[8..12]);327 c[3] = mem.readIntLittle(u32, nonce[8..12]);
148 chaCha20_internal(out, in, keyToWords(key), c);328 ChaCha20Impl.chaCha20Internal(out, in, keyToWords(key), c);
149 }329 }
150};330};
151331
...@@ -171,7 +351,7 @@ pub const ChaCha20With64BitNonce = struct {...@@ -171,7 +351,7 @@ pub const ChaCha20With64BitNonce = struct {
171351
172 // first partial big block352 // first partial big block
173 if (((@intCast(u64, maxInt(u32) - @truncate(u32, counter)) + 1) << 6) < in.len) {353 if (((@intCast(u64, maxInt(u32) - @truncate(u32, counter)) + 1) << 6) < in.len) {
174 chaCha20_internal(out[cursor..big_block], in[cursor..big_block], k, c);354 ChaCha20Impl.chaCha20Internal(out[cursor..big_block], in[cursor..big_block], k, c);
175 cursor = big_block - cursor;355 cursor = big_block - cursor;
176 c[1] += 1;356 c[1] += 1;
177 if (comptime @sizeOf(usize) > 4) {357 if (comptime @sizeOf(usize) > 4) {
...@@ -179,14 +359,14 @@ pub const ChaCha20With64BitNonce = struct {...@@ -179,14 +359,14 @@ pub const ChaCha20With64BitNonce = struct {
179 var remaining_blocks: u32 = @intCast(u32, (in.len / big_block));359 var remaining_blocks: u32 = @intCast(u32, (in.len / big_block));
180 var i: u32 = 0;360 var i: u32 = 0;
181 while (remaining_blocks > 0) : (remaining_blocks -= 1) {361 while (remaining_blocks > 0) : (remaining_blocks -= 1) {
182 chaCha20_internal(out[cursor .. cursor + big_block], in[cursor .. cursor + big_block], k, c);362 ChaCha20Impl.chaCha20Internal(out[cursor .. cursor + big_block], in[cursor .. cursor + big_block], k, c);
183 c[1] += 1; // upper 32-bit of counter, generic chaCha20_internal() doesn't know about this.363 c[1] += 1; // upper 32-bit of counter, generic chaCha20Internal() doesn't know about this.
184 cursor += big_block;364 cursor += big_block;
185 }365 }
186 }366 }
187 }367 }
188368
189 chaCha20_internal(out[cursor..], in[cursor..], k, c);369 ChaCha20Impl.chaCha20Internal(out[cursor..], in[cursor..], k, c);
190 }370 }
191};371};
192372
...@@ -533,33 +713,12 @@ fn chacha20poly1305Open(dst: []u8, ciphertextAndTag: []const u8, data: []const u...@@ -533,33 +713,12 @@ fn chacha20poly1305Open(dst: []u8, ciphertextAndTag: []const u8, data: []const u
533 return try chacha20poly1305OpenDetached(dst, ciphertextAndTag[0..ciphertextLen], ciphertextAndTag[ciphertextLen..][0..chacha20poly1305_tag_size], data, key, nonce);713 return try chacha20poly1305OpenDetached(dst, ciphertextAndTag[0..ciphertextLen], ciphertextAndTag[ciphertextLen..][0..chacha20poly1305_tag_size], data, key, nonce);
534}714}
535715
536fn hchacha20(input: [16]u8, key: [32]u8) [32]u8 {
537 var c: [4]u32 = undefined;
538 for (c) |_, i| {
539 c[i] = mem.readIntLittle(u32, input[4 * i ..][0..4]);
540 }
541 const ctx = initContext(keyToWords(key), c);
542 var x: [16]u32 = undefined;
543 chacha20Core(x[0..], ctx);
544 var out: [32]u8 = undefined;
545 mem.writeIntLittle(u32, out[0..4], x[0]);
546 mem.writeIntLittle(u32, out[4..8], x[1]);
547 mem.writeIntLittle(u32, out[8..12], x[2]);
548 mem.writeIntLittle(u32, out[12..16], x[3]);
549 mem.writeIntLittle(u32, out[16..20], x[12]);
550 mem.writeIntLittle(u32, out[20..24], x[13]);
551 mem.writeIntLittle(u32, out[24..28], x[14]);
552 mem.writeIntLittle(u32, out[28..32], x[15]);
553
554 return out;
555}
556
557fn extend(key: [32]u8, nonce: [24]u8) struct { key: [32]u8, nonce: [12]u8 } {716fn extend(key: [32]u8, nonce: [24]u8) struct { key: [32]u8, nonce: [12]u8 } {
558 var subnonce: [12]u8 = undefined;717 var subnonce: [12]u8 = undefined;
559 mem.set(u8, subnonce[0..4], 0);718 mem.set(u8, subnonce[0..4], 0);
560 mem.copy(u8, subnonce[4..], nonce[16..24]);719 mem.copy(u8, subnonce[4..], nonce[16..24]);
561 return .{720 return .{
562 .key = hchacha20(nonce[0..16].*, key),721 .key = ChaCha20Impl.hchacha20(nonce[0..16].*, key),
563 .nonce = subnonce,722 .nonce = subnonce,
564 };723 };
565}724}