authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-08 14:41:06+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-08 14:55:08+02:00
logf39dc00ed467dd34999f71adf560b71383755a80
tree87ad759ca9ac975a10396fd98f1db84d8dbf796b
parent83eda214882e33e65794cee1e7c0d87777b31bf4

std/crypto/aes: add AES hardware acceleration on aarch64

2 files changed, 499 insertions(+), 1 deletions(-)

lib/std/crypto/aes.zig+9-1
...@@ -10,7 +10,15 @@ const builtin = std.builtin;...@@ -10,7 +10,15 @@ const builtin = std.builtin;
1010
11const has_aesni = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .aes);11const has_aesni = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .aes);
12const has_avx = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .avx);12const has_avx = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .avx);
13const impl = if (std.Target.current.cpu.arch == .x86_64 and has_aesni and has_avx) @import("aes/aesni.zig") else @import("aes/soft.zig");13const has_armaes = comptime std.Target.aarch64.featureSetHas(std.Target.current.cpu.features, .aes);
14const impl = if (std.Target.current.cpu.arch == .x86_64 and has_aesni and has_avx) impl: {
15 break :impl @import("aes/aesni.zig");
16} else if (std.Target.current.cpu.arch == .aarch64 and has_armaes)
17impl: {
18 break :impl @import("aes/armcrypto.zig");
19} else impl: {
20 break :impl @import("aes/soft.zig");
21};
1422
15pub const Block = impl.Block;23pub const Block = impl.Block;
16pub const AESEncryptCtx = impl.AESEncryptCtx;24pub const AESEncryptCtx = impl.AESEncryptCtx;
lib/std/crypto/aes/armcrypto.zig created+490
...@@ -0,0 +1,490 @@
1// SPDX-License-Identifier: MIT
2// Copyright (c) 2015-2020 Zig Contributors
3// This file is part of [zig](https://ziglang.org/), which is MIT licensed.
4// The MIT license requires this copyright notice to be included in all copies
5// and substantial portions of the software.
6
7const std = @import("../../std.zig");
8const mem = std.mem;
9const debug = std.debug;
10const Vector = std.meta.Vector;
11
12const BlockVec = Vector(2, u64);
13
14/// A single AES block.
15pub const Block = struct {
16 pub const block_size: usize = 16;
17
18 /// Internal representation of a block.
19 repr: BlockVec,
20
21 /// Convert a byte sequence into an internal representation.
22 pub inline fn fromBytes(bytes: *const [16]u8) Block {
23 const repr = mem.bytesToValue(BlockVec, bytes);
24 return Block{ .repr = repr };
25 }
26
27 /// Convert the internal representation of a block into a byte sequence.
28 pub inline fn toBytes(block: Block) [16]u8 {
29 return mem.toBytes(block.repr);
30 }
31
32 /// XOR the block with a byte sequence.
33 pub inline fn xorBytes(block: Block, bytes: *const [16]u8) [16]u8 {
34 const x = block.repr ^ fromBytes(bytes).repr;
35 return mem.toBytes(x);
36 }
37
38 const zero = Vector(2, u64){ 0, 0 };
39
40 /// Encrypt a block with a round key.
41 pub inline fn encrypt(block: Block, round_key: Block) Block {
42 return Block{
43 .repr = asm (
44 \\ mov %[out].16b, %[in].16b
45 \\ aese %[out].16b, %[zero].16b
46 \\ aesmc %[out].16b, %[out].16b
47 \\ eor %[out].16b, %[out].16b, %[rk].16b
48 : [out] "=&x" (-> BlockVec)
49 : [in] "x" (block.repr),
50 [rk] "x" (round_key.repr),
51 [zero] "x" (zero)
52 ),
53 };
54 }
55
56 /// Encrypt a block with the last round key.
57 pub inline fn encryptLast(block: Block, round_key: Block) Block {
58 return Block{
59 .repr = asm (
60 \\ mov %[out].16b, %[in].16b
61 \\ aese %[out].16b, %[zero].16b
62 \\ eor %[out].16b, %[out].16b, %[rk].16b
63 : [out] "=&x" (-> BlockVec)
64 : [in] "x" (block.repr),
65 [rk] "x" (round_key.repr),
66 [zero] "x" (zero)
67 ),
68 };
69 }
70
71 /// Decrypt a block with a round key.
72 pub inline fn decrypt(block: Block, inv_round_key: Block) Block {
73 return Block{
74 .repr = asm (
75 \\ mov %[out].16b, %[in].16b
76 \\ aesd %[out].16b, %[zero].16b
77 \\ aesimc %[out].16b, %[out].16b
78 \\ eor %[out].16b, %[out].16b, %[rk].16b
79 : [out] "=&x" (-> BlockVec)
80 : [in] "x" (block.repr),
81 [rk] "x" (inv_round_key.repr),
82 [zero] "x" (zero)
83 ),
84 };
85 }
86
87 /// Decrypt a block with the last round key.
88 pub inline fn decryptLast(block: Block, inv_round_key: Block) Block {
89 return Block{
90 .repr = asm (
91 \\ mov %[out].16b, %[in].16b
92 \\ aesd %[out].16b, %[zero].16b
93 \\ eor %[out].16b, %[out].16b, %[rk].16b
94 : [out] "=&x" (-> BlockVec)
95 : [in] "x" (block.repr),
96 [rk] "x" (inv_round_key.repr),
97 [zero] "x" (zero)
98 ),
99 };
100 }
101
102 /// Apply the bitwise XOR operation to the content of two blocks.
103 pub inline fn xorBlocks(block1: Block, block2: Block) Block {
104 return Block{ .repr = block1.repr ^ block2.repr };
105 }
106
107 /// Apply the bitwise AND operation to the content of two blocks.
108 pub inline fn andBlocks(block1: Block, block2: Block) Block {
109 return Block{ .repr = block1.repr & block2.repr };
110 }
111
112 /// Apply the bitwise OR operation to the content of two blocks.
113 pub inline fn orBlocks(block1: Block, block2: Block) Block {
114 return Block{ .repr = block1.repr | block2.repr };
115 }
116
117 /// Perform operations on multiple blocks in parallel.
118 pub const parallel = struct {
119 /// The recommended number of AES encryption/decryption to perform in parallel for the chosen implementation.
120 pub const optimal_parallel_blocks = 8;
121
122 /// Encrypt multiple blocks in parallel, each their own round key.
123 pub inline fn encryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block {
124 comptime var i = 0;
125 var out: [count]Block = undefined;
126 inline while (i < count) : (i += 1) {
127 out[i] = blocks[i].encrypt(round_keys[i]);
128 }
129 return out;
130 }
131
132 /// Decrypt multiple blocks in parallel, each their own round key.
133 pub inline fn decryptParallel(comptime count: usize, blocks: [count]Block, round_keys: [count]Block) [count]Block {
134 comptime var i = 0;
135 var out: [count]Block = undefined;
136 inline while (i < count) : (i += 1) {
137 out[i] = blocks[i].decrypt(round_keys[i]);
138 }
139 return out;
140 }
141
142 /// Encrypt multple blocks in parallel with the same round key.
143 pub inline fn encryptWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block {
144 comptime var i = 0;
145 var out: [count]Block = undefined;
146 inline while (i < count) : (i += 1) {
147 out[i] = blocks[i].encrypt(round_key);
148 }
149 return out;
150 }
151
152 /// Decrypt multple blocks in parallel with the same round key.
153 pub inline fn decryptWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block {
154 comptime var i = 0;
155 var out: [count]Block = undefined;
156 inline while (i < count) : (i += 1) {
157 out[i] = blocks[i].decrypt(round_key);
158 }
159 return out;
160 }
161
162 /// Encrypt multple blocks in parallel with the same last round key.
163 pub inline fn encryptLastWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block {
164 comptime var i = 0;
165 var out: [count]Block = undefined;
166 inline while (i < count) : (i += 1) {
167 out[i] = blocks[i].encryptLast(round_key);
168 }
169 return out;
170 }
171
172 /// Decrypt multple blocks in parallel with the same last round key.
173 pub inline fn decryptLastWide(comptime count: usize, blocks: [count]Block, round_key: Block) [count]Block {
174 comptime var i = 0;
175 var out: [count]Block = undefined;
176 inline while (i < count) : (i += 1) {
177 out[i] = blocks[i].decryptLast(round_key);
178 }
179 return out;
180 }
181 };
182};
183
184fn KeySchedule(comptime AES: type) type {
185 std.debug.assert(AES.rounds == 10 or AES.rounds == 14);
186 const rounds = AES.rounds;
187
188 return struct {
189 const Self = @This();
190
191 const zero = Vector(2, u64){ 0, 0 };
192 const mask1 = @Vector(16, u8){ 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12 };
193 const mask2 = @Vector(16, u8){ 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15, 12, 13, 14, 15 };
194
195 round_keys: [rounds + 1]Block,
196
197 fn drc128(comptime rc: u8, t: BlockVec) BlockVec {
198 var v1: BlockVec = undefined;
199 var v2: BlockVec = undefined;
200 var v3: BlockVec = undefined;
201 var v4: BlockVec = undefined;
202
203 return asm (
204 \\ movi %[v2].4s, %[rc]
205 \\ tbl %[v4].16b, {%[t].16b}, %[mask].16b
206 \\ ext %[r].16b, %[zero].16b, %[t].16b, #12
207 \\ aese %[v4].16b, %[zero].16b
208 \\ eor %[v2].16b, %[r].16b, %[v2].16b
209 \\ ext %[r].16b, %[zero].16b, %[r].16b, #12
210 \\ eor %[v1].16b, %[v2].16b, %[t].16b
211 \\ ext %[v3].16b, %[zero].16b, %[r].16b, #12
212 \\ eor %[v1].16b, %[v1].16b, %[r].16b
213 \\ eor %[r].16b, %[v1].16b, %[v3].16b
214 \\ eor %[r].16b, %[r].16b, %[v4].16b
215 : [r] "=&x" (-> BlockVec),
216 [v1] "=&x" (v1),
217 [v2] "=&x" (v2),
218 [v3] "=&x" (v3),
219 [v4] "=&x" (v4)
220 : [rc] "N" (rc),
221 [t] "x" (t),
222 [zero] "x" (zero),
223 [mask] "x" (mask1)
224 );
225 }
226
227 fn drc256(comptime second: bool, comptime rc: u8, t: BlockVec, tx: BlockVec) BlockVec {
228 var v1: BlockVec = undefined;
229 var v2: BlockVec = undefined;
230 var v3: BlockVec = undefined;
231 var v4: BlockVec = undefined;
232
233 return asm (
234 \\ movi %[v2].4s, %[rc]
235 \\ tbl %[v4].16b, {%[t].16b}, %[mask].16b
236 \\ ext %[r].16b, %[zero].16b, %[tx].16b, #12
237 \\ aese %[v4].16b, %[zero].16b
238 \\ eor %[v1].16b, %[tx].16b, %[r].16b
239 \\ ext %[r].16b, %[zero].16b, %[r].16b, #12
240 \\ eor %[v1].16b, %[v1].16b, %[r].16b
241 \\ ext %[v3].16b, %[zero].16b, %[r].16b, #12
242 \\ eor %[v1].16b, %[v1].16b, %[v2].16b
243 \\ eor %[v1].16b, %[v1].16b, %[v3].16b
244 \\ eor %[r].16b, %[v1].16b, %[v4].16b
245 : [r] "=&x" (-> BlockVec),
246 [v1] "=&x" (v1),
247 [v2] "=&x" (v2),
248 [v3] "=&x" (v3),
249 [v4] "=&x" (v4)
250 : [rc] "N" (if (second) @as(u8, 0) else rc),
251 [t] "x" (t),
252 [tx] "x" (tx),
253 [zero] "x" (zero),
254 [mask] "x" (if (second) mask2 else mask1)
255 );
256 }
257
258 fn expand128(t1: *Block) Self {
259 var round_keys: [11]Block = undefined;
260 const rcs = [_]u8{ 1, 2, 4, 8, 16, 32, 64, 128, 27, 54 };
261 inline for (rcs) |rc, round| {
262 round_keys[round] = t1.*;
263 t1.repr = drc128(rc, t1.repr);
264 }
265 round_keys[rcs.len] = t1.*;
266 return Self{ .round_keys = round_keys };
267 }
268
269 fn expand256(t1: *Block, t2: *Block) Self {
270 var round_keys: [15]Block = undefined;
271 const rcs = [_]u8{ 1, 2, 4, 8, 16, 32 };
272 round_keys[0] = t1.*;
273 inline for (rcs) |rc, round| {
274 round_keys[round * 2 + 1] = t2.*;
275 t1.repr = drc256(false, rc, t2.repr, t1.repr);
276 round_keys[round * 2 + 2] = t1.*;
277 t2.repr = drc256(true, rc, t1.repr, t2.repr);
278 }
279 round_keys[rcs.len * 2 + 1] = t2.*;
280 t1.repr = drc256(false, 64, t2.repr, t1.repr);
281 round_keys[rcs.len * 2 + 2] = t1.*;
282 return Self{ .round_keys = round_keys };
283 }
284
285 /// Invert the key schedule.
286 pub fn invert(key_schedule: Self) Self {
287 const round_keys = &key_schedule.round_keys;
288 var inv_round_keys: [rounds + 1]Block = undefined;
289 inv_round_keys[0] = round_keys[rounds];
290 comptime var i = 1;
291 inline while (i < rounds) : (i += 1) {
292 inv_round_keys[i] = Block{
293 .repr = asm (
294 \\ aesimc %[inv_rk].16b, %[rk].16b
295 : [inv_rk] "=x" (-> BlockVec)
296 : [rk] "x" (round_keys[rounds - i].repr)
297 ),
298 };
299 }
300 inv_round_keys[rounds] = round_keys[0];
301 return Self{ .round_keys = inv_round_keys };
302 }
303 };
304}
305
306/// A context to perform encryption using the standard AES key schedule.
307pub fn AESEncryptCtx(comptime AES: type) type {
308 std.debug.assert(AES.key_bits == 128 or AES.key_bits == 256);
309 const rounds = AES.rounds;
310
311 return struct {
312 const Self = @This();
313 pub const block = AES.block;
314 pub const block_size = block.block_size;
315 key_schedule: KeySchedule(AES),
316
317 /// Create a new encryption context with the given key.
318 pub fn init(key: [AES.key_bits / 8]u8) Self {
319 var t1 = Block.fromBytes(key[0..16]);
320 const key_schedule = if (AES.key_bits == 128) ks: {
321 break :ks KeySchedule(AES).expand128(&t1);
322 } else ks: {
323 var t2 = Block.fromBytes(key[16..32]);
324 break :ks KeySchedule(AES).expand256(&t1, &t2);
325 };
326 return Self{
327 .key_schedule = key_schedule,
328 };
329 }
330
331 /// Encrypt a single block.
332 pub fn encrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
333 const round_keys = ctx.key_schedule.round_keys;
334 var t = Block.fromBytes(src).xorBlocks(round_keys[0]);
335 comptime var i = 1;
336 inline while (i < rounds) : (i += 1) {
337 t = t.encrypt(round_keys[i]);
338 }
339 t = t.encryptLast(round_keys[rounds]);
340 dst.* = t.toBytes();
341 }
342
343 /// Encrypt+XOR a single block.
344 pub fn xor(ctx: Self, dst: *[16]u8, src: *const [16]u8, counter: [16]u8) void {
345 const round_keys = ctx.key_schedule.round_keys;
346 var t = Block.fromBytes(&counter).xorBlocks(round_keys[0]);
347 comptime var i = 1;
348 inline while (i < rounds) : (i += 1) {
349 t = t.encrypt(round_keys[i]);
350 }
351 t = t.encryptLast(round_keys[rounds]);
352 dst.* = t.xorBytes(src);
353 }
354
355 /// Encrypt multiple blocks, possibly leveraging parallelization.
356 pub fn encryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {
357 const round_keys = ctx.key_schedule.round_keys;
358 var ts: [count]Block = undefined;
359 comptime var j = 0;
360 inline while (j < count) : (j += 1) {
361 ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xorBlocks(round_keys[0]);
362 }
363 comptime var i = 1;
364 inline while (i < rounds) : (i += 1) {
365 ts = Block.parallel.encryptWide(count, ts, round_keys[i]);
366 }
367 i = 1;
368 inline while (i < count) : (i += 1) {
369 ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]);
370 }
371 j = 0;
372 inline while (j < count) : (j += 1) {
373 dst[16 * j .. 16 * j + 16].* = ts[j].toBytes();
374 }
375 }
376
377 /// Encrypt+XOR multiple blocks, possibly leveraging parallelization.
378 pub fn xorWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8, counters: [16 * count]u8) void {
379 const round_keys = ctx.key_schedule.round_keys;
380 var ts: [count]Block = undefined;
381 comptime var j = 0;
382 inline while (j < count) : (j += 1) {
383 ts[j] = Block.fromBytes(counters[j * 16 .. j * 16 + 16][0..16]).xorBlocks(round_keys[0]);
384 }
385 comptime var i = 1;
386 inline while (i < rounds) : (i += 1) {
387 ts = Block.parallel.encryptWide(count, ts, round_keys[i]);
388 }
389 ts = Block.parallel.encryptLastWide(count, ts, round_keys[i]);
390 j = 0;
391 inline while (j < count) : (j += 1) {
392 dst[16 * j .. 16 * j + 16].* = ts[j].xorBytes(src[16 * j .. 16 * j + 16]);
393 }
394 }
395 };
396}
397
398/// A context to perform decryption using the standard AES key schedule.
399pub fn AESDecryptCtx(comptime AES: type) type {
400 std.debug.assert(AES.key_bits == 128 or AES.key_bits == 256);
401 const rounds = AES.rounds;
402
403 return struct {
404 const Self = @This();
405 pub const block = AES.block;
406 pub const block_size = block.block_size;
407 key_schedule: KeySchedule(AES),
408
409 /// Create a decryption context from an existing encryption context.
410 pub fn initFromEnc(ctx: AESEncryptCtx(AES)) Self {
411 return Self{
412 .key_schedule = ctx.key_schedule.invert(),
413 };
414 }
415
416 /// Create a new decryption context with the given key.
417 pub fn init(key: [AES.key_bits / 8]u8) Self {
418 const enc_ctx = AESEncryptCtx(AES).init(key);
419 return initFromEnc(enc_ctx);
420 }
421
422 /// Decrypt a single block.
423 pub fn decrypt(ctx: Self, dst: *[16]u8, src: *const [16]u8) void {
424 const inv_round_keys = ctx.key_schedule.round_keys;
425 var t = Block.fromBytes(src).xorBlocks(inv_round_keys[0]);
426 comptime var i = 1;
427 inline while (i < rounds) : (i += 1) {
428 t = t.decrypt(inv_round_keys[i]);
429 }
430 t = t.decryptLast(inv_round_keys[rounds]);
431 dst.* = t.toBytes();
432 }
433
434 /// Decrypt multiple blocks, possibly leveraging parallelization.
435 pub fn decryptWide(ctx: Self, comptime count: usize, dst: *[16 * count]u8, src: *const [16 * count]u8) void {
436 const inv_round_keys = ctx.key_schedule.round_keys;
437 var ts: [count]Block = undefined;
438 comptime var j = 0;
439 inline while (j < count) : (j += 1) {
440 ts[j] = Block.fromBytes(src[j * 16 .. j * 16 + 16][0..16]).xorBlocks(inv_round_keys[0]);
441 }
442 comptime var i = 1;
443 inline while (i < rounds) : (i += 1) {
444 ts = Block.parallel.decryptWide(count, ts, inv_round_keys[i]);
445 }
446 i = 1;
447 inline while (i < count) : (i += 1) {
448 ts = Block.parallel.decryptLastWide(count, ts, inv_round_keys[i]);
449 }
450 j = 0;
451 inline while (j < count) : (j += 1) {
452 dst[16 * j .. 16 * j + 16].* = ts[j].toBytes();
453 }
454 }
455 };
456}
457
458/// AES-128 with the standard key schedule.
459pub const AES128 = struct {
460 pub const key_bits: usize = 128;
461 pub const rounds = ((key_bits - 64) / 32 + 8);
462 pub const block = Block;
463
464 /// Create a new context for encryption.
465 pub fn initEnc(key: [key_bits / 8]u8) AESEncryptCtx(AES128) {
466 return AESEncryptCtx(AES128).init(key);
467 }
468
469 /// Create a new context for decryption.
470 pub fn initDec(key: [key_bits / 8]u8) AESDecryptCtx(AES128) {
471 return AESDecryptCtx(AES128).init(key);
472 }
473};
474
475/// AES-256 with the standard key schedule.
476pub const AES256 = struct {
477 pub const key_bits: usize = 256;
478 pub const rounds = ((key_bits - 64) / 32 + 8);
479 pub const block = Block;
480
481 /// Create a new context for encryption.
482 pub fn initEnc(key: [key_bits / 8]u8) AESEncryptCtx(AES256) {
483 return AESEncryptCtx(AES256).init(key);
484 }
485
486 /// Create a new context for decryption.
487 pub fn initDec(key: [key_bits / 8]u8) AESDecryptCtx(AES256) {
488 return AESDecryptCtx(AES256).init(key);
489 }
490};