authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2022-11-20 13:07:40+01:00
committergravatar for andrew@ziglang.orgAndrew Kelley <andrew@ziglang.org> 2022-11-20 18:13:19-05:00
logc45c6cd492e9a5f351fa9290c7f7385d743b3818
tree9241dad5efc01f386b448ed102a0a44123b1235b
parent79bba5a9e639ddfb89af95ccf22501d974bd7fe3

Add the POLYVAL universal hash function

POLYVAL is GHASH's little brother, required by the AES-GCM-SIV construction. It's defined in RFC8452. The irreducible polynomial is a mirror of GHASH's (which doesn't change anything in our implementation that didn't reverse the raw bits to start with). But most importantly, POLYVAL encodes byte strings as little-endian instead of big-endian, which makes it a little bit faster on the vast majority of modern CPUs. So, both share the same code, just with comptime magic to use the correct endianness and only double the key for GHASH.

4 files changed, 447 insertions(+), 416 deletions(-)

lib/std/crypto.zig+2-1
...@@ -87,7 +87,8 @@ pub const kdf = struct {...@@ -87,7 +87,8 @@ pub const kdf = struct {
8787
88/// MAC functions requiring single-use secret keys.88/// MAC functions requiring single-use secret keys.
89pub const onetimeauth = struct {89pub const onetimeauth = struct {
90 pub const Ghash = @import("crypto/ghash.zig").Ghash;90 pub const Ghash = @import("crypto/ghash_polyval.zig").Ghash;
91 pub const Polyval = @import("crypto/ghash_polyval.zig").Polyval;
91 pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305;92 pub const Poly1305 = @import("crypto/poly1305.zig").Poly1305;
92};93};
9394
lib/std/crypto/benchmark.zig+1
...@@ -54,6 +54,7 @@ pub fn benchmarkHash(comptime Hash: anytype, comptime bytes: comptime_int) !u64...@@ -54,6 +54,7 @@ pub fn benchmarkHash(comptime Hash: anytype, comptime bytes: comptime_int) !u64
5454
55const macs = [_]Crypto{55const macs = [_]Crypto{
56 Crypto{ .ty = crypto.onetimeauth.Ghash, .name = "ghash" },56 Crypto{ .ty = crypto.onetimeauth.Ghash, .name = "ghash" },
57 Crypto{ .ty = crypto.onetimeauth.Polyval, .name = "polyval" },
57 Crypto{ .ty = crypto.onetimeauth.Poly1305, .name = "poly1305" },58 Crypto{ .ty = crypto.onetimeauth.Poly1305, .name = "poly1305" },
58 Crypto{ .ty = crypto.auth.hmac.HmacMd5, .name = "hmac-md5" },59 Crypto{ .ty = crypto.auth.hmac.HmacMd5, .name = "hmac-md5" },
59 Crypto{ .ty = crypto.auth.hmac.HmacSha1, .name = "hmac-sha1" },60 Crypto{ .ty = crypto.auth.hmac.HmacSha1, .name = "hmac-sha1" },
lib/std/crypto/ghash.zig deleted-415
...@@ -1,415 +0,0 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const assert = std.debug.assert;
4const math = std.math;
5const mem = std.mem;
6const utils = std.crypto.utils;
7
8const Precomp = u128;
9
10/// GHASH is a universal hash function that features multiplication
11/// by a fixed parameter within a Galois field.
12///
13/// It is not a general purpose hash function - The key must be secret, unpredictable and never reused.
14///
15/// GHASH is typically used to compute the authentication tag in the AES-GCM construction.
16pub const Ghash = struct {
17 pub const block_length: usize = 16;
18 pub const mac_length = 16;
19 pub const key_length = 16;
20
21 const pc_count = if (builtin.mode != .ReleaseSmall) 16 else 2;
22 const agg_4_treshold = 22;
23 const agg_8_treshold = 84;
24 const agg_16_treshold = 328;
25
26 // Before the Haswell architecture, the carryless multiplication instruction was
27 // extremely slow. Even with 128-bit operands, using Karatsuba multiplication was
28 // thus faster than a schoolbook multiplication.
29 // This is no longer the case -- Modern CPUs, including ARM-based ones, have a fast
30 // carryless multiplication instruction; using 4 multiplications is now faster than
31 // 3 multiplications with extra shifts and additions.
32 const mul_algorithm = if (builtin.cpu.arch == .x86) .karatsuba else .schoolbook;
33
34 hx: [pc_count]Precomp,
35 acc: u128 = 0,
36
37 leftover: usize = 0,
38 buf: [block_length]u8 align(16) = undefined,
39
40 /// Initialize the GHASH state with a key, and a minimum number of block count.
41 pub fn initForBlockCount(key: *const [key_length]u8, block_count: usize) Ghash {
42 const h0 = mem.readIntBig(u128, key[0..16]);
43
44 // We keep the values encoded as in GCM, not Polyval, i.e. without reversing the bits.
45 // This is fine, but the reversed result would be shifted by 1 bit. So, we shift h
46 // to compensate.
47 const carry = ((@as(u128, 0xc2) << 120) | 1) & (@as(u128, 0) -% (h0 >> 127));
48 const h = (h0 << 1) ^ carry;
49
50 var hx: [pc_count]Precomp = undefined;
51 hx[0] = h;
52 hx[1] = gcmReduce(clsq128(hx[0])); // h^2
53
54 if (builtin.mode != .ReleaseSmall) {
55 hx[2] = gcmReduce(clmul128(hx[1], h)); // h^3
56 hx[3] = gcmReduce(clsq128(hx[1])); // h^4 = h^2^2
57 if (block_count >= agg_8_treshold) {
58 hx[4] = gcmReduce(clmul128(hx[3], h)); // h^5
59 hx[5] = gcmReduce(clsq128(hx[2])); // h^6 = h^3^2
60 hx[6] = gcmReduce(clmul128(hx[5], h)); // h^7
61 hx[7] = gcmReduce(clsq128(hx[3])); // h^8 = h^4^2
62 }
63 if (block_count >= agg_16_treshold) {
64 var i: usize = 8;
65 while (i < 16) : (i += 2) {
66 hx[i] = gcmReduce(clmul128(hx[i - 1], h));
67 hx[i + 1] = gcmReduce(clsq128(hx[i / 2]));
68 }
69 }
70 }
71 return Ghash{ .hx = hx };
72 }
73
74 /// Initialize the GHASH state with a key.
75 pub fn init(key: *const [key_length]u8) Ghash {
76 return Ghash.initForBlockCount(key, math.maxInt(usize));
77 }
78
79 const Selector = enum { lo, hi, hi_lo };
80
81 // Carryless multiplication of two 64-bit integers for x86_64.
82 inline fn clmulPclmul(x: u128, y: u128, comptime half: Selector) u128 {
83 switch (half) {
84 .hi => {
85 const product = asm (
86 \\ vpclmulqdq $0x11, %[x], %[y], %[out]
87 : [out] "=x" (-> @Vector(2, u64)),
88 : [x] "x" (@bitCast(@Vector(2, u64), x)),
89 [y] "x" (@bitCast(@Vector(2, u64), y)),
90 );
91 return @bitCast(u128, product);
92 },
93 .lo => {
94 const product = asm (
95 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
96 : [out] "=x" (-> @Vector(2, u64)),
97 : [x] "x" (@bitCast(@Vector(2, u64), x)),
98 [y] "x" (@bitCast(@Vector(2, u64), y)),
99 );
100 return @bitCast(u128, product);
101 },
102 .hi_lo => {
103 const product = asm (
104 \\ vpclmulqdq $0x10, %[x], %[y], %[out]
105 : [out] "=x" (-> @Vector(2, u64)),
106 : [x] "x" (@bitCast(@Vector(2, u64), x)),
107 [y] "x" (@bitCast(@Vector(2, u64), y)),
108 );
109 return @bitCast(u128, product);
110 },
111 }
112 }
113
114 // Carryless multiplication of two 64-bit integers for ARM crypto.
115 inline fn clmulPmull(x: u128, y: u128, comptime half: Selector) u128 {
116 switch (half) {
117 .hi => {
118 const product = asm (
119 \\ pmull2 %[out].1q, %[x].2d, %[y].2d
120 : [out] "=w" (-> @Vector(2, u64)),
121 : [x] "w" (@bitCast(@Vector(2, u64), x)),
122 [y] "w" (@bitCast(@Vector(2, u64), y)),
123 );
124 return @bitCast(u128, product);
125 },
126 .lo => {
127 const product = asm (
128 \\ pmull %[out].1q, %[x].1d, %[y].1d
129 : [out] "=w" (-> @Vector(2, u64)),
130 : [x] "w" (@bitCast(@Vector(2, u64), x)),
131 [y] "w" (@bitCast(@Vector(2, u64), y)),
132 );
133 return @bitCast(u128, product);
134 },
135 .hi_lo => {
136 const product = asm (
137 \\ pmull %[out].1q, %[x].1d, %[y].1d
138 : [out] "=w" (-> @Vector(2, u64)),
139 : [x] "w" (@bitCast(@Vector(2, u64), x >> 64)),
140 [y] "w" (@bitCast(@Vector(2, u64), y)),
141 );
142 return @bitCast(u128, product);
143 },
144 }
145 }
146
147 // Software carryless multiplication of two 64-bit integers.
148 fn clmulSoft(x_: u128, y_: u128, comptime half: Selector) u128 {
149 const x = @truncate(u64, if (half == .hi or half == .hi_lo) x_ >> 64 else x_);
150 const y = @truncate(u64, if (half == .hi) y_ >> 64 else y_);
151
152 const x0 = x & 0x1111111111111110;
153 const x1 = x & 0x2222222222222220;
154 const x2 = x & 0x4444444444444440;
155 const x3 = x & 0x8888888888888880;
156 const y0 = y & 0x1111111111111111;
157 const y1 = y & 0x2222222222222222;
158 const y2 = y & 0x4444444444444444;
159 const y3 = y & 0x8888888888888888;
160 const z0 = (x0 * @as(u128, y0)) ^ (x1 * @as(u128, y3)) ^ (x2 * @as(u128, y2)) ^ (x3 * @as(u128, y1));
161 const z1 = (x0 * @as(u128, y1)) ^ (x1 * @as(u128, y0)) ^ (x2 * @as(u128, y3)) ^ (x3 * @as(u128, y2));
162 const z2 = (x0 * @as(u128, y2)) ^ (x1 * @as(u128, y1)) ^ (x2 * @as(u128, y0)) ^ (x3 * @as(u128, y3));
163 const z3 = (x0 * @as(u128, y3)) ^ (x1 * @as(u128, y2)) ^ (x2 * @as(u128, y1)) ^ (x3 * @as(u128, y0));
164
165 const x0_mask = @as(u64, 0) -% (x & 1);
166 const x1_mask = @as(u64, 0) -% ((x >> 1) & 1);
167 const x2_mask = @as(u64, 0) -% ((x >> 2) & 1);
168 const x3_mask = @as(u64, 0) -% ((x >> 3) & 1);
169 const extra = (x0_mask & y) ^ (@as(u128, x1_mask & y) << 1) ^
170 (@as(u128, x2_mask & y) << 2) ^ (@as(u128, x3_mask & y) << 3);
171
172 return (z0 & 0x11111111111111111111111111111111) ^
173 (z1 & 0x22222222222222222222222222222222) ^
174 (z2 & 0x44444444444444444444444444444444) ^
175 (z3 & 0x88888888888888888888888888888888) ^ extra;
176 }
177
178 const I256 = struct {
179 hi: u128,
180 lo: u128,
181 mid: u128,
182 };
183
184 inline fn xor256(x: *I256, y: I256) void {
185 x.* = I256{
186 .hi = x.hi ^ y.hi,
187 .lo = x.lo ^ y.lo,
188 .mid = x.mid ^ y.mid,
189 };
190 }
191
192 // Square a 128-bit integer in GF(2^128).
193 fn clsq128(x: u128) I256 {
194 return .{
195 .hi = clmul(x, x, .hi),
196 .lo = clmul(x, x, .lo),
197 .mid = 0,
198 };
199 }
200
201 // Multiply two 128-bit integers in GF(2^128).
202 inline fn clmul128(x: u128, y: u128) I256 {
203 if (mul_algorithm == .karatsuba) {
204 const x_hi = @truncate(u64, x >> 64);
205 const y_hi = @truncate(u64, y >> 64);
206 const r_lo = clmul(x, y, .lo);
207 const r_hi = clmul(x, y, .hi);
208 const r_mid = clmul(x ^ x_hi, y ^ y_hi, .lo) ^ r_lo ^ r_hi;
209 return .{
210 .hi = r_hi,
211 .lo = r_lo,
212 .mid = r_mid,
213 };
214 } else {
215 return .{
216 .hi = clmul(x, y, .hi),
217 .lo = clmul(x, y, .lo),
218 .mid = clmul(x, y, .hi_lo) ^ clmul(y, x, .hi_lo),
219 };
220 }
221 }
222
223 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.
224 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:
225 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html
226 inline fn gcmReduce(x: I256) u128 {
227 const hi = x.hi ^ (x.mid >> 64);
228 const lo = x.lo ^ (x.mid << 64);
229 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);
230 const a = clmul(lo, p64, .lo);
231 const b = ((lo << 64) | (lo >> 64)) ^ a;
232 const c = clmul(b, p64, .lo);
233 const d = ((b << 64) | (b >> 64)) ^ c;
234 return d ^ hi;
235 }
236
237 const has_pclmul = std.Target.x86.featureSetHas(builtin.cpu.features, .pclmul);
238 const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
239 const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
240 const clmul = if (builtin.cpu.arch == .x86_64 and has_pclmul and has_avx) impl: {
241 break :impl clmulPclmul;
242 } else if (builtin.cpu.arch == .aarch64 and has_armaes) impl: {
243 break :impl clmulPmull;
244 } else impl: {
245 break :impl clmulSoft;
246 };
247
248 // Process 16 byte blocks.
249 fn blocks(st: *Ghash, msg: []const u8) void {
250 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
251 var acc = st.acc;
252
253 var i: usize = 0;
254
255 if (builtin.mode != .ReleaseSmall and msg.len >= agg_16_treshold * block_length) {
256 // 16-blocks aggregated reduction
257 while (i + 256 <= msg.len) : (i += 256) {
258 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[15 - 0]);
259 comptime var j = 1;
260 inline while (j < 16) : (j += 1) {
261 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[15 - j]));
262 }
263 acc = gcmReduce(u);
264 }
265 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_8_treshold * block_length) {
266 // 8-blocks aggregated reduction
267 while (i + 128 <= msg.len) : (i += 128) {
268 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[7 - 0]);
269 comptime var j = 1;
270 inline while (j < 8) : (j += 1) {
271 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[7 - j]));
272 }
273 acc = gcmReduce(u);
274 }
275 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_4_treshold * block_length) {
276 // 4-blocks aggregated reduction
277 while (i + 64 <= msg.len) : (i += 64) {
278 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[3 - 0]);
279 comptime var j = 1;
280 inline while (j < 4) : (j += 1) {
281 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[3 - j]));
282 }
283 acc = gcmReduce(u);
284 }
285 }
286 // 2-blocks aggregated reduction
287 while (i + 32 <= msg.len) : (i += 32) {
288 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[1 - 0]);
289 comptime var j = 1;
290 inline while (j < 2) : (j += 1) {
291 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[1 - j]));
292 }
293 acc = gcmReduce(u);
294 }
295 // remaining blocks
296 if (i < msg.len) {
297 const u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[0]);
298 acc = gcmReduce(u);
299 i += 16;
300 }
301 assert(i == msg.len);
302 st.acc = acc;
303 }
304
305 /// Absorb a message into the GHASH state.
306 pub fn update(st: *Ghash, m: []const u8) void {
307 var mb = m;
308
309 if (st.leftover > 0) {
310 const want = math.min(block_length - st.leftover, mb.len);
311 const mc = mb[0..want];
312 for (mc) |x, i| {
313 st.buf[st.leftover + i] = x;
314 }
315 mb = mb[want..];
316 st.leftover += want;
317 if (st.leftover < block_length) {
318 return;
319 }
320 st.blocks(&st.buf);
321 st.leftover = 0;
322 }
323 if (mb.len >= block_length) {
324 const want = mb.len & ~(block_length - 1);
325 st.blocks(mb[0..want]);
326 mb = mb[want..];
327 }
328 if (mb.len > 0) {
329 for (mb) |x, i| {
330 st.buf[st.leftover + i] = x;
331 }
332 st.leftover += mb.len;
333 }
334 }
335
336 /// Zero-pad to align the next input to the first byte of a block
337 pub fn pad(st: *Ghash) void {
338 if (st.leftover == 0) {
339 return;
340 }
341 var i = st.leftover;
342 while (i < block_length) : (i += 1) {
343 st.buf[i] = 0;
344 }
345 st.blocks(&st.buf);
346 st.leftover = 0;
347 }
348
349 /// Compute the GHASH of the entire input.
350 pub fn final(st: *Ghash, out: *[mac_length]u8) void {
351 st.pad();
352 mem.writeIntBig(u128, out[0..16], st.acc);
353
354 utils.secureZero(u8, @ptrCast([*]u8, st)[0..@sizeOf(Ghash)]);
355 }
356
357 /// Compute the GHASH of a message.
358 pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void {
359 var st = Ghash.init(key);
360 st.update(msg);
361 st.final(out);
362 }
363};
364
365const htest = @import("test.zig");
366
367test "ghash" {
368 const key = [_]u8{0x42} ** 16;
369 const m = [_]u8{0x69} ** 256;
370
371 var st = Ghash.init(&key);
372 st.update(&m);
373 var out: [16]u8 = undefined;
374 st.final(&out);
375 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);
376
377 st = Ghash.init(&key);
378 st.update(m[0..100]);
379 st.update(m[100..]);
380 st.final(&out);
381 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);
382}
383
384test "ghash2" {
385 var key: [16]u8 = undefined;
386 var i: usize = 0;
387 while (i < key.len) : (i += 1) {
388 key[i] = @intCast(u8, i * 15 + 1);
389 }
390 const tvs = [_]struct { len: usize, hash: [:0]const u8 }{
391 .{ .len = 5263, .hash = "b9395f37c131cd403a327ccf82ec016a" },
392 .{ .len = 1361, .hash = "8c24cb3664e9a36e32ddef0c8178ab33" },
393 .{ .len = 1344, .hash = "015d7243b52d62eee8be33a66a9658cc" },
394 .{ .len = 1000, .hash = "56e148799944193f351f2014ef9dec9d" },
395 .{ .len = 512, .hash = "ca4882ce40d37546185c57709d17d1ca" },
396 .{ .len = 128, .hash = "d36dc3aac16cfe21a75cd5562d598c1c" },
397 .{ .len = 111, .hash = "6e2bea99700fd19cf1694e7b56543320" },
398 .{ .len = 80, .hash = "aa28f4092a7cca155f3de279cf21aa17" },
399 .{ .len = 16, .hash = "9d7eb5ed121a52a4b0996e4ec9b98911" },
400 .{ .len = 1, .hash = "968a203e5c7a98b6d4f3112f4d6b89a7" },
401 .{ .len = 0, .hash = "00000000000000000000000000000000" },
402 };
403 inline for (tvs) |tv| {
404 var m: [tv.len]u8 = undefined;
405 i = 0;
406 while (i < m.len) : (i += 1) {
407 m[i] = @truncate(u8, i % 254 + 1);
408 }
409 var st = Ghash.init(&key);
410 st.update(&m);
411 var out: [16]u8 = undefined;
412 st.final(&out);
413 try htest.assertEqual(tv.hash, &out);
414 }
415}
lib/std/crypto/ghash_polyval.zig created+444
...@@ -0,0 +1,444 @@
1const std = @import("../std.zig");
2const builtin = @import("builtin");
3const assert = std.debug.assert;
4const math = std.math;
5const mem = std.mem;
6const utils = std.crypto.utils;
7
8const Precomp = u128;
9
10/// GHASH is a universal hash function that uses multiplication by a fixed
11/// parameter within a Galois field.
12///
13/// It is not a general purpose hash function - The key must be secret, unpredictable and never reused.
14///
15/// GHASH is typically used to compute the authentication tag in the AES-GCM construction.
16pub const Ghash = Hash(.Big, true);
17
18/// POLYVAL is a universal hash function that uses multiplication by a fixed
19/// parameter within a Galois field.
20///
21/// It is not a general purpose hash function - The key must be secret, unpredictable and never reused.
22///
23/// POLYVAL is typically used to compute the authentication tag in the AES-GCM-SIV construction.
24pub const Polyval = Hash(.Little, false);
25
26fn Hash(comptime endian: std.builtin.Endian, comptime shift_key: bool) type {
27 return struct {
28 const Self = @This();
29
30 pub const block_length: usize = 16;
31 pub const mac_length = 16;
32 pub const key_length = 16;
33
34 const pc_count = if (builtin.mode != .ReleaseSmall) 16 else 2;
35 const agg_4_treshold = 22;
36 const agg_8_treshold = 84;
37 const agg_16_treshold = 328;
38
39 // Before the Haswell architecture, the carryless multiplication instruction was
40 // extremely slow. Even with 128-bit operands, using Karatsuba multiplication was
41 // thus faster than a schoolbook multiplication.
42 // This is no longer the case -- Modern CPUs, including ARM-based ones, have a fast
43 // carryless multiplication instruction; using 4 multiplications is now faster than
44 // 3 multiplications with extra shifts and additions.
45 const mul_algorithm = if (builtin.cpu.arch == .x86) .karatsuba else .schoolbook;
46
47 hx: [pc_count]Precomp,
48 acc: u128 = 0,
49
50 leftover: usize = 0,
51 buf: [block_length]u8 align(16) = undefined,
52
53 /// Initialize the GHASH state with a key, and a minimum number of block count.
54 pub fn initForBlockCount(key: *const [key_length]u8, block_count: usize) Self {
55 var h = mem.readInt(u128, key[0..16], endian);
56 if (shift_key) {
57 // Shift the key by 1 bit to the left & reduce for GCM.
58 const carry = ((@as(u128, 0xc2) << 120) | 1) & (@as(u128, 0) -% (h >> 127));
59 h = (h << 1) ^ carry;
60 }
61 var hx: [pc_count]Precomp = undefined;
62 hx[0] = h;
63 hx[1] = reduce(clsq128(hx[0])); // h^2
64
65 if (builtin.mode != .ReleaseSmall) {
66 hx[2] = reduce(clmul128(hx[1], h)); // h^3
67 hx[3] = reduce(clsq128(hx[1])); // h^4 = h^2^2
68 if (block_count >= agg_8_treshold) {
69 hx[4] = reduce(clmul128(hx[3], h)); // h^5
70 hx[5] = reduce(clsq128(hx[2])); // h^6 = h^3^2
71 hx[6] = reduce(clmul128(hx[5], h)); // h^7
72 hx[7] = reduce(clsq128(hx[3])); // h^8 = h^4^2
73 }
74 if (block_count >= agg_16_treshold) {
75 var i: usize = 8;
76 while (i < 16) : (i += 2) {
77 hx[i] = reduce(clmul128(hx[i - 1], h));
78 hx[i + 1] = reduce(clsq128(hx[i / 2]));
79 }
80 }
81 }
82 return Self{ .hx = hx };
83 }
84
85 /// Initialize the GHASH state with a key.
86 pub fn init(key: *const [key_length]u8) Self {
87 return Self.initForBlockCount(key, math.maxInt(usize));
88 }
89
90 const Selector = enum { lo, hi, hi_lo };
91
92 // Carryless multiplication of two 64-bit integers for x86_64.
93 inline fn clmulPclmul(x: u128, y: u128, comptime half: Selector) u128 {
94 switch (half) {
95 .hi => {
96 const product = asm (
97 \\ vpclmulqdq $0x11, %[x], %[y], %[out]
98 : [out] "=x" (-> @Vector(2, u64)),
99 : [x] "x" (@bitCast(@Vector(2, u64), x)),
100 [y] "x" (@bitCast(@Vector(2, u64), y)),
101 );
102 return @bitCast(u128, product);
103 },
104 .lo => {
105 const product = asm (
106 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
107 : [out] "=x" (-> @Vector(2, u64)),
108 : [x] "x" (@bitCast(@Vector(2, u64), x)),
109 [y] "x" (@bitCast(@Vector(2, u64), y)),
110 );
111 return @bitCast(u128, product);
112 },
113 .hi_lo => {
114 const product = asm (
115 \\ vpclmulqdq $0x10, %[x], %[y], %[out]
116 : [out] "=x" (-> @Vector(2, u64)),
117 : [x] "x" (@bitCast(@Vector(2, u64), x)),
118 [y] "x" (@bitCast(@Vector(2, u64), y)),
119 );
120 return @bitCast(u128, product);
121 },
122 }
123 }
124
125 // Carryless multiplication of two 64-bit integers for ARM crypto.
126 inline fn clmulPmull(x: u128, y: u128, comptime half: Selector) u128 {
127 switch (half) {
128 .hi => {
129 const product = asm (
130 \\ pmull2 %[out].1q, %[x].2d, %[y].2d
131 : [out] "=w" (-> @Vector(2, u64)),
132 : [x] "w" (@bitCast(@Vector(2, u64), x)),
133 [y] "w" (@bitCast(@Vector(2, u64), y)),
134 );
135 return @bitCast(u128, product);
136 },
137 .lo => {
138 const product = asm (
139 \\ pmull %[out].1q, %[x].1d, %[y].1d
140 : [out] "=w" (-> @Vector(2, u64)),
141 : [x] "w" (@bitCast(@Vector(2, u64), x)),
142 [y] "w" (@bitCast(@Vector(2, u64), y)),
143 );
144 return @bitCast(u128, product);
145 },
146 .hi_lo => {
147 const product = asm (
148 \\ pmull %[out].1q, %[x].1d, %[y].1d
149 : [out] "=w" (-> @Vector(2, u64)),
150 : [x] "w" (@bitCast(@Vector(2, u64), x >> 64)),
151 [y] "w" (@bitCast(@Vector(2, u64), y)),
152 );
153 return @bitCast(u128, product);
154 },
155 }
156 }
157
158 // Software carryless multiplication of two 64-bit integers.
159 fn clmulSoft(x_: u128, y_: u128, comptime half: Selector) u128 {
160 const x = @truncate(u64, if (half == .hi or half == .hi_lo) x_ >> 64 else x_);
161 const y = @truncate(u64, if (half == .hi) y_ >> 64 else y_);
162
163 const x0 = x & 0x1111111111111110;
164 const x1 = x & 0x2222222222222220;
165 const x2 = x & 0x4444444444444440;
166 const x3 = x & 0x8888888888888880;
167 const y0 = y & 0x1111111111111111;
168 const y1 = y & 0x2222222222222222;
169 const y2 = y & 0x4444444444444444;
170 const y3 = y & 0x8888888888888888;
171 const z0 = (x0 * @as(u128, y0)) ^ (x1 * @as(u128, y3)) ^ (x2 * @as(u128, y2)) ^ (x3 * @as(u128, y1));
172 const z1 = (x0 * @as(u128, y1)) ^ (x1 * @as(u128, y0)) ^ (x2 * @as(u128, y3)) ^ (x3 * @as(u128, y2));
173 const z2 = (x0 * @as(u128, y2)) ^ (x1 * @as(u128, y1)) ^ (x2 * @as(u128, y0)) ^ (x3 * @as(u128, y3));
174 const z3 = (x0 * @as(u128, y3)) ^ (x1 * @as(u128, y2)) ^ (x2 * @as(u128, y1)) ^ (x3 * @as(u128, y0));
175
176 const x0_mask = @as(u64, 0) -% (x & 1);
177 const x1_mask = @as(u64, 0) -% ((x >> 1) & 1);
178 const x2_mask = @as(u64, 0) -% ((x >> 2) & 1);
179 const x3_mask = @as(u64, 0) -% ((x >> 3) & 1);
180 const extra = (x0_mask & y) ^ (@as(u128, x1_mask & y) << 1) ^
181 (@as(u128, x2_mask & y) << 2) ^ (@as(u128, x3_mask & y) << 3);
182
183 return (z0 & 0x11111111111111111111111111111111) ^
184 (z1 & 0x22222222222222222222222222222222) ^
185 (z2 & 0x44444444444444444444444444444444) ^
186 (z3 & 0x88888888888888888888888888888888) ^ extra;
187 }
188
189 const I256 = struct {
190 hi: u128,
191 lo: u128,
192 mid: u128,
193 };
194
195 inline fn xor256(x: *I256, y: I256) void {
196 x.* = I256{
197 .hi = x.hi ^ y.hi,
198 .lo = x.lo ^ y.lo,
199 .mid = x.mid ^ y.mid,
200 };
201 }
202
203 // Square a 128-bit integer in GF(2^128).
204 fn clsq128(x: u128) I256 {
205 return .{
206 .hi = clmul(x, x, .hi),
207 .lo = clmul(x, x, .lo),
208 .mid = 0,
209 };
210 }
211
212 // Multiply two 128-bit integers in GF(2^128).
213 inline fn clmul128(x: u128, y: u128) I256 {
214 if (mul_algorithm == .karatsuba) {
215 const x_hi = @truncate(u64, x >> 64);
216 const y_hi = @truncate(u64, y >> 64);
217 const r_lo = clmul(x, y, .lo);
218 const r_hi = clmul(x, y, .hi);
219 const r_mid = clmul(x ^ x_hi, y ^ y_hi, .lo) ^ r_lo ^ r_hi;
220 return .{
221 .hi = r_hi,
222 .lo = r_lo,
223 .mid = r_mid,
224 };
225 } else {
226 return .{
227 .hi = clmul(x, y, .hi),
228 .lo = clmul(x, y, .lo),
229 .mid = clmul(x, y, .hi_lo) ^ clmul(y, x, .hi_lo),
230 };
231 }
232 }
233
234 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.
235 // This is done using Shay Gueron's black magic demysticated here:
236 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html
237 inline fn reduce(x: I256) u128 {
238 const hi = x.hi ^ (x.mid >> 64);
239 const lo = x.lo ^ (x.mid << 64);
240 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);
241 const a = clmul(lo, p64, .lo);
242 const b = ((lo << 64) | (lo >> 64)) ^ a;
243 const c = clmul(b, p64, .lo);
244 const d = ((b << 64) | (b >> 64)) ^ c;
245 return d ^ hi;
246 }
247
248 const has_pclmul = std.Target.x86.featureSetHas(builtin.cpu.features, .pclmul);
249 const has_avx = std.Target.x86.featureSetHas(builtin.cpu.features, .avx);
250 const has_armaes = std.Target.aarch64.featureSetHas(builtin.cpu.features, .aes);
251 const clmul = if (builtin.cpu.arch == .x86_64 and has_pclmul and has_avx) impl: {
252 break :impl clmulPclmul;
253 } else if (builtin.cpu.arch == .aarch64 and has_armaes) impl: {
254 break :impl clmulPmull;
255 } else impl: {
256 break :impl clmulSoft;
257 };
258
259 // Process 16 byte blocks.
260 fn blocks(st: *Self, msg: []const u8) void {
261 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
262 var acc = st.acc;
263
264 var i: usize = 0;
265
266 if (builtin.mode != .ReleaseSmall and msg.len >= agg_16_treshold * block_length) {
267 // 16-blocks aggregated reduction
268 while (i + 256 <= msg.len) : (i += 256) {
269 var u = clmul128(acc ^ mem.readInt(u128, msg[i..][0..16], endian), st.hx[15 - 0]);
270 comptime var j = 1;
271 inline while (j < 16) : (j += 1) {
272 xor256(&u, clmul128(mem.readInt(u128, msg[i..][j * 16 ..][0..16], endian), st.hx[15 - j]));
273 }
274 acc = reduce(u);
275 }
276 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_8_treshold * block_length) {
277 // 8-blocks aggregated reduction
278 while (i + 128 <= msg.len) : (i += 128) {
279 var u = clmul128(acc ^ mem.readInt(u128, msg[i..][0..16], endian), st.hx[7 - 0]);
280 comptime var j = 1;
281 inline while (j < 8) : (j += 1) {
282 xor256(&u, clmul128(mem.readInt(u128, msg[i..][j * 16 ..][0..16], endian), st.hx[7 - j]));
283 }
284 acc = reduce(u);
285 }
286 } else if (builtin.mode != .ReleaseSmall and msg.len >= agg_4_treshold * block_length) {
287 // 4-blocks aggregated reduction
288 while (i + 64 <= msg.len) : (i += 64) {
289 var u = clmul128(acc ^ mem.readInt(u128, msg[i..][0..16], endian), st.hx[3 - 0]);
290 comptime var j = 1;
291 inline while (j < 4) : (j += 1) {
292 xor256(&u, clmul128(mem.readInt(u128, msg[i..][j * 16 ..][0..16], endian), st.hx[3 - j]));
293 }
294 acc = reduce(u);
295 }
296 }
297 // 2-blocks aggregated reduction
298 while (i + 32 <= msg.len) : (i += 32) {
299 var u = clmul128(acc ^ mem.readInt(u128, msg[i..][0..16], endian), st.hx[1 - 0]);
300 comptime var j = 1;
301 inline while (j < 2) : (j += 1) {
302 xor256(&u, clmul128(mem.readInt(u128, msg[i..][j * 16 ..][0..16], endian), st.hx[1 - j]));
303 }
304 acc = reduce(u);
305 }
306 // remaining blocks
307 if (i < msg.len) {
308 const u = clmul128(acc ^ mem.readInt(u128, msg[i..][0..16], endian), st.hx[0]);
309 acc = reduce(u);
310 i += 16;
311 }
312 assert(i == msg.len);
313 st.acc = acc;
314 }
315
316 /// Absorb a message into the GHASH state.
317 pub fn update(st: *Self, m: []const u8) void {
318 var mb = m;
319
320 if (st.leftover > 0) {
321 const want = math.min(block_length - st.leftover, mb.len);
322 const mc = mb[0..want];
323 for (mc) |x, i| {
324 st.buf[st.leftover + i] = x;
325 }
326 mb = mb[want..];
327 st.leftover += want;
328 if (st.leftover < block_length) {
329 return;
330 }
331 st.blocks(&st.buf);
332 st.leftover = 0;
333 }
334 if (mb.len >= block_length) {
335 const want = mb.len & ~(block_length - 1);
336 st.blocks(mb[0..want]);
337 mb = mb[want..];
338 }
339 if (mb.len > 0) {
340 for (mb) |x, i| {
341 st.buf[st.leftover + i] = x;
342 }
343 st.leftover += mb.len;
344 }
345 }
346
347 /// Zero-pad to align the next input to the first byte of a block
348 pub fn pad(st: *Self) void {
349 if (st.leftover == 0) {
350 return;
351 }
352 var i = st.leftover;
353 while (i < block_length) : (i += 1) {
354 st.buf[i] = 0;
355 }
356 st.blocks(&st.buf);
357 st.leftover = 0;
358 }
359
360 /// Compute the GHASH of the entire input.
361 pub fn final(st: *Self, out: *[mac_length]u8) void {
362 st.pad();
363 mem.writeInt(u128, out[0..16], st.acc, endian);
364
365 utils.secureZero(u8, @ptrCast([*]u8, st)[0..@sizeOf(Self)]);
366 }
367
368 /// Compute the GHASH of a message.
369 pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void {
370 var st = Self.init(key);
371 st.update(msg);
372 st.final(out);
373 }
374 };
375}
376
377const htest = @import("test.zig");
378
379test "ghash" {
380 const key = [_]u8{0x42} ** 16;
381 const m = [_]u8{0x69} ** 256;
382
383 var st = Ghash.init(&key);
384 st.update(&m);
385 var out: [16]u8 = undefined;
386 st.final(&out);
387 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);
388
389 st = Ghash.init(&key);
390 st.update(m[0..100]);
391 st.update(m[100..]);
392 st.final(&out);
393 try htest.assertEqual("889295fa746e8b174bf4ec80a65dea41", &out);
394}
395
396test "ghash2" {
397 var key: [16]u8 = undefined;
398 var i: usize = 0;
399 while (i < key.len) : (i += 1) {
400 key[i] = @intCast(u8, i * 15 + 1);
401 }
402 const tvs = [_]struct { len: usize, hash: [:0]const u8 }{
403 .{ .len = 5263, .hash = "b9395f37c131cd403a327ccf82ec016a" },
404 .{ .len = 1361, .hash = "8c24cb3664e9a36e32ddef0c8178ab33" },
405 .{ .len = 1344, .hash = "015d7243b52d62eee8be33a66a9658cc" },
406 .{ .len = 1000, .hash = "56e148799944193f351f2014ef9dec9d" },
407 .{ .len = 512, .hash = "ca4882ce40d37546185c57709d17d1ca" },
408 .{ .len = 128, .hash = "d36dc3aac16cfe21a75cd5562d598c1c" },
409 .{ .len = 111, .hash = "6e2bea99700fd19cf1694e7b56543320" },
410 .{ .len = 80, .hash = "aa28f4092a7cca155f3de279cf21aa17" },
411 .{ .len = 16, .hash = "9d7eb5ed121a52a4b0996e4ec9b98911" },
412 .{ .len = 1, .hash = "968a203e5c7a98b6d4f3112f4d6b89a7" },
413 .{ .len = 0, .hash = "00000000000000000000000000000000" },
414 };
415 inline for (tvs) |tv| {
416 var m: [tv.len]u8 = undefined;
417 i = 0;
418 while (i < m.len) : (i += 1) {
419 m[i] = @truncate(u8, i % 254 + 1);
420 }
421 var st = Ghash.init(&key);
422 st.update(&m);
423 var out: [16]u8 = undefined;
424 st.final(&out);
425 try htest.assertEqual(tv.hash, &out);
426 }
427}
428
429test "polyval" {
430 const key = [_]u8{0x42} ** 16;
431 const m = [_]u8{0x69} ** 256;
432
433 var st = Polyval.init(&key);
434 st.update(&m);
435 var out: [16]u8 = undefined;
436 st.final(&out);
437 try htest.assertEqual("0713c82b170eef25c8955ddf72c85ccb", &out);
438
439 st = Polyval.init(&key);
440 st.update(m[0..100]);
441 st.update(m[100..]);
442 st.final(&out);
443 try htest.assertEqual("0713c82b170eef25c8955ddf72c85ccb", &out);
444}