authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2022-11-17 13:07:07+01:00
committergravatar for noreply@github.comGitHub <noreply@github.com> 2022-11-17 13:07:07+01:00
log7cfeae1ce7aa9f1b3a219d032c43bc2e694ba63b
tree7818e427398bef4e3415a095db5bfe600fbd22fc
parent58d9004cea5f8aa73c76382cd21e1c88b1bc21e1
signaturebadge-question-mark Signed by PGP key 4AEE18F83AFDEB23

std.crypto.onetimeauth.ghash: faster GHASH on modern CPUs (#13566)

* std.crypto.onetimeauth.ghash: faster GHASH on modern CPUs Carryless multiplication was slow on older Intel CPUs, justifying the need for using Karatsuba multiplication. This is not the case any more; using 4 multiplications to multiply two 128-bit numbers is actually faster than 3 multiplications + shifts and additions. This is also true on aarch64. Keep using Karatsuba only when targeting x86 (granted, this is a bit of a brutal shortcut, we should really list all the CPU models that had a slow clmul instruction). Also remove useless agg_2 treshold and restore the ability to precompute only H and H^2 in ReleaseSmall. Finally, avoid using u256. Using 128-bit registers is actually faster. * Use a switch, add some comments

1 files changed, 124 insertions(+), 74 deletions(-)

lib/std/crypto/ghash.zig+124-74
...@@ -18,12 +18,19 @@ pub const Ghash = struct {...@@ -18,12 +18,19 @@ pub const Ghash = struct {
18 pub const mac_length = 16;18 pub const mac_length = 16;
19 pub const key_length = 16;19 pub const key_length = 16;
2020
21 const pc_count = if (builtin.mode != .ReleaseSmall) 16 else 4;21 const pc_count = if (builtin.mode != .ReleaseSmall) 16 else 2;
22 const agg_2_treshold = 5;
23 const agg_4_treshold = 22;22 const agg_4_treshold = 22;
24 const agg_8_treshold = 84;23 const agg_8_treshold = 84;
25 const agg_16_treshold = 328;24 const agg_16_treshold = 328;
2625
26 // Before the Haswell architecture, the carryless multiplication instruction was
27 // extremely slow. Even with 128-bit operands, using Karatsuba multiplication was
28 // thus faster than a schoolbook multiplication.
29 // This is no longer the case -- Modern CPUs, including ARM-based ones, have a fast
30 // carryless multiplication instruction; using 4 multiplications is now faster than
31 // 3 multiplications with extra shifts and additions.
32 const mul_algorithm = if (builtin.cpu.arch == .x86) .karatsuba else .schoolbook;
33
27 hx: [pc_count]Precomp,34 hx: [pc_count]Precomp,
28 acc: u128 = 0,35 acc: u128 = 0,
2936
...@@ -43,10 +50,10 @@ pub const Ghash = struct {...@@ -43,10 +50,10 @@ pub const Ghash = struct {
43 var hx: [pc_count]Precomp = undefined;50 var hx: [pc_count]Precomp = undefined;
44 hx[0] = h;51 hx[0] = h;
45 hx[1] = gcmReduce(clsq128(hx[0])); // h^252 hx[1] = gcmReduce(clsq128(hx[0])); // h^2
46 hx[2] = gcmReduce(clmul128(hx[1], h)); // h^3
47 hx[3] = gcmReduce(clsq128(hx[1])); // h^4 = h^2^2
4853
49 if (builtin.mode != .ReleaseSmall) {54 if (builtin.mode != .ReleaseSmall) {
55 hx[2] = gcmReduce(clmul128(hx[1], h)); // h^3
56 hx[3] = gcmReduce(clsq128(hx[1])); // h^4 = h^2^2
50 if (block_count >= agg_8_treshold) {57 if (block_count >= agg_8_treshold) {
51 hx[4] = gcmReduce(clmul128(hx[3], h)); // h^558 hx[4] = gcmReduce(clmul128(hx[3], h)); // h^5
52 hx[5] = gcmReduce(clsq128(hx[2])); // h^6 = h^3^259 hx[5] = gcmReduce(clsq128(hx[2])); // h^6 = h^3^2
...@@ -69,47 +76,71 @@ pub const Ghash = struct {...@@ -69,47 +76,71 @@ pub const Ghash = struct {
69 return Ghash.initForBlockCount(key, math.maxInt(usize));76 return Ghash.initForBlockCount(key, math.maxInt(usize));
70 }77 }
7178
72 const Selector = enum { lo, hi };79 const Selector = enum { lo, hi, hi_lo };
7380
74 // Carryless multiplication of two 64-bit integers for x86_64.81 // Carryless multiplication of two 64-bit integers for x86_64.
75 inline fn clmulPclmul(x: u128, y: u128, comptime half: Selector) u128 {82 inline fn clmulPclmul(x: u128, y: u128, comptime half: Selector) u128 {
76 if (half == .hi) {83 switch (half) {
77 const product = asm (84 .hi => {
78 \\ vpclmulqdq $0x11, %[x], %[y], %[out]85 const product = asm (
79 : [out] "=x" (-> @Vector(2, u64)),86 \\ vpclmulqdq $0x11, %[x], %[y], %[out]
80 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),87 : [out] "=x" (-> @Vector(2, u64)),
81 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),88 : [x] "x" (@bitCast(@Vector(2, u64), x)),
82 );89 [y] "x" (@bitCast(@Vector(2, u64), y)),
83 return @bitCast(u128, product);90 );
84 } else {91 return @bitCast(u128, product);
85 const product = asm (92 },
86 \\ vpclmulqdq $0x00, %[x], %[y], %[out]93 .lo => {
87 : [out] "=x" (-> @Vector(2, u64)),94 const product = asm (
88 : [x] "x" (@bitCast(@Vector(2, u64), @as(u128, x))),95 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
89 [y] "x" (@bitCast(@Vector(2, u64), @as(u128, y))),96 : [out] "=x" (-> @Vector(2, u64)),
90 );97 : [x] "x" (@bitCast(@Vector(2, u64), x)),
91 return @bitCast(u128, product);98 [y] "x" (@bitCast(@Vector(2, u64), y)),
99 );
100 return @bitCast(u128, product);
101 },
102 .hi_lo => {
103 const product = asm (
104 \\ vpclmulqdq $0x10, %[x], %[y], %[out]
105 : [out] "=x" (-> @Vector(2, u64)),
106 : [x] "x" (@bitCast(@Vector(2, u64), x)),
107 [y] "x" (@bitCast(@Vector(2, u64), y)),
108 );
109 return @bitCast(u128, product);
110 },
92 }111 }
93 }112 }
94113
95 // Carryless multiplication of two 64-bit integers for ARM crypto.114 // Carryless multiplication of two 64-bit integers for ARM crypto.
96 inline fn clmulPmull(x: u128, y: u128, comptime half: Selector) u128 {115 inline fn clmulPmull(x: u128, y: u128, comptime half: Selector) u128 {
97 if (half == .hi) {116 switch (half) {
98 const product = asm (117 .hi => {
99 \\ pmull2 %[out].1q, %[x].2d, %[y].2d118 const product = asm (
100 : [out] "=w" (-> @Vector(2, u64)),119 \\ pmull2 %[out].1q, %[x].2d, %[y].2d
101 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),120 : [out] "=w" (-> @Vector(2, u64)),
102 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),121 : [x] "w" (@bitCast(@Vector(2, u64), x)),
103 );122 [y] "w" (@bitCast(@Vector(2, u64), y)),
104 return @bitCast(u128, product);123 );
105 } else {124 return @bitCast(u128, product);
106 const product = asm (125 },
107 \\ pmull %[out].1q, %[x].1d, %[y].1d126 .lo => {
108 : [out] "=w" (-> @Vector(2, u64)),127 const product = asm (
109 : [x] "w" (@bitCast(@Vector(2, u64), @as(u128, x))),128 \\ pmull %[out].1q, %[x].1d, %[y].1d
110 [y] "w" (@bitCast(@Vector(2, u64), @as(u128, y))),129 : [out] "=w" (-> @Vector(2, u64)),
111 );130 : [x] "w" (@bitCast(@Vector(2, u64), x)),
112 return @bitCast(u128, product);131 [y] "w" (@bitCast(@Vector(2, u64), y)),
132 );
133 return @bitCast(u128, product);
134 },
135 .hi_lo => {
136 const product = asm (
137 \\ pmull %[out].1q, %[x].1d, %[y].1d
138 : [out] "=w" (-> @Vector(2, u64)),
139 : [x] "w" (@bitCast(@Vector(2, u64), x >> 64)),
140 [y] "w" (@bitCast(@Vector(2, u64), y)),
141 );
142 return @bitCast(u128, product);
143 },
113 }144 }
114 }145 }
115146
...@@ -144,38 +175,63 @@ pub const Ghash = struct {...@@ -144,38 +175,63 @@ pub const Ghash = struct {
144 (z3 & 0x88888888888888888888888888888888) ^ extra;175 (z3 & 0x88888888888888888888888888888888) ^ extra;
145 }176 }
146177
178 const I256 = struct {
179 hi: u128,
180 lo: u128,
181 mid: u128,
182 };
183
184 inline fn xor256(x: *I256, y: I256) void {
185 x.* = I256{
186 .hi = x.hi ^ y.hi,
187 .lo = x.lo ^ y.lo,
188 .mid = x.mid ^ y.mid,
189 };
190 }
191
147 // Square a 128-bit integer in GF(2^128).192 // Square a 128-bit integer in GF(2^128).
148 fn clsq128(x: u128) u256 {193 fn clsq128(x: u128) I256 {
149 const lo = @truncate(u64, x);194 return .{
150 const hi = @truncate(u64, x >> 64);195 .hi = clmul(x, x, .hi),
151 const mid = lo ^ hi;196 .lo = clmul(x, x, .lo),
152 const r_lo = clmul(x, x, .lo);197 .mid = 0,
153 const r_hi = clmul(x, x, .hi);198 };
154 const r_mid = clmul(mid, mid, .lo) ^ r_lo ^ r_hi;
155 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;
156 }199 }
157200
158 // Multiply two 128-bit integers in GF(2^128).201 // Multiply two 128-bit integers in GF(2^128).
159 inline fn clmul128(x: u128, y: u128) u256 {202 inline fn clmul128(x: u128, y: u128) I256 {
160 const x_hi = @truncate(u64, x >> 64);203 if (mul_algorithm == .karatsuba) {
161 const y_hi = @truncate(u64, y >> 64);204 const x_hi = @truncate(u64, x >> 64);
162 const r_lo = clmul(x, y, .lo);205 const y_hi = @truncate(u64, y >> 64);
163 const r_hi = clmul(x, y, .hi);206 const r_lo = clmul(x, y, .lo);
164 const r_mid = clmul(x ^ x_hi, y ^ y_hi, .lo) ^ r_lo ^ r_hi;207 const r_hi = clmul(x, y, .hi);
165 return (@as(u256, r_hi) << 128) ^ (@as(u256, r_mid) << 64) ^ r_lo;208 const r_mid = clmul(x ^ x_hi, y ^ y_hi, .lo) ^ r_lo ^ r_hi;
209 return .{
210 .hi = r_hi,
211 .lo = r_lo,
212 .mid = r_mid,
213 };
214 } else {
215 return .{
216 .hi = clmul(x, y, .hi),
217 .lo = clmul(x, y, .lo),
218 .mid = clmul(x, y, .hi_lo) ^ clmul(y, x, .hi_lo),
219 };
220 }
166 }221 }
167222
168 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.223 // Reduce a 256-bit representative of a polynomial modulo the irreducible polynomial x^128 + x^127 + x^126 + x^121 + 1.
169 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:224 // This is done *without reversing the bits*, using Shay Gueron's black magic demysticated here:
170 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html225 // https://blog.quarkslab.com/reversing-a-finite-field-multiplication-optimization.html
171 inline fn gcmReduce(x: u256) u128 {226 inline fn gcmReduce(x: I256) u128 {
227 const hi = x.hi ^ (x.mid >> 64);
228 const lo = x.lo ^ (x.mid << 64);
172 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);229 const p64 = (((1 << 121) | (1 << 126) | (1 << 127)) >> 64);
173 const lo = @truncate(u128, x);
174 const a = clmul(lo, p64, .lo);230 const a = clmul(lo, p64, .lo);
175 const b = ((lo << 64) | (lo >> 64)) ^ a;231 const b = ((lo << 64) | (lo >> 64)) ^ a;
176 const c = clmul(b, p64, .lo);232 const c = clmul(b, p64, .lo);
177 const d = ((b << 64) | (b >> 64)) ^ c;233 const d = ((b << 64) | (b >> 64)) ^ c;
178 return d ^ @truncate(u128, x >> 128);234 return d ^ hi;
179 }235 }
180236
181 const has_pclmul = std.Target.x86.featureSetHas(builtin.cpu.features, .pclmul);237 const has_pclmul = std.Target.x86.featureSetHas(builtin.cpu.features, .pclmul);
...@@ -202,7 +258,7 @@ pub const Ghash = struct {...@@ -202,7 +258,7 @@ pub const Ghash = struct {
202 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[15 - 0]);258 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[15 - 0]);
203 comptime var j = 1;259 comptime var j = 1;
204 inline while (j < 16) : (j += 1) {260 inline while (j < 16) : (j += 1) {
205 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[15 - j]);261 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[15 - j]));
206 }262 }
207 acc = gcmReduce(u);263 acc = gcmReduce(u);
208 }264 }
...@@ -212,7 +268,7 @@ pub const Ghash = struct {...@@ -212,7 +268,7 @@ pub const Ghash = struct {
212 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[7 - 0]);268 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[7 - 0]);
213 comptime var j = 1;269 comptime var j = 1;
214 inline while (j < 8) : (j += 1) {270 inline while (j < 8) : (j += 1) {
215 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[7 - j]);271 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[7 - j]));
216 }272 }
217 acc = gcmReduce(u);273 acc = gcmReduce(u);
218 }274 }
...@@ -222,31 +278,25 @@ pub const Ghash = struct {...@@ -222,31 +278,25 @@ pub const Ghash = struct {
222 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[3 - 0]);278 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[3 - 0]);
223 comptime var j = 1;279 comptime var j = 1;
224 inline while (j < 4) : (j += 1) {280 inline while (j < 4) : (j += 1) {
225 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[3 - j]);281 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[3 - j]));
226 }282 }
227 acc = gcmReduce(u);283 acc = gcmReduce(u);
228 }284 }
229 } else if (msg.len >= agg_2_treshold * block_length) {285 }
230 // 2-blocks aggregated reduction286 // 2-blocks aggregated reduction
231 while (i + 32 <= msg.len) : (i += 32) {287 while (i + 32 <= msg.len) : (i += 32) {
232 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[1 - 0]);288 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[1 - 0]);
233 comptime var j = 1;289 comptime var j = 1;
234 inline while (j < 2) : (j += 1) {290 inline while (j < 2) : (j += 1) {
235 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[1 - j]);291 xor256(&u, clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[1 - j]));
236 }
237 acc = gcmReduce(u);
238 }292 }
293 acc = gcmReduce(u);
239 }294 }
240 // remaining blocks295 // remaining blocks
241 if (i < msg.len) {296 if (i < msg.len) {
242 const n = (msg.len - i) / 16;297 const u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[0]);
243 var u = clmul128(acc ^ mem.readIntBig(u128, msg[i..][0..16]), st.hx[n - 1 - 0]);
244 var j: usize = 1;
245 while (j < n) : (j += 1) {
246 u ^= clmul128(mem.readIntBig(u128, msg[i..][j * 16 ..][0..16]), st.hx[n - 1 - j]);
247 }
248 i += n * 16;
249 acc = gcmReduce(u);298 acc = gcmReduce(u);
299 i += 16;
250 }300 }
251 assert(i == msg.len);301 assert(i == msg.len);
252 st.acc = acc;302 st.acc = acc;