authorgravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-01 01:34:23+02:00
committergravatar for 124872+jedisct1@users.noreply.github.comFrank Denis <124872+jedisct1@users.noreply.github.com> 2020-10-01 02:05:11+02:00
log97fd0974b9372f65c661908fb1ffc01d68255dae
tree9e4d7d96dd7a4bf8d6434a59d3822bf5ae79ca1b
parent8161de7fa44393c08cf96c9f9aa5379f98925472

ghash: add pclmul support on x86_64

1 files changed, 34 insertions(+), 20 deletions(-)

lib/std/crypto/ghash.zig+34-20
......@@ -94,7 +94,18 @@ pub const Ghash = struct {
9494 }
9595 }
9696
97 fn bmul(x: u64, y: u64) u64 {
97 inline fn clmul_pclmul(x: u64, y: u64) u64 {
98 const Vector = std.meta.Vector;
99 const product = asm (
100 \\ vpclmulqdq $0x00, %[x], %[y], %[out]
101 : [out] "=x" (-> Vector(2, u64))
102 : [x] "x" (@bitCast(Vector(2, u64), @as(u128, x))),
103 [y] "x" (@bitCast(Vector(2, u64), @as(u128, y)))
104 );
105 return product[0];
106 }
107
108 fn clmul_soft(x: u64, y: u64) u64 {
98109 const x0 = x & 0x1111111111111111;
99110 const x1 = x & 0x2222222222222222;
100111 const x2 = x & 0x4444444444444444;
......@@ -111,10 +122,13 @@ pub const Ghash = struct {
111122 z1 &= 0x2222222222222222;
112123 z2 &= 0x4444444444444444;
113124 z3 &= 0x8888888888888888;
114
115125 return z0 | z1 | z2 | z3;
116126 }
117127
128 const has_pclmul = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .pclmul);
129 const has_avx = comptime std.Target.x86.featureSetHas(std.Target.current.cpu.features, .avx);
130 const clmul = if (std.Target.current.cpu.arch == .x86_64 and has_pclmul and has_avx) clmul_pclmul else clmul_soft;
131
118132 fn blocks(st: *Ghash, msg: []const u8) void {
119133 assert(msg.len % 16 == 0); // GHASH blocks() expects full blocks
120134 var y1 = st.y1;
......@@ -134,12 +148,12 @@ pub const Ghash = struct {
134148 const y2 = y0 ^ y1;
135149 const y2r = y0r ^ y1r;
136150
137 var z0 = bmul(y0, st.hh0);
138 var z1 = bmul(y1, st.hh1);
139 var z2 = bmul(y2, st.hh2) ^ z0 ^ z1;
140 var z0h = bmul(y0r, st.hh0r);
141 var z1h = bmul(y1r, st.hh1r);
142 var z2h = bmul(y2r, st.hh2r) ^ z0h ^ z1h;
151 var z0 = clmul(y0, st.hh0);
152 var z1 = clmul(y1, st.hh1);
153 var z2 = clmul(y2, st.hh2) ^ z0 ^ z1;
154 var z0h = clmul(y0r, st.hh0r);
155 var z1h = clmul(y1r, st.hh1r);
156 var z2h = clmul(y2r, st.hh2r) ^ z0h ^ z1h;
143157
144158 // B1 * H unreduced
145159 const sy1 = mem.readIntBig(u64, msg[i..][16..24]);
......@@ -150,12 +164,12 @@ pub const Ghash = struct {
150164 const sy2 = sy0 ^ sy1;
151165 const sy2r = sy0r ^ sy1r;
152166
153 const sz0 = bmul(sy0, st.h0);
154 const sz1 = bmul(sy1, st.h1);
155 const sz2 = bmul(sy2, st.h2) ^ sz0 ^ sz1;
156 const sz0h = bmul(sy0r, st.h0r);
157 const sz1h = bmul(sy1r, st.h1r);
158 const sz2h = bmul(sy2r, st.h2r) ^ sz0h ^ sz1h;
167 const sz0 = clmul(sy0, st.h0);
168 const sz1 = clmul(sy1, st.h1);
169 const sz2 = clmul(sy2, st.h2) ^ sz0 ^ sz1;
170 const sz0h = clmul(sy0r, st.h0r);
171 const sz1h = clmul(sy1r, st.h1r);
172 const sz2h = clmul(sy2r, st.h2r) ^ sz0h ^ sz1h;
159173
160174 // ((B0 * H^2) + B1 * H) (mod M)
161175 z0 ^= sz0;
......@@ -195,12 +209,12 @@ pub const Ghash = struct {
195209 const y2 = y0 ^ y1;
196210 const y2r = y0r ^ y1r;
197211
198 const z0 = bmul(y0, st.h0);
199 const z1 = bmul(y1, st.h1);
200 var z2 = bmul(y2, st.h2) ^ z0 ^ z1;
201 var z0h = bmul(y0r, st.h0r);
202 var z1h = bmul(y1r, st.h1r);
203 var z2h = bmul(y2r, st.h2r) ^ z0h ^ z1h;
212 const z0 = clmul(y0, st.h0);
213 const z1 = clmul(y1, st.h1);
214 var z2 = clmul(y2, st.h2) ^ z0 ^ z1;
215 var z0h = clmul(y0r, st.h0r);
216 var z1h = clmul(y1r, st.h1r);
217 var z2h = clmul(y2r, st.h2r) ^ z0h ^ z1h;
204218 z0h = @bitReverse(u64, z0h) >> 1;
205219 z1h = @bitReverse(u64, z1h) >> 1;
206220 z2h = @bitReverse(u64, z2h) >> 1;