| author |  Jacob Young <jacobly@ziglang.org> 2024-02-16 07:59:16+01:00 |
| committer | Jacob Young <jacobly@ziglang.org> 2024-02-25 11:22:10+01:00 |
| log | 513c4c145ee163bb178c615a216534ac1f7e9e91 |
| tree | 6cd1ae03642e2f260675f44c9669173a48aebb36 |
| parent | 2fcb2f597549edd0b1241cebf98c11efe2f25884 |
x86_64: fix avx2 `@truncacte`
4 files changed, 39 insertions(+), 14 deletions(-)
src
src/arch/x86_64/CodeGen.zig+28-10
| ... | @@ -3274,8 +3274,8 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3274,8 +3274,8 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { |
| 3274 | try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); | 3274 | try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); |
| 3275 | break :dst dst_mcv; | 3275 | break :dst dst_mcv; |
| 3276 | } else dst: { | 3276 | } else dst: { |
| 3277 | const dst_mcv = try self.allocRegOrMem(inst, true); | 3277 | const dst_mcv = try self.allocRegOrMemAdvanced(src_ty, inst, true); |
| 3278 | try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); | 3278 | try self.genCopy(src_ty, dst_mcv, src_mcv, .{}); |
| 3279 | break :dst dst_mcv; | 3279 | break :dst dst_mcv; |
| 3280 | }; | 3280 | }; |
| 3281 | 3281 | ||
| ... | @@ -3333,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -3333,22 +3333,40 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { |
| 3333 | else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) }, | 3333 | else => .{ .register = try self.copyToTmpRegister(Type.usize, splat_mcv.address()) }, |
| 3334 | }; | 3334 | }; |
| 3335 | 3335 | ||
| 3336 | const dst_reg = registerAlias(dst_mcv.getReg().?, src_abi_size); | 3336 | const dst_reg = dst_mcv.getReg().?; |
| 3337 | const dst_alias = registerAlias(dst_reg, src_abi_size); | ||
| 3337 | if (self.hasFeature(.avx)) { | 3338 | if (self.hasFeature(.avx)) { |
| 3338 | try self.asmRegisterRegisterMemory( | 3339 | try self.asmRegisterRegisterMemory( |
| 3339 | .{ .vp_, .@"and" }, | 3340 | .{ .vp_, .@"and" }, |
| 3340 | dst_reg, | 3341 | dst_alias, |
| 3341 | dst_reg, | 3342 | dst_alias, |
| 3342 | try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)), | 3343 | try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)), |
| 3343 | ); | 3344 | ); |
| 3344 | try self.asmRegisterRegisterRegister(mir_tag, dst_reg, dst_reg, dst_reg); | 3345 | if (src_abi_size > 16) { |
| 3346 | const temp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.sse); | ||
| 3347 | const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); | ||
| 3348 | defer self.register_manager.unlockReg(temp_lock); | ||
| 3349 | |||
| 3350 | try self.asmRegisterRegisterImmediate( | ||
| 3351 | .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract }, | ||
| 3352 | registerAlias(temp_reg, dst_abi_size), | ||
| 3353 | dst_alias, | ||
| 3354 | Immediate.u(1), | ||
| 3355 | ); | ||
| 3356 | try self.asmRegisterRegisterRegister( | ||
| 3357 | mir_tag, | ||
| 3358 | registerAlias(dst_reg, dst_abi_size), | ||
| 3359 | registerAlias(dst_reg, dst_abi_size), | ||
| 3360 | registerAlias(temp_reg, dst_abi_size), | ||
| 3361 | ); | ||
| 3362 | } else try self.asmRegisterRegisterRegister(mir_tag, dst_alias, dst_alias, dst_alias); | ||
| 3345 | } else { | 3363 | } else { |
| 3346 | try self.asmRegisterMemory( | 3364 | try self.asmRegisterMemory( |
| 3347 | .{ .p_, .@"and" }, | 3365 | .{ .p_, .@"and" }, |
| 3348 | dst_reg, | 3366 | dst_alias, |
| 3349 | try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)), | 3367 | try splat_addr_mcv.deref().mem(self, Memory.Size.fromSize(splat_abi_size)), |
| 3350 | ); | 3368 | ); |
| 3351 | try self.asmRegisterRegister(mir_tag, dst_reg, dst_reg); | 3369 | try self.asmRegisterRegister(mir_tag, dst_alias, dst_alias); |
| 3352 | } | 3370 | } |
| 3353 | break :result dst_mcv; | 3371 | break :result dst_mcv; |
| 3354 | } | 3372 | } |
| ... | @@ -16404,7 +16422,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -16404,7 +16422,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { |
| 16404 | }, | 16422 | }, |
| 16405 | 65...128 => switch (vector_len) { | 16423 | 65...128 => switch (vector_len) { |
| 16406 | else => null, | 16424 | else => null, |
| 16407 | 1...2 => .{ .vp_i128, .broadcast }, | 16425 | 1...2 => .{ .v_i128, .broadcast }, |
| 16408 | }, | 16426 | }, |
| 16409 | }) orelse break :avx2; | 16427 | }) orelse break :avx2; |
| 16410 | 16428 | ||
| ... | @@ -16418,7 +16436,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { | ... | @@ -16418,7 +16436,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { |
| 16418 | registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))), | 16436 | registerAlias(dst_reg, @intCast(vector_ty.abiSize(mod))), |
| 16419 | try src_mcv.mem(self, self.memSize(scalar_ty)), | 16437 | try src_mcv.mem(self, self.memSize(scalar_ty)), |
| 16420 | ) else { | 16438 | ) else { |
| 16421 | if (mir_tag[0] == .vp_i128) break :avx2; | 16439 | if (mir_tag[0] == .v_i128) break :avx2; |
| 16422 | try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); | 16440 | try self.genSetReg(dst_reg, scalar_ty, src_mcv, .{}); |
| 16423 | try self.asmRegisterRegister( | 16441 | try self.asmRegisterRegister( |
| 16424 | mir_tag, | 16442 | mir_tag, |
src/arch/x86_64/Encoding.zig+2-1
| ... | @@ -415,7 +415,8 @@ pub const Mnemonic = enum { | ... | @@ -415,7 +415,8 @@ pub const Mnemonic = enum { |
| 415 | vfmadd132sd, vfmadd213sd, vfmadd231sd, | 415 | vfmadd132sd, vfmadd213sd, vfmadd231sd, |
| 416 | vfmadd132ss, vfmadd213ss, vfmadd231ss, | 416 | vfmadd132ss, vfmadd213ss, vfmadd231ss, |
| 417 | // AVX2 | 417 | // AVX2 |
| 418 | vpbroadcastb, vpbroadcastd, vpbroadcasti128, vpbroadcastq, vpbroadcastw, | 418 | vbroadcasti128, vpbroadcastb, vpbroadcastd, vpbroadcastq, vpbroadcastw, |
| 419 | vextracti128, vinserti128, | ||
| 419 | // zig fmt: on | 420 | // zig fmt: on |
| 420 | }; | 421 | }; |
| 421 | 422 |
src/arch/x86_64/Mir.zig+4-2
| ... | @@ -230,6 +230,8 @@ pub const Inst = struct { | ... | @@ -230,6 +230,8 @@ pub const Inst = struct { |
| 230 | v_d, | 230 | v_d, |
| 231 | /// VEX-Encoded ___ QuadWord | 231 | /// VEX-Encoded ___ QuadWord |
| 232 | v_q, | 232 | v_q, |
| 233 | /// VEX-Encoded ___ Integer Data | ||
| 234 | v_i128, | ||
| 233 | /// VEX-Encoded Packed ___ | 235 | /// VEX-Encoded Packed ___ |
| 234 | vp_, | 236 | vp_, |
| 235 | /// VEX-Encoded Packed ___ Byte | 237 | /// VEX-Encoded Packed ___ Byte |
| ... | @@ -242,8 +244,6 @@ pub const Inst = struct { | ... | @@ -242,8 +244,6 @@ pub const Inst = struct { |
| 242 | vp_q, | 244 | vp_q, |
| 243 | /// VEX-Encoded Packed ___ Double Quadword | 245 | /// VEX-Encoded Packed ___ Double Quadword |
| 244 | vp_dq, | 246 | vp_dq, |
| 245 | /// VEX-Encoded Packed ___ Integer Data | ||
| 246 | vp_i128, | ||
| 247 | /// VEX-Encoded ___ Scalar Single-Precision Values | 247 | /// VEX-Encoded ___ Scalar Single-Precision Values |
| 248 | v_ss, | 248 | v_ss, |
| 249 | /// VEX-Encoded ___ Packed Single-Precision Values | 249 | /// VEX-Encoded ___ Packed Single-Precision Values |
| ... | @@ -654,6 +654,7 @@ pub const Inst = struct { | ... | @@ -654,6 +654,7 @@ pub const Inst = struct { |
| 654 | /// Variable blend scalar double-precision floating-point values | 654 | /// Variable blend scalar double-precision floating-point values |
| 655 | blendv, | 655 | blendv, |
| 656 | /// Extract packed floating-point values | 656 | /// Extract packed floating-point values |
| 657 | /// Extract packed integer values | ||
| 657 | extract, | 658 | extract, |
| 658 | /// Insert scalar single-precision floating-point value | 659 | /// Insert scalar single-precision floating-point value |
| 659 | /// Insert packed floating-point values | 660 | /// Insert packed floating-point values |
| ... | @@ -696,6 +697,7 @@ pub const Inst = struct { | ... | @@ -696,6 +697,7 @@ pub const Inst = struct { |
| 696 | sha256rnds2, | 697 | sha256rnds2, |
| 697 | 698 | ||
| 698 | /// Load with broadcast floating-point data | 699 | /// Load with broadcast floating-point data |
| 700 | /// Load integer and broadcast | ||
| 699 | broadcast, | 701 | broadcast, |
| 700 | 702 | ||
| 701 | /// Convert 16-bit floating-point values to single-precision floating-point values | 703 | /// Convert 16-bit floating-point values to single-precision floating-point values |
src/arch/x86_64/encodings.zig+5-1
| ... | @@ -1769,6 +1769,10 @@ pub const table = [_]Entry{ | ... | @@ -1769,6 +1769,10 @@ pub const table = [_]Entry{ |
| 1769 | .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 }, | 1769 | .{ .vbroadcastss, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x18 }, 0, .vex_256_w0, .avx2 }, |
| 1770 | .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 }, | 1770 | .{ .vbroadcastsd, .rm, &.{ .ymm, .xmm }, &.{ 0x66, 0x0f, 0x38, 0x19 }, 0, .vex_256_w0, .avx2 }, |
| 1771 | 1771 | ||
| 1772 | .{ .vextracti128, .mri, &.{ .xmm_m128, .ymm, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x39 }, 0, .vex_256_w0, .avx2 }, | ||
| 1773 | |||
| 1774 | .{ .vinserti128, .rvmi, &.{ .ymm, .ymm, .xmm_m128, .imm8 }, &.{ 0x66, 0x0f, 0x3a, 0x38 }, 0, .vex_256_w0, .avx2 }, | ||
| 1775 | |||
| 1772 | .{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 }, | 1776 | .{ .vpabsb, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1c }, 0, .vex_256_wig, .avx2 }, |
| 1773 | .{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 }, | 1777 | .{ .vpabsd, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1e }, 0, .vex_256_wig, .avx2 }, |
| 1774 | .{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 }, | 1778 | .{ .vpabsw, .rm, &.{ .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x38, 0x1d }, 0, .vex_256_wig, .avx2 }, |
| ... | @@ -1809,7 +1813,7 @@ pub const table = [_]Entry{ | ... | @@ -1809,7 +1813,7 @@ pub const table = [_]Entry{ |
| 1809 | .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 }, | 1813 | .{ .vpbroadcastd, .rm, &.{ .ymm, .xmm_m32 }, &.{ 0x66, 0x0f, 0x38, 0x58 }, 0, .vex_256_w0, .avx2 }, |
| 1810 | .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 }, | 1814 | .{ .vpbroadcastq, .rm, &.{ .xmm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_128_w0, .avx2 }, |
| 1811 | .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 }, | 1815 | .{ .vpbroadcastq, .rm, &.{ .ymm, .xmm_m64 }, &.{ 0x66, 0x0f, 0x38, 0x59 }, 0, .vex_256_w0, .avx2 }, |
| 1812 | .{ .vpbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 }, | 1816 | .{ .vbroadcasti128, .rm, &.{ .ymm, .m128 }, &.{ 0x66, 0x0f, 0x38, 0x5a }, 0, .vex_256_w0, .avx2 }, |
| 1813 | 1817 | ||
| 1814 | .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 }, | 1818 | .{ .vpcmpeqb, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x74 }, 0, .vex_256_wig, .avx2 }, |
| 1815 | .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 }, | 1819 | .{ .vpcmpeqw, .rvm, &.{ .ymm, .ymm, .ymm_m256 }, &.{ 0x66, 0x0f, 0x75 }, 0, .vex_256_wig, .avx2 }, |