authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-12 00:53:14+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-12 05:25:07+01:00
loge27db373ec276f7839fac0a024b92a0945b5f905
tree5a0e337c41ed5dbcf8729356ce003d2d309ea5b0
parentd89472787353bc69f6a13c80c6f576ee3471c1a8

x86_64: implement `@clz` and `@ctz` of big integers

3 files changed, 141 insertions(+), 67 deletions(-)

lib/std/math/big/int_test.zig-2
...@@ -918,7 +918,6 @@ test "big.int mul multi-single" {...@@ -918,7 +918,6 @@ test "big.int mul multi-single" {
918918
919test "big.int mul multi-multi" {919test "big.int mul multi-multi" {
920 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;920 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
921 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
922921
923 var op1: u256 = 0x998888efefefefefefefef;922 var op1: u256 = 0x998888efefefefefefefef;
924 var op2: u256 = 0x333000abababababababab;923 var op2: u256 = 0x333000abababababababab;
...@@ -1042,7 +1041,6 @@ test "big.int mulWrap single-single signed" {...@@ -1042,7 +1041,6 @@ test "big.int mulWrap single-single signed" {
10421041
1043test "big.int mulWrap multi-multi unsigned" {1042test "big.int mulWrap multi-multi unsigned" {
1044 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;1043 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1045 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
10461044
1047 var op1: u256 = 0x998888efefefefefefefef;1045 var op1: u256 = 0x998888efefefefefefefef;
1048 var op2: u256 = 0x333000abababababababab;1046 var op2: u256 = 0x333000abababababababab;
lib/std/mem.zig-1
...@@ -635,7 +635,6 @@ test "lessThan" {...@@ -635,7 +635,6 @@ test "lessThan" {
635const backend_can_use_eql_bytes = switch (builtin.zig_backend) {635const backend_can_use_eql_bytes = switch (builtin.zig_backend) {
636 // The SPIR-V backend does not support the optimized path yet.636 // The SPIR-V backend does not support the optimized path yet.
637 .stage2_spirv64 => false,637 .stage2_spirv64 => false,
638 .stage2_x86_64 => !std.Target.x86.featureSetHas(builtin.cpu.features, .avx2),
639 else => true,638 else => true,
640};639};
641640
src/arch/x86_64/CodeGen.zig+141-64
...@@ -5727,30 +5727,13 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5727,30 +5727,13 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5727 const mod = self.bin_file.comp.module.?;5727 const mod = self.bin_file.comp.module.?;
5728 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5728 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5729 const result = result: {5729 const result = result: {
5730 try self.spillEflagsIfOccupied();
5731
5730 const dst_ty = self.typeOfIndex(inst);5732 const dst_ty = self.typeOfIndex(inst);
5731 const src_ty = self.typeOf(ty_op.operand);5733 const src_ty = self.typeOf(ty_op.operand);
5732 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airClz for {}", .{5734 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airClz for {}", .{
5733 src_ty.fmt(mod),5735 src_ty.fmt(mod),
5734 });5736 });
5735 const src_bits: u32 = @intCast(src_ty.bitSize(mod));
5736
5737 const has_lzcnt = self.hasFeature(.lzcnt);
5738 if (src_bits > 64 and !has_lzcnt) {
5739 var callee_buf: ["__clz?i2".len]u8 = undefined;
5740 const result = try self.genCall(.{ .lib = .{
5741 .return_type = .i32_type,
5742 .param_types = &.{src_ty.toIntern()},
5743 .callee = std.fmt.bufPrint(&callee_buf, "__clz{c}i2", .{
5744 intCompilerRtAbiName(src_bits),
5745 }) catch unreachable,
5746 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
5747 if (src_bits < 128) try self.asmRegisterImmediate(
5748 .{ ._, .sub },
5749 result.register,
5750 Immediate.u(128 - src_bits),
5751 );
5752 break :result result;
5753 }
57545737
5755 const src_mcv = try self.resolveInst(ty_op.operand);5738 const src_mcv = try self.resolveInst(ty_op.operand);
5756 const mat_src_mcv = switch (src_mcv) {5739 const mat_src_mcv = switch (src_mcv) {
...@@ -5768,6 +5751,61 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5768,6 +5751,61 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5768 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);5751 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
5769 defer self.register_manager.unlockReg(dst_lock);5752 defer self.register_manager.unlockReg(dst_lock);
57705753
5754 const abi_size: u31 = @intCast(src_ty.abiSize(mod));
5755 const src_bits: u31 = @intCast(src_ty.bitSize(mod));
5756 const has_lzcnt = self.hasFeature(.lzcnt);
5757 if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
5758 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
5759 const extra_bits = abi_size * 8 - src_bits;
5760
5761 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5762 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
5763 defer self.register_manager.unlockReg(index_lock);
5764
5765 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len));
5766 switch (extra_bits) {
5767 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
5768 else => try self.asmRegisterImmediate(
5769 .{ ._, .mov },
5770 dst_reg.to32(),
5771 Immediate.s(@as(i32, extra_bits) - 1),
5772 ),
5773 }
5774 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
5775 try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
5776 const zero = try self.asmJccReloc(.z, undefined);
5777 if (self.hasFeature(.slow_incdec)) {
5778 try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1));
5779 } else {
5780 try self.asmRegister(.{ ._, .dec }, index_reg.to32());
5781 }
5782 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
5783 .base = .{ .frame = src_mcv.load_frame.index },
5784 .mod = .{ .rm = .{
5785 .size = .qword,
5786 .index = index_reg.to64(),
5787 .scale = .@"8",
5788 .disp = src_mcv.load_frame.off,
5789 } },
5790 }, Immediate.u(0));
5791 _ = try self.asmJccReloc(.e, loop);
5792 try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{
5793 .base = .{ .frame = src_mcv.load_frame.index },
5794 .mod = .{ .rm = .{
5795 .size = .qword,
5796 .index = index_reg.to64(),
5797 .scale = .@"8",
5798 .disp = src_mcv.load_frame.off,
5799 } },
5800 });
5801 self.performReloc(zero);
5802 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
5803 try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
5804 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1));
5805 try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
5806 break :result dst_mcv;
5807 }
5808
5771 if (has_lzcnt) {5809 if (has_lzcnt) {
5772 if (src_bits <= 8) {5810 if (src_bits <= 8) {
5773 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);5811 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
...@@ -5785,7 +5823,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5785,7 +5823,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5785 if (extra_bits > 0) {5823 if (extra_bits > 0) {
5786 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });5824 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
5787 }5825 }
5788 } else if (src_bits <= 128) {5826 } else {
5827 assert(src_bits <= 128);
5789 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);5828 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5790 const tmp_mcv = MCValue{ .register = tmp_reg };5829 const tmp_mcv = MCValue{ .register = tmp_reg };
5791 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);5830 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...@@ -5818,12 +5857,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5818,12 +5857,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5818 dst_mcv,5857 dst_mcv,
5819 .{ .immediate = 128 - src_bits },5858 .{ .immediate = 128 - src_bits },
5820 );5859 );
5821 } else return self.fail("TODO airClz of {}", .{src_ty.fmt(mod)});5860 }
5822 break :result dst_mcv;5861 break :result dst_mcv;
5823 }5862 }
58245863
5825 if (src_bits > 64)5864 assert(src_bits <= 64);
5826 return self.fail("TODO airClz of {}", .{src_ty.fmt(mod)});5865 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
5827 if (math.isPowerOfTwo(src_bits)) {5866 if (math.isPowerOfTwo(src_bits)) {
5828 const imm_reg = try self.copyToTmpRegister(dst_ty, .{5867 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
5829 .immediate = src_bits ^ (src_bits - 1),5868 .immediate = src_bits ^ (src_bits - 1),
...@@ -5840,7 +5879,6 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5840,7 +5879,6 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5840 try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });5879 try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });
5841 } else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);5880 } else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
58425881
5843 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
5844 try self.asmCmovccRegisterRegister(5882 try self.asmCmovccRegisterRegister(
5845 .z,5883 .z,
5846 registerAlias(dst_reg, cmov_abi_size),5884 registerAlias(dst_reg, cmov_abi_size),
...@@ -5867,7 +5905,6 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5867,7 +5905,6 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
5867 .{ .register = wide_reg },5905 .{ .register = wide_reg },
5868 );5906 );
58695907
5870 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
5871 try self.asmCmovccRegisterRegister(5908 try self.asmCmovccRegisterRegister(
5872 .nz,5909 .nz,
5873 registerAlias(imm_reg, cmov_abi_size),5910 registerAlias(imm_reg, cmov_abi_size),
...@@ -5886,24 +5923,13 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5886,24 +5923,13 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
5886 const mod = self.bin_file.comp.module.?;5923 const mod = self.bin_file.comp.module.?;
5887 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;5924 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
5888 const result = result: {5925 const result = result: {
5926 try self.spillEflagsIfOccupied();
5927
5889 const dst_ty = self.typeOfIndex(inst);5928 const dst_ty = self.typeOfIndex(inst);
5890 const src_ty = self.typeOf(ty_op.operand);5929 const src_ty = self.typeOf(ty_op.operand);
5891 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airClz for {}", .{5930 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airCtz for {}", .{
5892 src_ty.fmt(mod),5931 src_ty.fmt(mod),
5893 });5932 });
5894 const src_bits: u32 = @intCast(src_ty.bitSize(mod));
5895
5896 const has_bmi = self.hasFeature(.bmi);
5897 if (src_bits > 64 and !has_bmi) {
5898 var callee_buf: ["__ctz?i2".len]u8 = undefined;
5899 break :result try self.genCall(.{ .lib = .{
5900 .return_type = .i32_type,
5901 .param_types = &.{src_ty.toIntern()},
5902 .callee = std.fmt.bufPrint(&callee_buf, "__ctz{c}i2", .{
5903 intCompilerRtAbiName(src_bits),
5904 }) catch unreachable,
5905 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
5906 }
59075933
5908 const src_mcv = try self.resolveInst(ty_op.operand);5934 const src_mcv = try self.resolveInst(ty_op.operand);
5909 const mat_src_mcv = switch (src_mcv) {5935 const mat_src_mcv = switch (src_mcv) {
...@@ -5921,8 +5947,62 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5921,8 +5947,62 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
5921 const dst_lock = self.register_manager.lockReg(dst_reg);5947 const dst_lock = self.register_manager.lockReg(dst_reg);
5922 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);5948 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
59235949
5950 const abi_size: u31 = @intCast(src_ty.abiSize(mod));
5951 const src_bits: u31 = @intCast(src_ty.bitSize(mod));
5952 const has_bmi = self.hasFeature(.bmi);
5953 if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
5954 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
5955 const extra_bits = abi_size * 8 - src_bits;
5956
5957 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5958 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
5959 defer self.register_manager.unlockReg(index_lock);
5960
5961 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1));
5962 switch (extra_bits) {
5963 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
5964 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
5965 else => try self.asmRegisterImmediate(
5966 .{ ._, .mov },
5967 dst_reg.to32(),
5968 Immediate.s(-@as(i32, extra_bits)),
5969 ),
5970 }
5971 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
5972 if (self.hasFeature(.slow_incdec)) {
5973 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
5974 } else {
5975 try self.asmRegister(.{ ._, .inc }, index_reg.to32());
5976 }
5977 try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len));
5978 const zero = try self.asmJccReloc(.nb, undefined);
5979 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
5980 .base = .{ .frame = src_mcv.load_frame.index },
5981 .mod = .{ .rm = .{
5982 .size = .qword,
5983 .index = index_reg.to64(),
5984 .scale = .@"8",
5985 .disp = src_mcv.load_frame.off,
5986 } },
5987 }, Immediate.u(0));
5988 _ = try self.asmJccReloc(.e, loop);
5989 try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{
5990 .base = .{ .frame = src_mcv.load_frame.index },
5991 .mod = .{ .rm = .{
5992 .size = .qword,
5993 .index = index_reg.to64(),
5994 .scale = .@"8",
5995 .disp = src_mcv.load_frame.off,
5996 } },
5997 });
5998 self.performReloc(zero);
5999 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
6000 try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
6001 break :result dst_mcv;
6002 }
6003
5924 const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;6004 const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;
5925 if (self.hasFeature(.bmi)) {6005 if (has_bmi) {
5926 if (src_bits <= 64) {6006 if (src_bits <= 64) {
5927 const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);6007 const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
5928 const masked_mcv = if (extra_bits > 0) masked: {6008 const masked_mcv = if (extra_bits > 0) masked: {
...@@ -5942,7 +6022,8 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5942,7 +6022,8 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
5942 break :masked tmp_mcv;6022 break :masked tmp_mcv;
5943 } else mat_src_mcv;6023 } else mat_src_mcv;
5944 try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv);6024 try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv);
5945 } else if (src_bits <= 128) {6025 } else {
6026 assert(src_bits <= 128);
5946 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);6027 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5947 const tmp_mcv = MCValue{ .register = tmp_reg };6028 const tmp_mcv = MCValue{ .register = tmp_reg };
5948 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);6029 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
...@@ -5970,12 +6051,11 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -5970,12 +6051,11 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
5970 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });6051 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
5971 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, lo_mat_src_mcv);6052 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, lo_mat_src_mcv);
5972 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());6053 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
5973 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});6054 }
5974 break :result dst_mcv;6055 break :result dst_mcv;
5975 }6056 }
59766057
5977 if (src_bits > 64) return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});6058 assert(src_bits <= 64);
5978
5979 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });6059 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
5980 const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);6060 const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
5981 defer self.register_manager.unlockReg(width_lock);6061 defer self.register_manager.unlockReg(width_lock);
...@@ -6158,7 +6238,7 @@ fn genByteSwap(...@@ -6158,7 +6238,7 @@ fn genByteSwap(
6158) !MCValue {6238) !MCValue {
6159 const mod = self.bin_file.comp.module.?;6239 const mod = self.bin_file.comp.module.?;
6160 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;6240 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6161 const have_movbe = self.hasFeature(.movbe);6241 const has_movbe = self.hasFeature(.movbe);
61626242
6163 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail(6243 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail(
6164 "TODO implement genByteSwap for {}",6244 "TODO implement genByteSwap for {}",
...@@ -6206,11 +6286,11 @@ fn genByteSwap(...@@ -6206,11 +6286,11 @@ fn genByteSwap(
6206 for (dst_regs, 0..) |dst_reg, limb_index| {6286 for (dst_regs, 0..) |dst_reg, limb_index| {
6207 if (src_mcv.isMemory()) {6287 if (src_mcv.isMemory()) {
6208 try self.asmRegisterMemory(6288 try self.asmRegisterMemory(
6209 .{ ._, if (have_movbe) .movbe else .mov },6289 .{ ._, if (has_movbe) .movbe else .mov },
6210 dst_reg.to64(),6290 dst_reg.to64(),
6211 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),6291 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),
6212 );6292 );
6213 if (!have_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());6293 if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
6214 } else {6294 } else {
6215 try self.asmRegisterRegister(6295 try self.asmRegisterRegister(
6216 .{ ._, .mov },6296 .{ ._, .mov },
...@@ -6227,9 +6307,8 @@ fn genByteSwap(...@@ -6227,9 +6307,8 @@ fn genByteSwap(
62276307
6228 const temp_regs =6308 const temp_regs =
6229 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);6309 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
6230 const temp_locks = self.register_manager.lockRegs(4, temp_regs);6310 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
6231 defer for (temp_locks) |temp_lock| if (temp_lock) |lock|6311 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
6232 self.register_manager.unlockReg(lock);
62336312
6234 const dst_mcv = try self.allocRegOrMem(inst, false);6313 const dst_mcv = try self.allocRegOrMem(inst, false);
6235 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());6314 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
...@@ -6241,7 +6320,7 @@ fn genByteSwap(...@@ -6241,7 +6320,7 @@ fn genByteSwap(
62416320
6242 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);6321 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
6243 try self.asmRegisterMemory(6322 try self.asmRegisterMemory(
6244 .{ ._, if (have_movbe) .movbe else .mov },6323 .{ ._, if (has_movbe) .movbe else .mov },
6245 temp_regs[2].to64(),6324 temp_regs[2].to64(),
6246 .{6325 .{
6247 .base = .{ .frame = dst_mcv.load_frame.index },6326 .base = .{ .frame = dst_mcv.load_frame.index },
...@@ -6254,7 +6333,7 @@ fn genByteSwap(...@@ -6254,7 +6333,7 @@ fn genByteSwap(
6254 },6333 },
6255 );6334 );
6256 try self.asmRegisterMemory(6335 try self.asmRegisterMemory(
6257 .{ ._, if (have_movbe) .movbe else .mov },6336 .{ ._, if (has_movbe) .movbe else .mov },
6258 temp_regs[3].to64(),6337 temp_regs[3].to64(),
6259 .{6338 .{
6260 .base = .{ .frame = dst_mcv.load_frame.index },6339 .base = .{ .frame = dst_mcv.load_frame.index },
...@@ -6266,7 +6345,7 @@ fn genByteSwap(...@@ -6266,7 +6345,7 @@ fn genByteSwap(
6266 } },6345 } },
6267 },6346 },
6268 );6347 );
6269 if (!have_movbe) {6348 if (!has_movbe) {
6270 try self.asmRegister(.{ ._, .bswap }, temp_regs[2].to64());6349 try self.asmRegister(.{ ._, .bswap }, temp_regs[2].to64());
6271 try self.asmRegister(.{ ._, .bswap }, temp_regs[3].to64());6350 try self.asmRegister(.{ ._, .bswap }, temp_regs[3].to64());
6272 }6351 }
...@@ -6301,7 +6380,7 @@ fn genByteSwap(...@@ -6301,7 +6380,7 @@ fn genByteSwap(
6301 },6380 },
6302 }6381 }
63036382
6304 const dst_mcv: MCValue = if (mem_ok and have_movbe and src_mcv.isRegister())6383 const dst_mcv: MCValue = if (mem_ok and has_movbe and src_mcv.isRegister())
6305 try self.allocRegOrMem(inst, true)6384 try self.allocRegOrMem(inst, true)
6306 else6385 else
6307 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) };6386 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) };
...@@ -6359,8 +6438,8 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {...@@ -6359,8 +6438,8 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
6359 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);6438 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
63606439
6361 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);6440 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6362 const tmp_lock = self.register_manager.lockReg(tmp_reg);6441 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6363 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);6442 defer self.register_manager.unlockReg(tmp_lock);
63646443
6365 const limb_abi_size: u32 = @min(abi_size, 8);6444 const limb_abi_size: u32 = @min(abi_size, 8);
6366 const tmp = registerAlias(tmp_reg, limb_abi_size);6445 const tmp = registerAlias(tmp_reg, limb_abi_size);
...@@ -8067,8 +8146,8 @@ fn genShiftBinOpMir(...@@ -8067,8 +8146,8 @@ fn genShiftBinOpMir(
8067 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);8146 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
80688147
8069 const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);8148 const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
8070 const temp_locks = self.register_manager.lockRegs(4, temp_regs);8149 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
8071 defer for (temp_locks) |temp_lock| if (temp_lock) |lock| self.register_manager.unlockReg(lock);8150 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
80728151
8073 switch (tag[0]) {8152 switch (tag[0]) {
8074 ._l => {8153 ._l => {
...@@ -8857,9 +8936,8 @@ fn genMulDivBinOp(...@@ -8857,9 +8936,8 @@ fn genMulDivBinOp(
88578936
8858 const temp_regs =8937 const temp_regs =
8859 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);8938 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
8860 const temp_locks = self.register_manager.lockRegs(4, temp_regs);8939 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
8861 defer for (temp_locks) |temp_lock| if (temp_lock) |lock|8940 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
8862 self.register_manager.unlockReg(lock);
88638941
8864 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());8942 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
88658943
...@@ -9543,9 +9621,8 @@ fn genBinOp(...@@ -9543,9 +9621,8 @@ fn genBinOp(
9543 .{null} ** 2,9621 .{null} ** 2,
9544 abi.RegisterClass.gp,9622 abi.RegisterClass.gp,
9545 );9623 );
9546 const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs);9624 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
9547 defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock|9625 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);
9548 self.register_manager.unlockReg(lock);
95499626
9550 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{});9627 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{});
9551 break :dst dst_regs;9628 break :dst dst_regs;