authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-12 00:53:14+01:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2024-02-12 05:25:07+01:00
loge27db373ec276f7839fac0a024b92a0945b5f905
tree5a0e337c41ed5dbcf8729356ce003d2d309ea5b0
parentd89472787353bc69f6a13c80c6f576ee3471c1a8

x86_64: implement `@clz` and `@ctz` of big integers

3 files changed, 141 insertions(+), 67 deletions(-)

lib/std/math/big/int_test.zig-2
......@@ -918,7 +918,6 @@ test "big.int mul multi-single" {
918918
919919test "big.int mul multi-multi" {
920920 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
921 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
922921
923922 var op1: u256 = 0x998888efefefefefefefef;
924923 var op2: u256 = 0x333000abababababababab;
......@@ -1042,7 +1041,6 @@ test "big.int mulWrap single-single signed" {
10421041
10431042test "big.int mulWrap multi-multi unsigned" {
10441043 if (builtin.zig_backend == .stage2_c) return error.SkipZigTest;
1045 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest;
10461044
10471045 var op1: u256 = 0x998888efefefefefefefef;
10481046 var op2: u256 = 0x333000abababababababab;
lib/std/mem.zig-1
......@@ -635,7 +635,6 @@ test "lessThan" {
635635const backend_can_use_eql_bytes = switch (builtin.zig_backend) {
636636 // The SPIR-V backend does not support the optimized path yet.
637637 .stage2_spirv64 => false,
638 .stage2_x86_64 => !std.Target.x86.featureSetHas(builtin.cpu.features, .avx2),
639638 else => true,
640639};
641640
src/arch/x86_64/CodeGen.zig+141-64
......@@ -5727,30 +5727,13 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
57275727 const mod = self.bin_file.comp.module.?;
57285728 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
57295729 const result = result: {
5730 try self.spillEflagsIfOccupied();
5731
57305732 const dst_ty = self.typeOfIndex(inst);
57315733 const src_ty = self.typeOf(ty_op.operand);
57325734 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airClz for {}", .{
57335735 src_ty.fmt(mod),
57345736 });
5735 const src_bits: u32 = @intCast(src_ty.bitSize(mod));
5736
5737 const has_lzcnt = self.hasFeature(.lzcnt);
5738 if (src_bits > 64 and !has_lzcnt) {
5739 var callee_buf: ["__clz?i2".len]u8 = undefined;
5740 const result = try self.genCall(.{ .lib = .{
5741 .return_type = .i32_type,
5742 .param_types = &.{src_ty.toIntern()},
5743 .callee = std.fmt.bufPrint(&callee_buf, "__clz{c}i2", .{
5744 intCompilerRtAbiName(src_bits),
5745 }) catch unreachable,
5746 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
5747 if (src_bits < 128) try self.asmRegisterImmediate(
5748 .{ ._, .sub },
5749 result.register,
5750 Immediate.u(128 - src_bits),
5751 );
5752 break :result result;
5753 }
57545737
57555738 const src_mcv = try self.resolveInst(ty_op.operand);
57565739 const mat_src_mcv = switch (src_mcv) {
......@@ -5768,6 +5751,61 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
57685751 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
57695752 defer self.register_manager.unlockReg(dst_lock);
57705753
5754 const abi_size: u31 = @intCast(src_ty.abiSize(mod));
5755 const src_bits: u31 = @intCast(src_ty.bitSize(mod));
5756 const has_lzcnt = self.hasFeature(.lzcnt);
5757 if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) {
5758 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
5759 const extra_bits = abi_size * 8 - src_bits;
5760
5761 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5762 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
5763 defer self.register_manager.unlockReg(index_lock);
5764
5765 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len));
5766 switch (extra_bits) {
5767 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
5768 else => try self.asmRegisterImmediate(
5769 .{ ._, .mov },
5770 dst_reg.to32(),
5771 Immediate.s(@as(i32, extra_bits) - 1),
5772 ),
5773 }
5774 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
5775 try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32());
5776 const zero = try self.asmJccReloc(.z, undefined);
5777 if (self.hasFeature(.slow_incdec)) {
5778 try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1));
5779 } else {
5780 try self.asmRegister(.{ ._, .dec }, index_reg.to32());
5781 }
5782 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
5783 .base = .{ .frame = src_mcv.load_frame.index },
5784 .mod = .{ .rm = .{
5785 .size = .qword,
5786 .index = index_reg.to64(),
5787 .scale = .@"8",
5788 .disp = src_mcv.load_frame.off,
5789 } },
5790 }, Immediate.u(0));
5791 _ = try self.asmJccReloc(.e, loop);
5792 try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{
5793 .base = .{ .frame = src_mcv.load_frame.index },
5794 .mod = .{ .rm = .{
5795 .size = .qword,
5796 .index = index_reg.to64(),
5797 .scale = .@"8",
5798 .disp = src_mcv.load_frame.off,
5799 } },
5800 });
5801 self.performReloc(zero);
5802 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
5803 try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32());
5804 try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1));
5805 try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32());
5806 break :result dst_mcv;
5807 }
5808
57715809 if (has_lzcnt) {
57725810 if (src_bits <= 8) {
57735811 const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv);
......@@ -5785,7 +5823,8 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
57855823 if (extra_bits > 0) {
57865824 try self.genBinOpMir(.{ ._, .sub }, dst_ty, dst_mcv, .{ .immediate = extra_bits });
57875825 }
5788 } else if (src_bits <= 128) {
5826 } else {
5827 assert(src_bits <= 128);
57895828 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
57905829 const tmp_mcv = MCValue{ .register = tmp_reg };
57915830 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
......@@ -5818,12 +5857,12 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
58185857 dst_mcv,
58195858 .{ .immediate = 128 - src_bits },
58205859 );
5821 } else return self.fail("TODO airClz of {}", .{src_ty.fmt(mod)});
5860 }
58225861 break :result dst_mcv;
58235862 }
58245863
5825 if (src_bits > 64)
5826 return self.fail("TODO airClz of {}", .{src_ty.fmt(mod)});
5864 assert(src_bits <= 64);
5865 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
58275866 if (math.isPowerOfTwo(src_bits)) {
58285867 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
58295868 .immediate = src_bits ^ (src_bits - 1),
......@@ -5840,7 +5879,6 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
58405879 try self.genBinOpMir(.{ ._, .bsr }, Type.u16, dst_mcv, .{ .register = wide_reg });
58415880 } else try self.genBinOpMir(.{ ._, .bsr }, src_ty, dst_mcv, mat_src_mcv);
58425881
5843 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
58445882 try self.asmCmovccRegisterRegister(
58455883 .z,
58465884 registerAlias(dst_reg, cmov_abi_size),
......@@ -5867,7 +5905,6 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
58675905 .{ .register = wide_reg },
58685906 );
58695907
5870 const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(mod))), 2);
58715908 try self.asmCmovccRegisterRegister(
58725909 .nz,
58735910 registerAlias(imm_reg, cmov_abi_size),
......@@ -5886,24 +5923,13 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
58865923 const mod = self.bin_file.comp.module.?;
58875924 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
58885925 const result = result: {
5926 try self.spillEflagsIfOccupied();
5927
58895928 const dst_ty = self.typeOfIndex(inst);
58905929 const src_ty = self.typeOf(ty_op.operand);
5891 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airClz for {}", .{
5930 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail("TODO implement airCtz for {}", .{
58925931 src_ty.fmt(mod),
58935932 });
5894 const src_bits: u32 = @intCast(src_ty.bitSize(mod));
5895
5896 const has_bmi = self.hasFeature(.bmi);
5897 if (src_bits > 64 and !has_bmi) {
5898 var callee_buf: ["__ctz?i2".len]u8 = undefined;
5899 break :result try self.genCall(.{ .lib = .{
5900 .return_type = .i32_type,
5901 .param_types = &.{src_ty.toIntern()},
5902 .callee = std.fmt.bufPrint(&callee_buf, "__ctz{c}i2", .{
5903 intCompilerRtAbiName(src_bits),
5904 }) catch unreachable,
5905 } }, &.{src_ty}, &.{.{ .air_ref = ty_op.operand }});
5906 }
59075933
59085934 const src_mcv = try self.resolveInst(ty_op.operand);
59095935 const mat_src_mcv = switch (src_mcv) {
......@@ -5921,8 +5947,62 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
59215947 const dst_lock = self.register_manager.lockReg(dst_reg);
59225948 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
59235949
5950 const abi_size: u31 = @intCast(src_ty.abiSize(mod));
5951 const src_bits: u31 = @intCast(src_ty.bitSize(mod));
5952 const has_bmi = self.hasFeature(.bmi);
5953 if (src_bits > @as(u32, if (has_bmi) 128 else 64)) {
5954 const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable;
5955 const extra_bits = abi_size * 8 - src_bits;
5956
5957 const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
5958 const index_lock = self.register_manager.lockRegAssumeUnused(index_reg);
5959 defer self.register_manager.unlockReg(index_lock);
5960
5961 try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1));
5962 switch (extra_bits) {
5963 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()),
5964 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()),
5965 else => try self.asmRegisterImmediate(
5966 .{ ._, .mov },
5967 dst_reg.to32(),
5968 Immediate.s(-@as(i32, extra_bits)),
5969 ),
5970 }
5971 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
5972 if (self.hasFeature(.slow_incdec)) {
5973 try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1));
5974 } else {
5975 try self.asmRegister(.{ ._, .inc }, index_reg.to32());
5976 }
5977 try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len));
5978 const zero = try self.asmJccReloc(.nb, undefined);
5979 try self.asmMemoryImmediate(.{ ._, .cmp }, .{
5980 .base = .{ .frame = src_mcv.load_frame.index },
5981 .mod = .{ .rm = .{
5982 .size = .qword,
5983 .index = index_reg.to64(),
5984 .scale = .@"8",
5985 .disp = src_mcv.load_frame.off,
5986 } },
5987 }, Immediate.u(0));
5988 _ = try self.asmJccReloc(.e, loop);
5989 try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{
5990 .base = .{ .frame = src_mcv.load_frame.index },
5991 .mod = .{ .rm = .{
5992 .size = .qword,
5993 .index = index_reg.to64(),
5994 .scale = .@"8",
5995 .disp = src_mcv.load_frame.off,
5996 } },
5997 });
5998 self.performReloc(zero);
5999 try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6));
6000 try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32());
6001 break :result dst_mcv;
6002 }
6003
59246004 const wide_ty = if (src_bits <= 8) Type.u16 else src_ty;
5925 if (self.hasFeature(.bmi)) {
6005 if (has_bmi) {
59266006 if (src_bits <= 64) {
59276007 const extra_bits = self.regExtraBits(src_ty) + @as(u64, if (src_bits <= 8) 8 else 0);
59286008 const masked_mcv = if (extra_bits > 0) masked: {
......@@ -5942,7 +6022,8 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
59426022 break :masked tmp_mcv;
59436023 } else mat_src_mcv;
59446024 try self.genBinOpMir(.{ ._, .tzcnt }, wide_ty, dst_mcv, masked_mcv);
5945 } else if (src_bits <= 128) {
6025 } else {
6026 assert(src_bits <= 128);
59466027 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
59476028 const tmp_mcv = MCValue{ .register = tmp_reg };
59486029 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
......@@ -5970,12 +6051,11 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
59706051 try self.genBinOpMir(.{ ._, .add }, dst_ty, dst_mcv, .{ .immediate = 64 });
59716052 try self.genBinOpMir(.{ ._, .tzcnt }, Type.u64, tmp_mcv, lo_mat_src_mcv);
59726053 try self.asmCmovccRegisterRegister(.nc, dst_reg.to32(), tmp_reg.to32());
5973 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});
6054 }
59746055 break :result dst_mcv;
59756056 }
59766057
5977 if (src_bits > 64) return self.fail("TODO airCtz of {}", .{src_ty.fmt(mod)});
5978
6058 assert(src_bits <= 64);
59796059 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
59806060 const width_lock = self.register_manager.lockRegAssumeUnused(width_reg);
59816061 defer self.register_manager.unlockReg(width_lock);
......@@ -6158,7 +6238,7 @@ fn genByteSwap(
61586238) !MCValue {
61596239 const mod = self.bin_file.comp.module.?;
61606240 const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op;
6161 const have_movbe = self.hasFeature(.movbe);
6241 const has_movbe = self.hasFeature(.movbe);
61626242
61636243 if (src_ty.zigTypeTag(mod) == .Vector) return self.fail(
61646244 "TODO implement genByteSwap for {}",
......@@ -6206,11 +6286,11 @@ fn genByteSwap(
62066286 for (dst_regs, 0..) |dst_reg, limb_index| {
62076287 if (src_mcv.isMemory()) {
62086288 try self.asmRegisterMemory(
6209 .{ ._, if (have_movbe) .movbe else .mov },
6289 .{ ._, if (has_movbe) .movbe else .mov },
62106290 dst_reg.to64(),
62116291 try src_mcv.address().offset(@intCast(limb_index * 8)).deref().mem(self, .qword),
62126292 );
6213 if (!have_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
6293 if (!has_movbe) try self.asmRegister(.{ ._, .bswap }, dst_reg.to64());
62146294 } else {
62156295 try self.asmRegisterRegister(
62166296 .{ ._, .mov },
......@@ -6227,9 +6307,8 @@ fn genByteSwap(
62276307
62286308 const temp_regs =
62296309 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
6230 const temp_locks = self.register_manager.lockRegs(4, temp_regs);
6231 defer for (temp_locks) |temp_lock| if (temp_lock) |lock|
6232 self.register_manager.unlockReg(lock);
6310 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
6311 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
62336312
62346313 const dst_mcv = try self.allocRegOrMem(inst, false);
62356314 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
......@@ -6241,7 +6320,7 @@ fn genByteSwap(
62416320
62426321 const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len);
62436322 try self.asmRegisterMemory(
6244 .{ ._, if (have_movbe) .movbe else .mov },
6323 .{ ._, if (has_movbe) .movbe else .mov },
62456324 temp_regs[2].to64(),
62466325 .{
62476326 .base = .{ .frame = dst_mcv.load_frame.index },
......@@ -6254,7 +6333,7 @@ fn genByteSwap(
62546333 },
62556334 );
62566335 try self.asmRegisterMemory(
6257 .{ ._, if (have_movbe) .movbe else .mov },
6336 .{ ._, if (has_movbe) .movbe else .mov },
62586337 temp_regs[3].to64(),
62596338 .{
62606339 .base = .{ .frame = dst_mcv.load_frame.index },
......@@ -6266,7 +6345,7 @@ fn genByteSwap(
62666345 } },
62676346 },
62686347 );
6269 if (!have_movbe) {
6348 if (!has_movbe) {
62706349 try self.asmRegister(.{ ._, .bswap }, temp_regs[2].to64());
62716350 try self.asmRegister(.{ ._, .bswap }, temp_regs[3].to64());
62726351 }
......@@ -6301,7 +6380,7 @@ fn genByteSwap(
63016380 },
63026381 }
63036382
6304 const dst_mcv: MCValue = if (mem_ok and have_movbe and src_mcv.isRegister())
6383 const dst_mcv: MCValue = if (mem_ok and has_movbe and src_mcv.isRegister())
63056384 try self.allocRegOrMem(inst, true)
63066385 else
63076386 .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.gp) };
......@@ -6359,8 +6438,8 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void {
63596438 defer for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock);
63606439
63616440 const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp);
6362 const tmp_lock = self.register_manager.lockReg(tmp_reg);
6363 defer if (tmp_lock) |lock| self.register_manager.unlockReg(lock);
6441 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
6442 defer self.register_manager.unlockReg(tmp_lock);
63646443
63656444 const limb_abi_size: u32 = @min(abi_size, 8);
63666445 const tmp = registerAlias(tmp_reg, limb_abi_size);
......@@ -8067,8 +8146,8 @@ fn genShiftBinOpMir(
80678146 defer if (rcx_lock) |lock| self.register_manager.unlockReg(lock);
80688147
80698148 const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
8070 const temp_locks = self.register_manager.lockRegs(4, temp_regs);
8071 defer for (temp_locks) |temp_lock| if (temp_lock) |lock| self.register_manager.unlockReg(lock);
8149 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
8150 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
80728151
80738152 switch (tag[0]) {
80748153 ._l => {
......@@ -8857,9 +8936,8 @@ fn genMulDivBinOp(
88578936
88588937 const temp_regs =
88598938 try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp);
8860 const temp_locks = self.register_manager.lockRegs(4, temp_regs);
8861 defer for (temp_locks) |temp_lock| if (temp_lock) |lock|
8862 self.register_manager.unlockReg(lock);
8939 const temp_locks = self.register_manager.lockRegsAssumeUnused(4, temp_regs);
8940 defer for (temp_locks) |lock| self.register_manager.unlockReg(lock);
88638941
88648942 try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32());
88658943
......@@ -9543,9 +9621,8 @@ fn genBinOp(
95439621 .{null} ** 2,
95449622 abi.RegisterClass.gp,
95459623 );
9546 const dst_regs_locks = self.register_manager.lockRegs(2, dst_regs);
9547 defer for (dst_regs_locks) |dst_lock| if (dst_lock) |lock|
9548 self.register_manager.unlockReg(lock);
9624 const dst_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dst_regs);
9625 defer for (dst_regs_locks) |lock| self.register_manager.unlockReg(lock);
95499626
95509627 try self.genCopy(lhs_ty, .{ .register_pair = dst_regs }, dst_mcv, .{});
95519628 break :dst dst_regs;