authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-19 08:26:05-04:00
committergravatar for kubkon@jakubkonka.comJakub Konka <kubkon@jakubkonka.com> 2023-03-21 08:49:54+01:00
log958c8e1ce97671bcd7dc5b2dd9ddc1b87d0d1196
tree070e89325e8f2e807a3d343e78ce5c715f1ec75f
parentf95faac5aeeebe0b77ff7023513cdd3e34b71de1

x86_64: implement @popCount for older processors

This fixes the behavior tests when compiled for baseline.

2 files changed, 95 insertions(+), 11 deletions(-)

src/arch/x86_64/CodeGen.zig+94-11
......@@ -2771,29 +2771,112 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
27712771
27722772fn airPopcount(self: *Self, inst: Air.Inst.Index) !void {
27732773 const ty_op = self.air.instructions.items(.data)[inst].ty_op;
2774 const result = result: {
2774 const result: MCValue = result: {
27752775 if (self.liveness.isUnused(inst)) break :result .dead;
27762776
2777 const op_ty = self.air.typeOf(ty_op.operand);
2777 const src_ty = self.air.typeOf(ty_op.operand);
2778 const src_abi_size = @intCast(u32, src_ty.abiSize(self.target.*));
2779 const src_mcv = try self.resolveInst(ty_op.operand);
27782780
27792781 if (Target.x86.featureSetHas(self.target.cpu.features, .popcnt)) {
2780 const op_mcv = try self.resolveInst(ty_op.operand);
2781 const mat_op_mcv = switch (op_mcv) {
2782 .immediate => MCValue{ .register = try self.copyToTmpRegister(op_ty, op_mcv) },
2783 else => op_mcv,
2782 const mat_src_mcv = switch (src_mcv) {
2783 .immediate => MCValue{ .register = try self.copyToTmpRegister(src_ty, src_mcv) },
2784 else => src_mcv,
27842785 };
2785 const mat_op_lock = switch (mat_op_mcv) {
2786 const mat_src_lock = switch (mat_src_mcv) {
27862787 .register => |reg| self.register_manager.lockReg(reg),
27872788 else => null,
27882789 };
2789 defer if (mat_op_lock) |lock| self.register_manager.unlockReg(lock);
2790 defer if (mat_src_lock) |lock| self.register_manager.unlockReg(lock);
2791
2792 const dst_mcv: MCValue = if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2793 src_mcv
2794 else
2795 .{ .register = try self.register_manager.allocReg(inst, gp) };
27902796
2791 const dst_mcv = MCValue{ .register = try self.register_manager.allocReg(inst, gp) };
2792 try self.genBinOpMir(.popcnt, op_ty, dst_mcv, mat_op_mcv);
2797 const popcnt_ty = if (src_abi_size > 1) src_ty else Type.u16;
2798 try self.genBinOpMir(.popcnt, popcnt_ty, dst_mcv, mat_src_mcv);
27932799 break :result dst_mcv;
27942800 }
27952801
2796 return self.fail("TODO implement airPopcount for {}", .{op_ty.fmt(self.bin_file.options.module.?)});
2802 const mask = @as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - src_abi_size * 8);
2803 const imm_0_1 = Immediate.u(mask / 0b1_1);
2804 const imm_00_11 = Immediate.u(mask / 0b01_01);
2805 const imm_0000_1111 = Immediate.u(mask / 0b0001_0001);
2806 const imm_0000_0001 = Immediate.u(mask / 0b1111_1111);
2807
2808 const tmp_reg = if (src_mcv.isRegister() and self.reuseOperand(inst, ty_op.operand, 0, src_mcv))
2809 src_mcv.register
2810 else
2811 try self.copyToTmpRegister(src_ty, src_mcv);
2812 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
2813 defer self.register_manager.unlockReg(tmp_lock);
2814
2815 const dst_reg = try self.register_manager.allocReg(inst, gp);
2816 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
2817 defer self.register_manager.unlockReg(dst_lock);
2818
2819 {
2820 const dst = registerAlias(dst_reg, src_abi_size);
2821 const tmp = registerAlias(tmp_reg, src_abi_size);
2822 const imm = if (src_abi_size > 4)
2823 try self.register_manager.allocReg(null, gp)
2824 else
2825 undefined;
2826
2827 // tmp = operand
2828 try self.asmRegisterRegister(.mov, dst, tmp);
2829 // dst = operand
2830 try self.asmRegisterImmediate(.shr, tmp, Immediate.u(1));
2831 // tmp = operand >> 1
2832 if (src_abi_size > 4) {
2833 try self.asmRegisterImmediate(.mov, imm, imm_0_1);
2834 try self.asmRegisterRegister(.@"and", tmp, imm);
2835 } else try self.asmRegisterImmediate(.@"and", tmp, imm_0_1);
2836 // tmp = (operand >> 1) & 0x55...55
2837 try self.asmRegisterRegister(.sub, dst, tmp);
2838 // dst = temp1 = operand - ((operand >> 1) & 0x55...55)
2839 try self.asmRegisterRegister(.mov, tmp, dst);
2840 // tmp = temp1
2841 try self.asmRegisterImmediate(.shr, dst, Immediate.u(2));
2842 // dst = temp1 >> 2
2843 if (src_abi_size > 4) {
2844 try self.asmRegisterImmediate(.mov, imm, imm_00_11);
2845 try self.asmRegisterRegister(.@"and", tmp, imm);
2846 try self.asmRegisterRegister(.@"and", dst, imm);
2847 } else {
2848 try self.asmRegisterImmediate(.@"and", tmp, imm_00_11);
2849 try self.asmRegisterImmediate(.@"and", dst, imm_00_11);
2850 }
2851 // tmp = temp1 & 0x33...33
2852 // dst = (temp1 >> 2) & 0x33...33
2853 try self.asmRegisterRegister(.add, tmp, dst);
2854 // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33)
2855 try self.asmRegisterRegister(.mov, dst, tmp);
2856 // dst = temp2
2857 try self.asmRegisterImmediate(.shr, tmp, Immediate.u(4));
2858 // tmp = temp2 >> 4
2859 try self.asmRegisterRegister(.add, dst, tmp);
2860 // dst = temp2 + (temp2 >> 4)
2861 if (src_abi_size > 4) {
2862 try self.asmRegisterImmediate(.mov, imm, imm_0000_1111);
2863 try self.asmRegisterImmediate(.mov, tmp, imm_0000_0001);
2864 try self.asmRegisterRegister(.@"and", dst, imm);
2865 try self.asmRegisterRegister(.imul, dst, tmp);
2866 } else {
2867 try self.asmRegisterImmediate(.@"and", dst, imm_0000_1111);
2868 if (src_abi_size > 1) {
2869 try self.asmRegisterRegisterImmediate(.imul, dst, dst, imm_0000_0001);
2870 }
2871 }
2872 // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f
2873 // dst = temp3 * 0x01...01
2874 if (src_abi_size > 1) {
2875 try self.asmRegisterImmediate(.shr, dst, Immediate.u((src_abi_size - 1) * 8));
2876 }
2877 // dst = (temp3 * 0x01...01) >> (bits - 8)
2878 }
2879 break :result .{ .register = dst_reg };
27972880 };
27982881 return self.finishAir(inst, result, .{ ty_op.operand, .none, .none });
27992882}
src/arch/x86_64/bits.zig+1
......@@ -476,6 +476,7 @@ pub const Memory = union(enum) {
476476 base: ?Register = null,
477477 scale_index: ?ScaleIndex = null,
478478 }) Memory {
479 if (args.scale_index) |si| assert(std.math.isPowerOfTwo(si.scale));
479480 return .{ .sib = .{
480481 .base = args.base,
481482 .disp = args.disp,