authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-04 03:36:04-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-05-08 07:36:19-04:00
log1a261917ce41efb49fe41ea0c6d9083212c17797
treee9e3ef244c9c659b5e5d512e782bb038e3662a16
parent9bea854dc2af293cba1d000b31f6e82d9c431285

x86_64: implement `@ctz` and `@clz` for `u128`

3 files changed, 74 insertions(+), 25 deletions(-)

src/arch/x86_64/CodeGen.zig+70-23
...@@ -3798,19 +3798,38 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3798,19 +3798,38 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void {
37983798
3799 const dst_reg = try self.register_manager.allocReg(inst, gp);3799 const dst_reg = try self.register_manager.allocReg(inst, gp);
3800 const dst_mcv = MCValue{ .register = dst_reg };3800 const dst_mcv = MCValue{ .register = dst_reg };
3801 const dst_lock = self.register_manager.lockReg(dst_reg);3801 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
3802 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);3802 defer self.register_manager.unlockReg(dst_lock);
38033803
3804 const src_bits = src_ty.bitSize(self.target.*);
3804 if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {3805 if (Target.x86.featureSetHas(self.target.cpu.features, .lzcnt)) {
3805 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);3806 if (src_bits <= 64) {
3806 const extra_bits = self.regExtraBits(src_ty);3807 try self.genBinOpMir(.lzcnt, src_ty, dst_mcv, mat_src_mcv);
3807 if (extra_bits > 0) {3808
3808 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });3809 const extra_bits = self.regExtraBits(src_ty);
3809 }3810 if (extra_bits > 0) {
3811 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = extra_bits });
3812 }
3813 } else if (src_bits <= 128) {
3814 const tmp_reg = try self.register_manager.allocReg(null, gp);
3815 const tmp_mcv = MCValue{ .register = tmp_reg };
3816 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3817 defer self.register_manager.unlockReg(tmp_lock);
3818
3819 try self.genBinOpMir(.lzcnt, Type.u64, dst_mcv, mat_src_mcv);
3820 try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
3821 try self.genBinOpMir(.lzcnt, Type.u64, tmp_mcv, mat_src_mcv.address().offset(8).deref());
3822 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
3823
3824 if (src_bits < 128) {
3825 try self.genBinOpMir(.sub, dst_ty, dst_mcv, .{ .immediate = 128 - src_bits });
3826 }
3827 } else return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3810 break :result dst_mcv;3828 break :result dst_mcv;
3811 }3829 }
38123830
3813 const src_bits = src_ty.bitSize(self.target.*);3831 if (src_bits > 64)
3832 return self.fail("TODO airClz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3814 if (math.isPowerOfTwo(src_bits)) {3833 if (math.isPowerOfTwo(src_bits)) {
3815 const imm_reg = try self.copyToTmpRegister(dst_ty, .{3834 const imm_reg = try self.copyToTmpRegister(dst_ty, .{
3816 .immediate = src_bits ^ (src_bits - 1),3835 .immediate = src_bits ^ (src_bits - 1),
...@@ -3870,24 +3889,52 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {...@@ -3870,24 +3889,52 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void {
3870 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);3889 defer if (dst_lock) |lock| self.register_manager.unlockReg(lock);
38713890
3872 if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {3891 if (Target.x86.featureSetHas(self.target.cpu.features, .bmi)) {
3873 const extra_bits = self.regExtraBits(src_ty);3892 if (src_bits <= 64) {
3874 const masked_mcv = if (extra_bits > 0) masked: {3893 const extra_bits = self.regExtraBits(src_ty);
3875 const mask_mcv = MCValue{3894 const masked_mcv = if (extra_bits > 0) masked: {
3876 .immediate = ((@as(u64, 1) << @intCast(u6, extra_bits)) - 1) <<3895 const tmp_mcv = tmp: {
3877 @intCast(u6, src_bits),3896 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0))
3878 };3897 break :tmp src_mcv;
3879 const tmp_mcv = tmp: {3898 try self.genSetReg(dst_reg, src_ty, src_mcv);
3880 if (src_mcv.isImmediate() or self.liveness.operandDies(inst, 0)) break :tmp src_mcv;3899 break :tmp dst_mcv;
3881 try self.genSetReg(dst_reg, src_ty, src_mcv);3900 };
3882 break :tmp dst_mcv;3901 try self.genBinOpMir(
3883 };3902 .@"or",
3884 try self.genBinOpMir(.@"or", src_ty, tmp_mcv, mask_mcv);3903 src_ty,
3885 break :masked tmp_mcv;3904 tmp_mcv,
3886 } else mat_src_mcv;3905 .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(u6, 64 - extra_bits)) <<
3887 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);3906 @intCast(u6, src_bits) },
3907 );
3908 break :masked tmp_mcv;
3909 } else mat_src_mcv;
3910 try self.genBinOpMir(.tzcnt, src_ty, dst_mcv, masked_mcv);
3911 } else if (src_bits <= 128) {
3912 const tmp_reg = try self.register_manager.allocReg(null, gp);
3913 const tmp_mcv = MCValue{ .register = tmp_reg };
3914 const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg);
3915 defer self.register_manager.unlockReg(tmp_lock);
3916
3917 const masked_mcv = if (src_bits < 128) masked: {
3918 try self.genCopy(Type.u64, dst_mcv, mat_src_mcv.address().offset(8).deref());
3919 try self.genBinOpMir(
3920 .@"or",
3921 Type.u64,
3922 dst_mcv,
3923 .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(u6, src_bits - 64) },
3924 );
3925 break :masked dst_mcv;
3926 } else mat_src_mcv.address().offset(8).deref();
3927 try self.genBinOpMir(.tzcnt, Type.u64, dst_mcv, masked_mcv);
3928 try self.genBinOpMir(.add, dst_ty, dst_mcv, .{ .immediate = 64 });
3929 try self.genBinOpMir(.tzcnt, Type.u64, tmp_mcv, mat_src_mcv);
3930 try self.asmCmovccRegisterRegister(dst_reg.to32(), tmp_reg.to32(), .nc);
3931 } else return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3888 break :result dst_mcv;3932 break :result dst_mcv;
3889 }3933 }
38903934
3935 if (src_bits > 64)
3936 return self.fail("TODO airCtz of {}", .{src_ty.fmt(self.bin_file.options.module.?)});
3937
3891 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });3938 const width_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits });
3892 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);3939 try self.genBinOpMir(.bsf, src_ty, dst_mcv, mat_src_mcv);
38933940
test/behavior/bugs/2114.zig+2-1
...@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {...@@ -9,7 +9,8 @@ fn ctz(x: anytype) usize {
99
10test "fixed" {10test "fixed" {
11 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO11 if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO
12 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO12 if (builtin.zig_backend == .stage2_x86_64 and
13 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .bmi)) return error.SkipZigTest; // TODO
13 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO14 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
14 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO15 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
15 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO16 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO
test/behavior/math.zig+2-1
...@@ -77,7 +77,8 @@ fn testClz() !void {...@@ -77,7 +77,8 @@ fn testClz() !void {
77}77}
7878
79test "@clz big ints" {79test "@clz big ints" {
80 if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO80 if (builtin.zig_backend == .stage2_x86_64 and
81 !comptime std.Target.x86.featureSetHas(builtin.cpu.features, .lzcnt)) return error.SkipZigTest; // TODO
81 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO82 if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO
82 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO83 if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO
83 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO84 if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO