authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-25 00:05:54-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-25 16:23:55-04:00
log3ece9758574e600c1d97add143c24fefed3d2d82
treefa316053e3ac54dcb8a4960550f6aaa31cf1848c
parent37f6f7990e882ac9869513cf920f8cd0c4850039

x86_64: implement saturating arithmetic

3 files changed, 175 insertions(+), 45 deletions(-)

src/arch/x86_64/CodeGen.zig+160-39
...@@ -1550,28 +1550,161 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {...@@ -1550,28 +1550,161 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
15501550
1551fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {1551fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
1552 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1552 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1553 const result: MCValue = if (self.liveness.isUnused(inst))1553 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1554 .dead1554 const ty = self.air.typeOf(bin_op.lhs);
1555 else1555
1556 return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});1556 const lhs_mcv = try self.resolveInst(bin_op.lhs);
1557 const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
1558 lhs_mcv
1559 else
1560 try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv);
1561 const dst_reg = dst_mcv.register;
1562 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1563 defer self.register_manager.unlockReg(dst_lock);
1564
1565 const rhs_mcv = try self.resolveInst(bin_op.rhs);
1566 const rhs_lock = switch (rhs_mcv) {
1567 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1568 else => null,
1569 };
1570 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1571
1572 const limit_reg = try self.register_manager.allocReg(null, gp);
1573 const limit_mcv = MCValue{ .register = limit_reg };
1574 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
1575 defer self.register_manager.unlockReg(limit_lock);
1576
1577 const reg_bits = self.regBitSize(ty);
1578 const cc: Condition = if (ty.isSignedInt()) cc: {
1579 try self.genSetReg(ty, limit_reg, dst_mcv);
1580 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1581 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1582 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1583 });
1584 break :cc .o;
1585 } else cc: {
1586 try self.genSetReg(ty, limit_reg, .{
1587 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1588 });
1589 break :cc .c;
1590 };
1591 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
1592
1593 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1594 try self.asmCmovccRegisterRegister(
1595 registerAlias(dst_reg, abi_size),
1596 registerAlias(limit_reg, abi_size),
1597 cc,
1598 );
1599 break :result dst_mcv;
1600 };
1557 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1601 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1558}1602}
15591603
1560fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {1604fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
1561 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1605 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1562 const result: MCValue = if (self.liveness.isUnused(inst))1606 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1563 .dead1607 const ty = self.air.typeOf(bin_op.lhs);
1564 else1608
1565 return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});1609 const lhs_mcv = try self.resolveInst(bin_op.lhs);
1610 const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
1611 lhs_mcv
1612 else
1613 try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv);
1614 const dst_reg = dst_mcv.register;
1615 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1616 defer self.register_manager.unlockReg(dst_lock);
1617
1618 const rhs_mcv = try self.resolveInst(bin_op.rhs);
1619 const rhs_lock = switch (rhs_mcv) {
1620 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1621 else => null,
1622 };
1623 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1624
1625 const limit_reg = try self.register_manager.allocReg(null, gp);
1626 const limit_mcv = MCValue{ .register = limit_reg };
1627 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
1628 defer self.register_manager.unlockReg(limit_lock);
1629
1630 const reg_bits = self.regBitSize(ty);
1631 const cc: Condition = if (ty.isSignedInt()) cc: {
1632 try self.genSetReg(ty, limit_reg, dst_mcv);
1633 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1634 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1635 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1636 });
1637 break :cc .o;
1638 } else cc: {
1639 try self.genSetReg(ty, limit_reg, .{ .immediate = 0 });
1640 break :cc .c;
1641 };
1642 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
1643
1644 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1645 try self.asmCmovccRegisterRegister(
1646 registerAlias(dst_reg, abi_size),
1647 registerAlias(limit_reg, abi_size),
1648 cc,
1649 );
1650 break :result dst_mcv;
1651 };
1566 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1652 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1567}1653}
15681654
1569fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {1655fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
1570 const bin_op = self.air.instructions.items(.data)[inst].bin_op;1656 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1571 const result: MCValue = if (self.liveness.isUnused(inst))1657 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1572 .dead1658 const ty = self.air.typeOf(bin_op.lhs);
1573 else1659
1574 return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});1660 try self.spillRegisters(&.{ .rax, .rdx });
1661 const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
1662 defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
1663
1664 const lhs_mcv = try self.resolveInst(bin_op.lhs);
1665 const lhs_lock = switch (lhs_mcv) {
1666 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1667 else => null,
1668 };
1669 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1670
1671 const rhs_mcv = try self.resolveInst(bin_op.rhs);
1672 const rhs_lock = switch (rhs_mcv) {
1673 .register => |reg| self.register_manager.lockReg(reg),
1674 else => null,
1675 };
1676 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1677
1678 const limit_reg = try self.register_manager.allocReg(null, gp);
1679 const limit_mcv = MCValue{ .register = limit_reg };
1680 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
1681 defer self.register_manager.unlockReg(limit_lock);
1682
1683 const reg_bits = self.regBitSize(ty);
1684 const cc: Condition = if (ty.isSignedInt()) cc: {
1685 try self.genSetReg(ty, limit_reg, lhs_mcv);
1686 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1687 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1688 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1689 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1690 });
1691 break :cc .o;
1692 } else cc: {
1693 try self.genSetReg(ty, limit_reg, .{
1694 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1695 });
1696 break :cc .c;
1697 };
1698
1699 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, lhs_mcv, rhs_mcv);
1700 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1701 try self.asmCmovccRegisterRegister(
1702 registerAlias(dst_mcv.register, abi_size),
1703 registerAlias(limit_reg, abi_size),
1704 cc,
1705 );
1706 break :result dst_mcv;
1707 };
1575 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });1708 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
1576}1709}
15771710
...@@ -3915,7 +4048,8 @@ fn genMulDivBinOp(...@@ -3915,7 +4048,8 @@ fn genMulDivBinOp(
3915 if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) {4048 if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) {
3916 return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});4049 return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
3917 }4050 }
3918 if (ty.abiSize(self.target.*) > 8) {4051 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4052 if (abi_size > 8) {
3919 return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});4053 return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
3920 }4054 }
3921 if (tag == .div_float) {4055 if (tag == .div_float) {
...@@ -3925,10 +4059,8 @@ fn genMulDivBinOp(...@@ -3925,10 +4059,8 @@ fn genMulDivBinOp(
3925 assert(self.register_manager.isRegFree(.rax));4059 assert(self.register_manager.isRegFree(.rax));
3926 assert(self.register_manager.isRegFree(.rdx));4060 assert(self.register_manager.isRegFree(.rdx));
39274061
3928 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });4062 const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
3929 defer for (reg_locks) |reg| {4063 defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
3930 self.register_manager.unlockReg(reg);
3931 };
39324064
3933 const int_info = ty.intInfo(self.target.*);4065 const int_info = ty.intInfo(self.target.*);
3934 const signedness = int_info.signedness;4066 const signedness = int_info.signedness;
...@@ -3953,35 +4085,24 @@ fn genMulDivBinOp(...@@ -3953,35 +4085,24 @@ fn genMulDivBinOp(
39534085
3954 const mir_tag: Mir.Inst.Tag = switch (signedness) {4086 const mir_tag: Mir.Inst.Tag = switch (signedness) {
3955 .signed => switch (tag) {4087 .signed => switch (tag) {
3956 .mul, .mulwrap => Mir.Inst.Tag.imul,4088 .mul, .mulwrap => .imul,
3957 .div_trunc, .div_exact, .rem => Mir.Inst.Tag.idiv,4089 .div_trunc, .div_exact, .rem => .idiv,
3958 else => unreachable,4090 else => unreachable,
3959 },4091 },
3960 .unsigned => switch (tag) {4092 .unsigned => switch (tag) {
3961 .mul, .mulwrap => Mir.Inst.Tag.mul,4093 .mul, .mulwrap => .mul,
3962 .div_trunc, .div_exact, .rem => Mir.Inst.Tag.div,4094 .div_trunc, .div_exact, .rem => .div,
3963 else => unreachable,4095 else => unreachable,
3964 },4096 },
3965 };4097 };
39664098
3967 try self.genIntMulDivOpMir(mir_tag, ty, .signed, lhs, rhs);4099 try self.genIntMulDivOpMir(mir_tag, ty, .signed, lhs, rhs);
39684100
3969 switch (signedness) {4101 return .{ .register = registerAlias(switch (tag) {
3970 .signed => switch (tag) {4102 .mul, .mulwrap, .div_trunc, .div_exact => .rax,
3971 .mul, .mulwrap, .div_trunc, .div_exact => return MCValue{ .register = .rax },4103 .rem => .rdx,
3972 .rem => return MCValue{ .register = .rdx },4104 else => unreachable,
3973 else => unreachable,4105 }, abi_size) };
3974 },
3975 .unsigned => switch (tag) {
3976 .mul, .mulwrap, .div_trunc, .div_exact => return MCValue{
3977 .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))),
3978 },
3979 .rem => return MCValue{
3980 .register = registerAlias(.rdx, @intCast(u32, ty.abiSize(self.target.*))),
3981 },
3982 else => unreachable,
3983 },
3984 }
3985 },4106 },
39864107
3987 .mod => {4108 .mod => {
...@@ -3998,14 +4119,14 @@ fn genMulDivBinOp(...@@ -3998,14 +4119,14 @@ fn genMulDivBinOp(
3998 const result: MCValue = if (maybe_inst) |inst|4119 const result: MCValue = if (maybe_inst) |inst|
3999 try self.copyToRegisterWithInstTracking(inst, ty, lhs)4120 try self.copyToRegisterWithInstTracking(inst, ty, lhs)
4000 else4121 else
4001 MCValue{ .register = try self.copyToTmpRegister(ty, lhs) };4122 .{ .register = try self.copyToTmpRegister(ty, lhs) };
4002 try self.genBinOpMir(.sub, ty, result, div_floor);4123 try self.genBinOpMir(.sub, ty, result, div_floor);
40034124
4004 return result;4125 return result;
4005 },4126 },
4006 .unsigned => {4127 .unsigned => {
4007 try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs);4128 try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs);
4008 return MCValue{ .register = registerAlias(.rdx, @intCast(u32, ty.abiSize(self.target.*))) };4129 return .{ .register = registerAlias(.rdx, abi_size) };
4009 },4130 },
4010 }4131 }
4011 },4132 },
src/arch/x86_64/encoder.zig+1-1
...@@ -117,7 +117,7 @@ pub const Instruction = struct {...@@ -117,7 +117,7 @@ pub const Instruction = struct {
117117
118 pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {118 pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {
119 const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {119 const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {
120 log.debug("no encoding found for: {s} {s} {s} {s} {s} {s}", .{120 log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
121 @tagName(args.prefix),121 @tagName(args.prefix),
122 @tagName(mnemonic),122 @tagName(mnemonic),
123 @tagName(Encoding.Op.fromOperand(args.op1)),123 @tagName(Encoding.Op.fromOperand(args.op1)),
src/register_manager.zig+14-5
...@@ -149,17 +149,26 @@ pub fn RegisterManager(...@@ -149,17 +149,26 @@ pub fn RegisterManager(
149 return RegisterLock{ .register = reg };149 return RegisterLock{ .register = reg };
150 }150 }
151151
152 /// Like `lockReg` but locks multiple registers.
153 pub fn lockRegs(
154 self: *Self,
155 comptime count: comptime_int,
156 regs: [count]Register,
157 ) [count]?RegisterLock {
158 var results: [count]?RegisterLock = undefined;
159 for (&results, regs) |*result, reg| result.* = self.lockReg(reg);
160 return results;
161 }
162
152 /// Like `lockRegAssumeUnused` but locks multiple registers.163 /// Like `lockRegAssumeUnused` but locks multiple registers.
153 pub fn lockRegsAssumeUnused(164 pub fn lockRegsAssumeUnused(
154 self: *Self,165 self: *Self,
155 comptime count: comptime_int,166 comptime count: comptime_int,
156 regs: [count]Register,167 regs: [count]Register,
157 ) [count]RegisterLock {168 ) [count]RegisterLock {
158 var buf: [count]RegisterLock = undefined;169 var results: [count]RegisterLock = undefined;
159 for (regs, 0..) |reg, i| {170 for (&results, regs) |*result, reg| result.* = self.lockRegAssumeUnused(reg);
160 buf[i] = self.lockRegAssumeUnused(reg);171 return results;
161 }
162 return buf;
163 }172 }
164173
165 /// Unlocks the register allowing its re-allocation and re-use.174 /// Unlocks the register allowing its re-allocation and re-use.