authorgravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-25 00:05:54-04:00
committergravatar for jacobly@ziglang.orgJacob Young <jacobly@ziglang.org> 2023-03-25 16:23:55-04:00
log3ece9758574e600c1d97add143c24fefed3d2d82
treefa316053e3ac54dcb8a4960550f6aaa31cf1848c
parent37f6f7990e882ac9869513cf920f8cd0c4850039

x86_64: implement saturating arithmetic

3 files changed, 175 insertions(+), 45 deletions(-)

src/arch/x86_64/CodeGen.zig+160-39
......@@ -1550,28 +1550,161 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void {
15501550
15511551fn airAddSat(self: *Self, inst: Air.Inst.Index) !void {
15521552 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1553 const result: MCValue = if (self.liveness.isUnused(inst))
1554 .dead
1555 else
1556 return self.fail("TODO implement add_sat for {}", .{self.target.cpu.arch});
1553 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1554 const ty = self.air.typeOf(bin_op.lhs);
1555
1556 const lhs_mcv = try self.resolveInst(bin_op.lhs);
1557 const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
1558 lhs_mcv
1559 else
1560 try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv);
1561 const dst_reg = dst_mcv.register;
1562 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1563 defer self.register_manager.unlockReg(dst_lock);
1564
1565 const rhs_mcv = try self.resolveInst(bin_op.rhs);
1566 const rhs_lock = switch (rhs_mcv) {
1567 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1568 else => null,
1569 };
1570 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1571
1572 const limit_reg = try self.register_manager.allocReg(null, gp);
1573 const limit_mcv = MCValue{ .register = limit_reg };
1574 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
1575 defer self.register_manager.unlockReg(limit_lock);
1576
1577 const reg_bits = self.regBitSize(ty);
1578 const cc: Condition = if (ty.isSignedInt()) cc: {
1579 try self.genSetReg(ty, limit_reg, dst_mcv);
1580 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1581 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1582 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1583 });
1584 break :cc .o;
1585 } else cc: {
1586 try self.genSetReg(ty, limit_reg, .{
1587 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1588 });
1589 break :cc .c;
1590 };
1591 try self.genBinOpMir(.add, ty, dst_mcv, rhs_mcv);
1592
1593 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1594 try self.asmCmovccRegisterRegister(
1595 registerAlias(dst_reg, abi_size),
1596 registerAlias(limit_reg, abi_size),
1597 cc,
1598 );
1599 break :result dst_mcv;
1600 };
15571601 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
15581602}
15591603
15601604fn airSubSat(self: *Self, inst: Air.Inst.Index) !void {
15611605 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1562 const result: MCValue = if (self.liveness.isUnused(inst))
1563 .dead
1564 else
1565 return self.fail("TODO implement sub_sat for {}", .{self.target.cpu.arch});
1606 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1607 const ty = self.air.typeOf(bin_op.lhs);
1608
1609 const lhs_mcv = try self.resolveInst(bin_op.lhs);
1610 const dst_mcv = if (lhs_mcv.isRegister() and self.reuseOperand(inst, bin_op.lhs, 0, lhs_mcv))
1611 lhs_mcv
1612 else
1613 try self.copyToRegisterWithInstTracking(inst, ty, lhs_mcv);
1614 const dst_reg = dst_mcv.register;
1615 const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg);
1616 defer self.register_manager.unlockReg(dst_lock);
1617
1618 const rhs_mcv = try self.resolveInst(bin_op.rhs);
1619 const rhs_lock = switch (rhs_mcv) {
1620 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1621 else => null,
1622 };
1623 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1624
1625 const limit_reg = try self.register_manager.allocReg(null, gp);
1626 const limit_mcv = MCValue{ .register = limit_reg };
1627 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
1628 defer self.register_manager.unlockReg(limit_lock);
1629
1630 const reg_bits = self.regBitSize(ty);
1631 const cc: Condition = if (ty.isSignedInt()) cc: {
1632 try self.genSetReg(ty, limit_reg, dst_mcv);
1633 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1634 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1635 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1636 });
1637 break :cc .o;
1638 } else cc: {
1639 try self.genSetReg(ty, limit_reg, .{ .immediate = 0 });
1640 break :cc .c;
1641 };
1642 try self.genBinOpMir(.sub, ty, dst_mcv, rhs_mcv);
1643
1644 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1645 try self.asmCmovccRegisterRegister(
1646 registerAlias(dst_reg, abi_size),
1647 registerAlias(limit_reg, abi_size),
1648 cc,
1649 );
1650 break :result dst_mcv;
1651 };
15661652 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
15671653}
15681654
15691655fn airMulSat(self: *Self, inst: Air.Inst.Index) !void {
15701656 const bin_op = self.air.instructions.items(.data)[inst].bin_op;
1571 const result: MCValue = if (self.liveness.isUnused(inst))
1572 .dead
1573 else
1574 return self.fail("TODO implement mul_sat for {}", .{self.target.cpu.arch});
1657 const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: {
1658 const ty = self.air.typeOf(bin_op.lhs);
1659
1660 try self.spillRegisters(&.{ .rax, .rdx });
1661 const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
1662 defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
1663
1664 const lhs_mcv = try self.resolveInst(bin_op.lhs);
1665 const lhs_lock = switch (lhs_mcv) {
1666 .register => |reg| self.register_manager.lockRegAssumeUnused(reg),
1667 else => null,
1668 };
1669 defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock);
1670
1671 const rhs_mcv = try self.resolveInst(bin_op.rhs);
1672 const rhs_lock = switch (rhs_mcv) {
1673 .register => |reg| self.register_manager.lockReg(reg),
1674 else => null,
1675 };
1676 defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock);
1677
1678 const limit_reg = try self.register_manager.allocReg(null, gp);
1679 const limit_mcv = MCValue{ .register = limit_reg };
1680 const limit_lock = self.register_manager.lockRegAssumeUnused(limit_reg);
1681 defer self.register_manager.unlockReg(limit_lock);
1682
1683 const reg_bits = self.regBitSize(ty);
1684 const cc: Condition = if (ty.isSignedInt()) cc: {
1685 try self.genSetReg(ty, limit_reg, lhs_mcv);
1686 try self.genBinOpMir(.xor, ty, limit_mcv, rhs_mcv);
1687 try self.genBinOpMir(.sar, ty, limit_mcv, .{ .immediate = reg_bits - 1 });
1688 try self.genBinOpMir(.xor, ty, limit_mcv, .{
1689 .immediate = (@as(u64, 1) << @intCast(u6, reg_bits - 1)) - 1,
1690 });
1691 break :cc .o;
1692 } else cc: {
1693 try self.genSetReg(ty, limit_reg, .{
1694 .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(u6, 64 - reg_bits),
1695 });
1696 break :cc .c;
1697 };
1698
1699 const dst_mcv = try self.genMulDivBinOp(.mul, inst, ty, lhs_mcv, rhs_mcv);
1700 const abi_size = @intCast(u32, @max(ty.abiSize(self.target.*), 2));
1701 try self.asmCmovccRegisterRegister(
1702 registerAlias(dst_mcv.register, abi_size),
1703 registerAlias(limit_reg, abi_size),
1704 cc,
1705 );
1706 break :result dst_mcv;
1707 };
15751708 return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none });
15761709}
15771710
......@@ -3915,7 +4048,8 @@ fn genMulDivBinOp(
39154048 if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) {
39164049 return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
39174050 }
3918 if (ty.abiSize(self.target.*) > 8) {
4051 const abi_size = @intCast(u32, ty.abiSize(self.target.*));
4052 if (abi_size > 8) {
39194053 return self.fail("TODO implement genMulDivBinOp for {}", .{ty.fmtDebug()});
39204054 }
39214055 if (tag == .div_float) {
......@@ -3925,10 +4059,8 @@ fn genMulDivBinOp(
39254059 assert(self.register_manager.isRegFree(.rax));
39264060 assert(self.register_manager.isRegFree(.rdx));
39274061
3928 const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx });
3929 defer for (reg_locks) |reg| {
3930 self.register_manager.unlockReg(reg);
3931 };
4062 const reg_locks = self.register_manager.lockRegs(2, .{ .rax, .rdx });
4063 defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock);
39324064
39334065 const int_info = ty.intInfo(self.target.*);
39344066 const signedness = int_info.signedness;
......@@ -3953,35 +4085,24 @@ fn genMulDivBinOp(
39534085
39544086 const mir_tag: Mir.Inst.Tag = switch (signedness) {
39554087 .signed => switch (tag) {
3956 .mul, .mulwrap => Mir.Inst.Tag.imul,
3957 .div_trunc, .div_exact, .rem => Mir.Inst.Tag.idiv,
4088 .mul, .mulwrap => .imul,
4089 .div_trunc, .div_exact, .rem => .idiv,
39584090 else => unreachable,
39594091 },
39604092 .unsigned => switch (tag) {
3961 .mul, .mulwrap => Mir.Inst.Tag.mul,
3962 .div_trunc, .div_exact, .rem => Mir.Inst.Tag.div,
4093 .mul, .mulwrap => .mul,
4094 .div_trunc, .div_exact, .rem => .div,
39634095 else => unreachable,
39644096 },
39654097 };
39664098
39674099 try self.genIntMulDivOpMir(mir_tag, ty, .signed, lhs, rhs);
39684100
3969 switch (signedness) {
3970 .signed => switch (tag) {
3971 .mul, .mulwrap, .div_trunc, .div_exact => return MCValue{ .register = .rax },
3972 .rem => return MCValue{ .register = .rdx },
3973 else => unreachable,
3974 },
3975 .unsigned => switch (tag) {
3976 .mul, .mulwrap, .div_trunc, .div_exact => return MCValue{
3977 .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))),
3978 },
3979 .rem => return MCValue{
3980 .register = registerAlias(.rdx, @intCast(u32, ty.abiSize(self.target.*))),
3981 },
3982 else => unreachable,
3983 },
3984 }
4101 return .{ .register = registerAlias(switch (tag) {
4102 .mul, .mulwrap, .div_trunc, .div_exact => .rax,
4103 .rem => .rdx,
4104 else => unreachable,
4105 }, abi_size) };
39854106 },
39864107
39874108 .mod => {
......@@ -3998,14 +4119,14 @@ fn genMulDivBinOp(
39984119 const result: MCValue = if (maybe_inst) |inst|
39994120 try self.copyToRegisterWithInstTracking(inst, ty, lhs)
40004121 else
4001 MCValue{ .register = try self.copyToTmpRegister(ty, lhs) };
4122 .{ .register = try self.copyToTmpRegister(ty, lhs) };
40024123 try self.genBinOpMir(.sub, ty, result, div_floor);
40034124
40044125 return result;
40054126 },
40064127 .unsigned => {
40074128 try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs);
4008 return MCValue{ .register = registerAlias(.rdx, @intCast(u32, ty.abiSize(self.target.*))) };
4129 return .{ .register = registerAlias(.rdx, abi_size) };
40094130 },
40104131 }
40114132 },
src/arch/x86_64/encoder.zig+1-1
......@@ -117,7 +117,7 @@ pub const Instruction = struct {
117117
118118 pub fn new(mnemonic: Mnemonic, args: Init) !Instruction {
119119 const encoding = (try Encoding.findByMnemonic(mnemonic, args)) orelse {
120 log.debug("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
120 log.err("no encoding found for: {s} {s} {s} {s} {s} {s}", .{
121121 @tagName(args.prefix),
122122 @tagName(mnemonic),
123123 @tagName(Encoding.Op.fromOperand(args.op1)),
src/register_manager.zig+14-5
......@@ -149,17 +149,26 @@ pub fn RegisterManager(
149149 return RegisterLock{ .register = reg };
150150 }
151151
152 /// Like `lockReg` but locks multiple registers.
153 pub fn lockRegs(
154 self: *Self,
155 comptime count: comptime_int,
156 regs: [count]Register,
157 ) [count]?RegisterLock {
158 var results: [count]?RegisterLock = undefined;
159 for (&results, regs) |*result, reg| result.* = self.lockReg(reg);
160 return results;
161 }
162
152163 /// Like `lockRegAssumeUnused` but locks multiple registers.
153164 pub fn lockRegsAssumeUnused(
154165 self: *Self,
155166 comptime count: comptime_int,
156167 regs: [count]Register,
157168 ) [count]RegisterLock {
158 var buf: [count]RegisterLock = undefined;
159 for (regs, 0..) |reg, i| {
160 buf[i] = self.lockRegAssumeUnused(reg);
161 }
162 return buf;
169 var results: [count]RegisterLock = undefined;
170 for (&results, regs) |*result, reg| result.* = self.lockRegAssumeUnused(reg);
171 return results;
163172 }
164173
165174 /// Unlocks the register allowing its re-allocation and re-use.